From 7a73f987b4fd8a661f12dec0ac9233b4c014e406 Mon Sep 17 00:00:00 2001 From: rodrigosa Date: Sat, 26 Sep 2026 10:26:43 +0100 Subject: [PATCH 1/5] fix: select fuller WHOOP 4 RR source per hour --- .../WhoopStore/Sources/WhoopStore/Reads.swift | 23 +++++---- .../RrSourceChannelTests.swift | 48 ++++++++++++++----- .../src/main/java/com/noop/data/WhoopDao.kt | 16 +++++-- .../java/com/noop/data/Whoop5RRSqliteTest.kt | 22 +++++++++ 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift index 689b503b8d..68fdbe25ab 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift @@ -466,8 +466,9 @@ extension WhoopStore { // One transport for the complete requested interval. Legacy WHOOP 5 rows mix units and // origins, so they remain stored but cannot be converted or spliced into a scored beat train. // This subquery uses the SAME time/suspect predicates as the outer read, before LIMIT. - // A WHOOP 4 takes the history branch below: its type-47 offload is labelled, its standard-BLE - // feed is not, and the two overlap, so one or the other is scored for the whole interval. + // A WHOOP 4 has labelled type-47 history and an unlabelled standard-BLE feed. Choose between + // them per UTC hour, so a partial offload cannot hide live rows from other hours in a broad + // analytics read. Each hour still uses only one transport, preventing overlap double-counts. // Every other source takes the Oura branch: one beat channel for the interval, the fuller one // (see the doc comment on `rrIntervals`), with NULL and non-Oura codes passing untouched. let strictWhoop4History: Bool @@ -485,12 +486,18 @@ extension WhoopStore { WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableWhoop5Channels) AND (tsSuspect IS NULL OR tsSuspect <> 1)) """ : (strictWhoop4History ? """ - ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) - AND srcChannel = :whoop4Historical) - OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) - AND srcChannel IS NULL)) + ((srcChannel = :whoop4Historical AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :d + AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 + AND h.srcChannel = :whoop4Historical AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) + >= (SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :d + AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 + AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1))) + OR (srcChannel IS NULL AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :d + AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 + AND h.srcChannel = :whoop4Historical AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) + < (SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :d + AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 + AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1)))) """ : """ (srcChannel IS NULL OR srcChannel NOT IN \(Self.scorableOuraChannels) OR (srcChannel = 1) = ( SELECT SUM(srcChannel = 1) > SUM(srcChannel <> 1) FROM rrInterval diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift index fb88680f3e..880a647121 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift @@ -18,12 +18,16 @@ import OuraProtocol final class RrSourceChannelTests: XCTestCase { private let ts = 1_750_000_000 - func testWhoop4HistoricalSourceWinsAndPromotesLegacyLiveRows() async throws { - let store = try await WhoopStore.inMemory() - try await store.upsertDevice(id: "strap", mac: nil, name: nil) + private func setRegistry(_ store: WhoopStore) throws { try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + } + + func testWhoop4HistoricalSourceWinsAndPromotesLegacyLiveRows() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") @@ -39,9 +43,7 @@ final class RrSourceChannelTests: XCTestCase { func testWhoop4FallsBackToUnlabelledRowsWhenNoHistoricalDataExists() async throws { let store = try await WhoopStore.inMemory() try await store.upsertDevice(id: "strap", mac: nil, name: nil) - try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( - id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, - capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800)]), deviceId: "strap") let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts, limit: 100) @@ -49,20 +51,42 @@ final class RrSourceChannelTests: XCTestCase { XCTAssertNil(selected.first?.srcChannel) } - func testWhoop4UsesOneHistoricalTransportForTheWholeWindow() async throws { + func testWhoop4UsesTheFillerSourceForTheHour() async throws { let store = try await WhoopStore.inMemory() try await store.upsertDevice(id: "strap", mac: nil, name: nil) - try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( - id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, - capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 805, srcChannel: .whoop4Historical)]), deviceId: "strap") let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts + 1, limit: 100) - XCTAssertEqual(selected.map(\.rrMs), [805]) - XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical]) + XCTAssertEqual(selected.map(\.rrMs), [800, 810]) + XCTAssertEqual(selected.map(\.srcChannel), [nil, nil]) + } + + func testWhoop4PartialHistoryKeepsUnlabelledRowsFromOtherHours() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) + + let base = (ts / 3600) * 3600 + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: base + 10, rrMs: 800), + RRInterval(ts: base + 3600 + 10, rrMs: 820), + RRInterval(ts: base + 3600 + 11, rrMs: 830), + RRInterval(ts: base + 3600 + 12, rrMs: 840), + ]), deviceId: "strap") + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: base + 10, rrMs: 805, srcChannel: .whoop4Historical), + RRInterval(ts: base + 11, rrMs: 815, srcChannel: .whoop4Historical), + RRInterval(ts: base + 3600 + 10, rrMs: 825, srcChannel: .whoop4Historical), + ]), deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: base, + to: base + 7200, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [805, 815, 820, 830, 840]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical, .whoop4Historical, nil, nil, nil]) } // MARK: - The label survives the mapping diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 5a1132f8ad..9b2075c80e 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -139,10 +139,18 @@ internal const val WHOOP4_RR_INTERVALS_SQL = "SELECT * FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + "AND (srcChannel IS NULL OR srcChannel <> 2) " + "AND (tsSuspect IS NULL OR tsSuspect <> 1) " + - "AND ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel = 8) " + - "OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel IS NULL)) " + + "AND ((srcChannel = 8 AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :deviceId " + + "AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 " + + "AND h.srcChannel = 8 AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) >= " + + "(SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :deviceId " + + "AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 " + + "AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1))) " + + "OR (srcChannel IS NULL AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :deviceId " + + "AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 " + + "AND h.srcChannel = 8 AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) < " + + "(SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :deviceId " + + "AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 " + + "AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1)))) " + "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" /** The earliest beat a device has banked AT ALL, labelled or not, or null when it has none. The lower diff --git a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt index f9f9aaa1d2..85d75ef847 100644 --- a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt +++ b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt @@ -173,6 +173,28 @@ class Whoop5RRSqliteTest { registry("5.0 MG") } + @Test fun whoop4PartialHistoryKeepsUnlabelledRowsFromOtherHours() { + val base = 1_750_000_000L / 3600 * 3600 + fun insert(ts: Long, rr: Int, channel: Int?) { + statement("INSERT INTO rrInterval(deviceId,ts,rrMs,seq,synced,ord,srcChannel,tsSuspect) " + + "VALUES(:d,:t,:r,0,0,0,:c,NULL)", + mapOf("d" to id, "t" to ts, "r" to rr, "c" to channel)).use { it.executeUpdate() } + } + insert(base + 10, 800, null) + insert(base + 3600 + 10, 820, null) + insert(base + 3600 + 11, 830, null) + insert(base + 3600 + 12, 840, null) + insert(base + 10, 805, 8) + insert(base + 11, 815, 8) + insert(base + 3600 + 10, 825, 8) + + val selected = query( + WHOOP4_RR_INTERVALS_SQL, + mapOf("deviceId" to id, "from" to base, "to" to base + 7200, "limit" to 100), + ) { it.getInt("rrMs") to it.getInt("srcChannel") } + assertEquals(listOf(805 to 8, 815 to 8, 820 to 0, 830 to 0, 840 to 0), selected) + } + @After fun close() { db.close() } private fun sql(sql: String) { db.createStatement().use { it.execute(sql) } } private fun statement(sql: String, values: Map) = run { From 353d0499b0fa9a1ce9121be90fba625cd5722cbc Mon Sep 17 00:00:00 2001 From: rodrigosa Date: Sat, 26 Sep 2026 10:38:10 +0100 Subject: [PATCH 2/5] test: expect fuller WHOOP 4 source per hour --- android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt index 85d75ef847..607e45febc 100644 --- a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt +++ b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt @@ -704,12 +704,12 @@ class Whoop5RRSqliteTest { assertEquals(listOf(8, 8), rows.map { it.srcChannel }) } - @Test fun whoop4PrefersOneHistoricalTransportForTheWholeWindow() = runBlocking { + @Test fun whoop4UsesTheFillerSourceForTheHour() = runBlocking { registry("4.0") repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(101, 810))), id) repo.insert(StreamBatch(rr = listOf(RrRow(100, 805, RrSourceChannel.WHOOP4_HISTORICAL))), id) val rows = repo.rrIntervalsForDevice(id, 100, 101, 100) - assertEquals(listOf(805), rows.map { it.rrMs }) + assertEquals(listOf(800, 810), rows.map { it.rrMs }) } @Test fun whoop4FallsBackToUnlabelledRowsWhenNoHistoryExists() = runBlocking { From d86b7f9d2a25b2074119298a7a5531d71d68d0e3 Mon Sep 17 00:00:00 2001 From: rodrigosa Date: Sat, 26 Sep 2026 11:59:05 +0100 Subject: [PATCH 3/5] fix: group WHOOP 4 RR source counts by hour --- .../WhoopStore/Sources/WhoopStore/Reads.swift | 67 ++++++++++++------- .../src/main/java/com/noop/data/WhoopDao.kt | 30 ++++----- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift index 68fdbe25ab..c78e43ff3a 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift @@ -485,35 +485,54 @@ extension WhoopStore { srcChannel = (SELECT MIN(srcChannel) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableWhoop5Channels) AND (tsSuspect IS NULL OR tsSuspect <> 1)) - """ : (strictWhoop4History ? """ - ((srcChannel = :whoop4Historical AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :d - AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 - AND h.srcChannel = :whoop4Historical AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) - >= (SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :d - AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 - AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1))) - OR (srcChannel IS NULL AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :d - AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 - AND h.srcChannel = :whoop4Historical AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) - < (SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :d - AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 - AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1)))) """ : """ (srcChannel IS NULL OR srcChannel NOT IN \(Self.scorableOuraChannels) OR (srcChannel = 1) = ( SELECT SUM(srcChannel = 1) > SUM(srcChannel <> 1) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableOuraChannels) AND (tsSuspect IS NULL OR tsSuspect <> 1))) - """) - let rows = try Row.fetchAll(db, sql: """ - SELECT ts, rrMs, srcChannel, ord, seq FROM rrInterval - WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND (srcChannel IS NULL OR srcChannel <> :rrx) - AND \(sourcePredicate) - AND (tsSuspect IS NULL OR tsSuspect <> 1) -- #1073: exclude future-stamped beats - ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :lim - """, arguments: ["d": deviceId, "f": from, "t": to, - "rrx": RRSourceChannel.spo2Ibi.rawValue, - "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, "lim": limit]) + """ + // The WHOOP 4 source decision is per UTC hour. Aggregate the two source counts once per + // hour, then join that small result to the requested beats. Counting the same hour from + // every beat made wide HRV reads quadratic in the number of rows per hour and could leave + // every metric screen appearing to load forever on a long history. + let sql: String + if strictWhoop4History { + sql = """ + WITH whoop4HourCounts AS MATERIALIZED ( + SELECT ts / 3600 AS hour, + SUM(CASE WHEN srcChannel = :whoop4Historical THEN 1 ELSE 0 END) AS historyCount, + SUM(CASE WHEN srcChannel IS NULL THEN 1 ELSE 0 END) AS liveCount + FROM rrInterval + WHERE deviceId = :d + AND ts >= (:f / 3600) * 3600 + AND ts < ((:t / 3600) + 1) * 3600 + AND (tsSuspect IS NULL OR tsSuspect <> 1) + GROUP BY ts / 3600 + ) + SELECT r.ts, r.rrMs, r.srcChannel, r.ord, r.seq FROM rrInterval r + JOIN whoop4HourCounts h ON h.hour = r.ts / 3600 + WHERE r.deviceId = :d AND r.ts >= :f AND r.ts <= :t + AND (r.srcChannel IS NULL OR r.srcChannel <> :rrx) + AND ((r.srcChannel = :whoop4Historical AND h.historyCount >= h.liveCount) + OR (r.srcChannel IS NULL AND h.historyCount < h.liveCount)) + AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) + ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :lim + """ + } else { + sql = """ + SELECT ts, rrMs, srcChannel, ord, seq FROM rrInterval + WHERE deviceId = :d AND ts >= :f AND ts <= :t + AND (srcChannel IS NULL OR srcChannel <> :rrx) + AND \(sourcePredicate) + AND (tsSuspect IS NULL OR tsSuspect <> 1) -- #1073: exclude future-stamped beats + ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :lim + """ + } + let rows = try Row.fetchAll(db, sql: sql, + arguments: ["d": deviceId, "f": from, "t": to, + "rrx": RRSourceChannel.spo2Ibi.rawValue, + "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, + "lim": limit]) .map { row in RRInterval(ts: row["ts"], rrMs: row["rrMs"], srcChannel: (row["srcChannel"] as Int?).flatMap(RRSourceChannel.init(rawValue:)), diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 9b2075c80e..d0a82b3b41 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -136,22 +136,20 @@ internal const val WHOOP5_RR_INTERVALS_SQL = "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" internal const val WHOOP4_RR_INTERVALS_SQL = - "SELECT * FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND (srcChannel IS NULL OR srcChannel <> 2) " + - "AND (tsSuspect IS NULL OR tsSuspect <> 1) " + - "AND ((srcChannel = 8 AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :deviceId " + - "AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 " + - "AND h.srcChannel = 8 AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) >= " + - "(SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :deviceId " + - "AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 " + - "AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1))) " + - "OR (srcChannel IS NULL AND (SELECT COUNT(*) FROM rrInterval h WHERE h.deviceId = :deviceId " + - "AND h.ts >= (rrInterval.ts / 3600) * 3600 AND h.ts < (rrInterval.ts / 3600 + 1) * 3600 " + - "AND h.srcChannel = 8 AND (h.tsSuspect IS NULL OR h.tsSuspect <> 1)) < " + - "(SELECT COUNT(*) FROM rrInterval l WHERE l.deviceId = :deviceId " + - "AND l.ts >= (rrInterval.ts / 3600) * 3600 AND l.ts < (rrInterval.ts / 3600 + 1) * 3600 " + - "AND l.srcChannel IS NULL AND (l.tsSuspect IS NULL OR l.tsSuspect <> 1)))) " + - "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" + "WITH whoop4HourCounts AS (" + + "SELECT ts / 3600 AS hour, " + + "SUM(CASE WHEN srcChannel = 8 THEN 1 ELSE 0 END) AS historyCount, " + + "SUM(CASE WHEN srcChannel IS NULL THEN 1 ELSE 0 END) AS liveCount " + + "FROM rrInterval WHERE deviceId = :deviceId " + + "AND ts >= (:from / 3600) * 3600 AND ts < ((:to / 3600) + 1) * 3600 " + + "AND (tsSuspect IS NULL OR tsSuspect <> 1) GROUP BY ts / 3600) " + + "SELECT r.* FROM rrInterval r JOIN whoop4HourCounts h ON h.hour = r.ts / 3600 " + + "WHERE r.deviceId = :deviceId AND r.ts >= :from AND r.ts <= :to " + + "AND (r.srcChannel IS NULL OR r.srcChannel <> 2) " + + "AND ((r.srcChannel = 8 AND h.historyCount >= h.liveCount) " + + "OR (r.srcChannel IS NULL AND h.historyCount < h.liveCount)) " + + "AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) " + + "ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :limit" /** The earliest beat a device has banked AT ALL, labelled or not, or null when it has none. The lower * bound on the "cannot be scored" explanation: it separates history this strap actually recorded from From f5ca3a7ec408b318e1e82cffe3b64e2fc247d2ae Mon Sep 17 00:00:00 2001 From: rodrigosa Date: Sat, 26 Sep 2026 13:56:24 +0100 Subject: [PATCH 4/5] fix: separate WHOOP 4 RR transports for HRV --- .../Sources/WhoopProtocol/Streams.swift | 11 +++++- .../WhoopProtocolTests/StreamsTests.swift | 12 ++++++ .../WhoopProtocolTests/Whoop5RRTests.swift | 2 +- .../WhoopStore/Sources/WhoopStore/Reads.swift | 37 +++++++++++-------- .../RrSourceChannelTests.swift | 26 ++++++++++--- Strand/Collect/Collector.swift | 6 ++- .../main/java/com/noop/ble/WhoopBleClient.kt | 10 +++-- .../src/main/java/com/noop/data/WhoopDao.kt | 21 ++++++----- .../main/java/com/noop/protocol/Streams.kt | 8 +++- .../java/com/noop/data/HrFingerprintTest.kt | 4 +- .../java/com/noop/data/Whoop5RRSqliteTest.kt | 18 +++++++-- .../java/com/noop/protocol/FramingTest.kt | 4 ++ .../java/com/noop/protocol/Whoop5RRTest.kt | 2 +- 13 files changed, 117 insertions(+), 44 deletions(-) diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift index 12655a2244..5dc5aab43d 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift @@ -37,7 +37,8 @@ public enum StandardHRContact: String, Equatable, Codable, Sendable { /// The sensor channel or transport that produced an R-R interval. /// /// WHOOP 5 exposes one beat train over multiple transports; codes 5–7 distinguish those observations. -/// WHOOP 4 standard-BLE rows and legacy data keep nil; its type-47 historical rows use code 8. An Oura ring has more than one optical channel: +/// WHOOP 4 type-40 realtime and standard-BLE rows use codes 9 and 10; legacy data keeps nil. +/// Its type-47 historical rows use code 8. An Oura ring has more than one optical channel: /// the green-quality tag (0x80) and the SpO2 tag (0x6E) both /// decode to R-R and both were stored, so the table held roughly TWO complete copies of every night — /// not duplicate rows to de-duplicate, but the SAME heartbeats measured twice. Labelling the channel is @@ -73,6 +74,10 @@ public enum RRSourceChannel: Int, Equatable, Codable, Sendable, CaseIterable { /// WHOOP 4.0 type-47 historical R-R intervals. The 4.0 standard BLE feed has no source label, /// so this distinguishes the clock-anchored offload from the overlapping live feed at read time. case whoop4Historical = 8 + /// WHOOP 4.0 type-40 live R-R intervals, timestamped from the strap's realtime record clock. + case whoop4Realtime = 9 + /// WHOOP 4.0 standard BLE 0x2A37 R-R intervals, timestamped on notification receipt. + case whoop4Standard = 10 public var isWhoop5Transport: Bool { (5...7).contains(rawValue) } @@ -826,7 +831,8 @@ private func toWall(_ deviceTs: Int?, _ deviceClockRef: Int, _ wallClockRef: Int /// would double-count HR for the same instants. Frames whose integrity verdict is negative are /// skipped — no row is derived from a frame that is not intact. public func extractStreams(_ parsed: [ParsedFrame], - deviceClockRef: Int, wallClockRef: Int) -> Streams { + deviceClockRef: Int, wallClockRef: Int, + family: DeviceFamily? = nil) -> Streams { var out = Streams() for r in parsed { // `ok` is now the FULL verdict — header checksum, payload CRC32 and structural length — so it @@ -844,6 +850,7 @@ public func extractStreams(_ parsed: [ParsedFrame], // Unlike Python, drop RR rows when timestamp is absent (a ts-less RR row is unstorable). if let ts = ts, let rrs = p["rr_intervals"]?.intArrayValue { let source = p["rr_source_channel"]?.intValue.flatMap(RRSourceChannel.init(rawValue:)) + ?? (family == .whoop4 ? .whoop4Realtime : nil) for rr in rrs { out.rr.append(RRInterval(ts: ts, rrMs: rr, srcChannel: source)) } } case "EVENT": diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift index 97d932921c..866cfd9988 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift @@ -41,6 +41,18 @@ final class StreamsTests: XCTestCase { XCTAssertTrue(s.rr.isEmpty) } + func testRealtimeRRSourceIsTaggedOnlyWhenFamilyIsKnown() { + let frame = ParsedFrame(ok: true, typeName: "REALTIME_DATA", seq: nil, cmdName: nil, + crcOK: true, lenBytes: 0, rawHex: "", fields: [], + parsed: ["timestamp": .int(deviceClockRef), + "rr_intervals": .intArray([800])]) + let tagged = extractStreams([frame], deviceClockRef: deviceClockRef, + wallClockRef: wallClockRef, family: .whoop4) + XCTAssertEqual(tagged.rr.map(\.srcChannel), [.whoop4Realtime]) + let legacy = extractStreams([frame], deviceClockRef: deviceClockRef, wallClockRef: wallClockRef) + XCTAssertNil(legacy.rr.first?.srcChannel) + } + func testEventTimestampIsNotOffset() { let s = extractStreams(parsedFrames([ev]), deviceClockRef: deviceClockRef, wallClockRef: wallClockRef) diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift index 42c8eb9dfc..2479d1b80c 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift @@ -50,7 +50,7 @@ final class Whoop5RRTests: XCTestCase { } func testTransportCodes() { - XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...8)) + XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...10)) } /// The table below is the parity contract for the "this night cannot be scored" explanation. The diff --git a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift index c78e43ff3a..378ac6c844 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift @@ -231,7 +231,7 @@ extension WhoopStore { } let historyCount: Int = row["w5"] let registry: String = row["registry"] - return "v4|h\(hc):\(hm)|" + tails.joined(separator: "|") + return "v5|h\(hc):\(hm)|" + tails.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)|tagged\(row["w5tagged"] as Int)" + "|w4history\(row["w4history"] as Int)|registry\(registry)" } @@ -311,7 +311,7 @@ extension WhoopStore { let historyCount: Int = row["w5"] let registry: String = row["registry"] let strictRR = try Self.isWhoop5RRSource(db: db, deviceId: deviceId) - return "s3|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)" + return "s4|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)" + "|w4h\(row["w4h"] as Int)|ownerTagged\(row["w5owner"] as Int)|registry\(registry)|rr5=\(strictRR)" } } @@ -466,9 +466,9 @@ extension WhoopStore { // One transport for the complete requested interval. Legacy WHOOP 5 rows mix units and // origins, so they remain stored but cannot be converted or spliced into a scored beat train. // This subquery uses the SAME time/suspect predicates as the outer read, before LIMIT. - // A WHOOP 4 has labelled type-47 history and an unlabelled standard-BLE feed. Choose between - // them per UTC hour, so a partial offload cannot hide live rows from other hours in a broad - // analytics read. Each hour still uses only one transport, preventing overlap double-counts. + // A WHOOP 4 has labelled type-47 history, type-40 realtime, standard-BLE, and legacy + // unlabelled rows. Choose one source per UTC hour in provenance order, so overlapping live + // transports are never merged and partial history takes precedence only where it exists. // Every other source takes the Oura branch: one beat channel for the interval, the fuller one // (see the doc comment on `rrIntervals`), with NULL and non-Oura codes passing untouched. let strictWhoop4History: Bool @@ -491,30 +491,35 @@ extension WhoopStore { WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableOuraChannels) AND (tsSuspect IS NULL OR tsSuspect <> 1))) """ - // The WHOOP 4 source decision is per UTC hour. Aggregate the two source counts once per - // hour, then join that small result to the requested beats. Counting the same hour from - // every beat made wide HRV reads quadratic in the number of rows per hour and could leave - // every metric screen appearing to load forever on a long history. + // The WHOOP 4 source decision is per UTC hour. Materialize one choice per hour, then join + // that small result to the requested beats. Counting the same hour from every beat made + // wide HRV reads quadratic in the number of rows per hour and could stall metric screens. let sql: String if strictWhoop4History { sql = """ - WITH whoop4HourCounts AS MATERIALIZED ( + WITH whoop4HourChoice AS MATERIALIZED ( SELECT ts / 3600 AS hour, - SUM(CASE WHEN srcChannel = :whoop4Historical THEN 1 ELSE 0 END) AS historyCount, - SUM(CASE WHEN srcChannel IS NULL THEN 1 ELSE 0 END) AS liveCount + MIN(CASE WHEN srcChannel = :whoop4Historical THEN 1 + WHEN srcChannel = :whoop4Realtime THEN 2 + WHEN srcChannel = :whoop4Standard THEN 3 + WHEN srcChannel IS NULL THEN 4 END) AS sourceChoice FROM rrInterval WHERE deviceId = :d AND ts >= (:f / 3600) * 3600 AND ts < ((:t / 3600) + 1) * 3600 AND (tsSuspect IS NULL OR tsSuspect <> 1) + AND (srcChannel IS NULL OR srcChannel IN + (:whoop4Historical, :whoop4Realtime, :whoop4Standard)) GROUP BY ts / 3600 ) SELECT r.ts, r.rrMs, r.srcChannel, r.ord, r.seq FROM rrInterval r - JOIN whoop4HourCounts h ON h.hour = r.ts / 3600 + JOIN whoop4HourChoice h ON h.hour = r.ts / 3600 WHERE r.deviceId = :d AND r.ts >= :f AND r.ts <= :t AND (r.srcChannel IS NULL OR r.srcChannel <> :rrx) - AND ((r.srcChannel = :whoop4Historical AND h.historyCount >= h.liveCount) - OR (r.srcChannel IS NULL AND h.historyCount < h.liveCount)) + AND ((r.srcChannel = :whoop4Historical AND h.sourceChoice = 1) + OR (r.srcChannel = :whoop4Realtime AND h.sourceChoice = 2) + OR (r.srcChannel = :whoop4Standard AND h.sourceChoice = 3) + OR (r.srcChannel IS NULL AND h.sourceChoice = 4)) AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :lim """ @@ -532,6 +537,8 @@ extension WhoopStore { arguments: ["d": deviceId, "f": from, "t": to, "rrx": RRSourceChannel.spo2Ibi.rawValue, "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, + "whoop4Realtime": RRSourceChannel.whoop4Realtime.rawValue, + "whoop4Standard": RRSourceChannel.whoop4Standard.rawValue, "lim": limit]) .map { row in RRInterval(ts: row["ts"], rrMs: row["rrMs"], diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift index 880a647121..78657a1a15 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift @@ -51,7 +51,7 @@ final class RrSourceChannelTests: XCTestCase { XCTAssertNil(selected.first?.srcChannel) } - func testWhoop4UsesTheFillerSourceForTheHour() async throws { + func testWhoop4HistoricalSourceHasPriorityWithinItsHour() async throws { let store = try await WhoopStore.inMemory() try await store.upsertDevice(id: "strap", mac: nil, name: nil) try setRegistry(store) @@ -61,8 +61,23 @@ final class RrSourceChannelTests: XCTestCase { deviceId: "strap") let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts + 1, limit: 100) - XCTAssertEqual(selected.map(\.rrMs), [800, 810]) - XCTAssertEqual(selected.map(\.srcChannel), [nil, nil]) + XCTAssertEqual(selected.map(\.rrMs), [805]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical]) + } + + func testWhoop4RealtimeSourceWinsOverStandardAndLegacyRows() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: ts, rrMs: 800), + RRInterval(ts: ts, rrMs: 810, srcChannel: .whoop4Standard), + RRInterval(ts: ts, rrMs: 820, srcChannel: .whoop4Realtime), + ]), deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [820]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Realtime]) } func testWhoop4PartialHistoryKeepsUnlabelledRowsFromOtherHours() async throws { @@ -85,8 +100,9 @@ final class RrSourceChannelTests: XCTestCase { let selected = try await store.rrIntervals(deviceId: "strap", from: base, to: base + 7200, limit: 100) - XCTAssertEqual(selected.map(\.rrMs), [805, 815, 820, 830, 840]) - XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical, .whoop4Historical, nil, nil, nil]) + XCTAssertEqual(selected.map(\.rrMs), [805, 815, 825]) + XCTAssertEqual(selected.map(\.srcChannel), + [.whoop4Historical, .whoop4Historical, .whoop4Historical]) } // MARK: - The label survives the mapping diff --git a/Strand/Collect/Collector.swift b/Strand/Collect/Collector.swift index 35e9fde3fd..07fd73f3df 100644 --- a/Strand/Collect/Collector.swift +++ b/Strand/Collect/Collector.swift @@ -231,7 +231,8 @@ final class Collector { let frames = batch.map(\.frame) // still needed for the raw-capture outbox let parsed = batch.map(\.parsed) // #47: the seam already decoded these — don't re-parse - let streams = extractStreams(parsed, deviceClockRef: ref.device, wallClockRef: ref.wall) + let streams = extractStreams(parsed, deviceClockRef: ref.device, wallClockRef: ref.wall, + family: family) // #1118: the SECOND live transport. `flushStandardHR` stamps a beat at the second it arrived over // 0x2A37; this one stamps it from the strap's own record clock. The same beat reaching both lands // on two different seconds, which no same-second de-dup can collapse — the signature every @@ -291,7 +292,8 @@ final class Collector { let acceptedHR = (30...220).contains(hr) ? 1 : 0 let acceptedRR = rr.filter { (250...3000).contains($0) } if acceptedHR == 1 { stdHR.append(HRSample(ts: ts, bpm: hr)) } - let source: RRSourceChannel? = family == .whoop5 ? .whoop5Standard : nil + let source: RRSourceChannel? = family == .whoop5 ? .whoop5Standard + : (family == .whoop4 ? .whoop4Standard : nil) stdRR.append(contentsOf: acceptedRR.map { RRInterval(ts: ts, rrMs: $0, srcChannel: source) }) // Only the CHANGES. Advanced here rather than at flush because the event travels in the buffer // until it persists: a failed insert re-inserts it at the front, so nothing has to be unwound. diff --git a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt index 081e1a44b0..c7c6107cee 100644 --- a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt +++ b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt @@ -10266,7 +10266,8 @@ class WhoopBleClient( .filter { it.ok && it.typeName == "REALTIME_DATA" } .mapNotNull { (it.parsed["timestamp"] as? Number)?.toInt() } .maxOrNull() ?: now - val streams: Streams = extractStreams(parsed, deviceClockRef = newestRealtimeTs, wallClockRef = now) + val streams: Streams = extractStreams(parsed, deviceClockRef = newestRealtimeTs, + wallClockRef = now, family = connectedFamily) val batch = StreamPersistence.toBatch(streams) // #1118: the SECOND live transport. The standard 0x2A37 path above stamps a beat at the second // it arrived; this one stamps it from the strap's own record clock. The same beat reaching both @@ -10334,8 +10335,11 @@ class WhoopBleClient( family: DeviceFamily) { val shouldFlush = synchronized(collectorLock) { if (hr in 30..220) stdHr.add(HrRow(ts, hr)) - val source = if (family == DeviceFamily.WHOOP5) - com.noop.protocol.RrSourceChannel.WHOOP5_STANDARD else null + val source = when (family) { + DeviceFamily.WHOOP5 -> com.noop.protocol.RrSourceChannel.WHOOP5_STANDARD + DeviceFamily.WHOOP4 -> com.noop.protocol.RrSourceChannel.WHOOP4_STANDARD + else -> null + } for (r in rr) if (r in 250..3000) stdRr.add(RrRow(ts, r, source)) stdContact.add(StandardHrMapping.contactEvent(ts, contact)) standardHrBufferReachedFlushThreshold(stdHr.size, stdRr.size, stdContact.size) diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index d0a82b3b41..22274527e8 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -12,7 +12,7 @@ import kotlinx.coroutines.flow.Flow /** Kept as one compile-time constant so Room and the plain-JVM SQLite regression test execute the exact * same statement. Swift's twin lives in WhoopStore.analysisFingerprint(). */ internal const val ANALYSIS_FINGERPRINT_SQL = - "SELECT 'v4|' || " + + "SELECT 'v5|' || " + "'h' || (SELECT COUNT(*) FROM hrSample) || ':' || (SELECT COALESCE(MAX(ts), 0) FROM hrSample) || '|' || " + "'p' || (SELECT COALESCE(MAX(rowid), 0) FROM ppgHrSample) || '|' || " + "'r' || (SELECT COALESCE(MAX(rowid), 0) FROM rrInterval) || '|' || " + @@ -58,7 +58,7 @@ internal const val ANALYSIS_FINGERPRINT_SQL = * same statement; unlike [ANALYSIS_FINGERPRINT_SQL] it carries :deviceId/:from/:to binds, so a plain-JVM * SQLite harness could not run it verbatim. Room's KSP verification is what checks it. */ internal const val DAY_STREAM_FINGERPRINT_SQL = - "SELECT 's3|' || " + + "SELECT 's4|' || " + "'p' || (SELECT COUNT(*) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || " + "':' || (SELECT COALESCE(MAX(ts), 0) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || '|' || " + "'r' || (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + @@ -136,18 +136,21 @@ internal const val WHOOP5_RR_INTERVALS_SQL = "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" internal const val WHOOP4_RR_INTERVALS_SQL = - "WITH whoop4HourCounts AS (" + + "WITH whoop4HourChoice AS (" + "SELECT ts / 3600 AS hour, " + - "SUM(CASE WHEN srcChannel = 8 THEN 1 ELSE 0 END) AS historyCount, " + - "SUM(CASE WHEN srcChannel IS NULL THEN 1 ELSE 0 END) AS liveCount " + + "MIN(CASE WHEN srcChannel = 8 THEN 1 WHEN srcChannel = 9 THEN 2 " + + "WHEN srcChannel = 10 THEN 3 WHEN srcChannel IS NULL THEN 4 END) AS sourceChoice " + "FROM rrInterval WHERE deviceId = :deviceId " + "AND ts >= (:from / 3600) * 3600 AND ts < ((:to / 3600) + 1) * 3600 " + - "AND (tsSuspect IS NULL OR tsSuspect <> 1) GROUP BY ts / 3600) " + - "SELECT r.* FROM rrInterval r JOIN whoop4HourCounts h ON h.hour = r.ts / 3600 " + + "AND (tsSuspect IS NULL OR tsSuspect <> 1) " + + "AND (srcChannel IS NULL OR srcChannel IN (8, 9, 10)) GROUP BY ts / 3600) " + + "SELECT r.* FROM rrInterval r JOIN whoop4HourChoice h ON h.hour = r.ts / 3600 " + "WHERE r.deviceId = :deviceId AND r.ts >= :from AND r.ts <= :to " + "AND (r.srcChannel IS NULL OR r.srcChannel <> 2) " + - "AND ((r.srcChannel = 8 AND h.historyCount >= h.liveCount) " + - "OR (r.srcChannel IS NULL AND h.historyCount < h.liveCount)) " + + "AND ((r.srcChannel = 8 AND h.sourceChoice = 1) " + + "OR (r.srcChannel = 9 AND h.sourceChoice = 2) " + + "OR (r.srcChannel = 10 AND h.sourceChoice = 3) " + + "OR (r.srcChannel IS NULL AND h.sourceChoice = 4)) " + "AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) " + "ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :limit" diff --git a/android/app/src/main/java/com/noop/protocol/Streams.kt b/android/app/src/main/java/com/noop/protocol/Streams.kt index 5698d9f5e0..db4252cfe0 100644 --- a/android/app/src/main/java/com/noop/protocol/Streams.kt +++ b/android/app/src/main/java/com/noop/protocol/Streams.kt @@ -81,6 +81,10 @@ enum class RrSourceChannel(val code: Int) { WHOOP5_STANDARD(7), /** WHOOP 4.0 type-47 historical stream, distinguished from its unlabelled live BLE feed. */ WHOOP4_HISTORICAL(8), + /** WHOOP 4.0 type-40 realtime RR, timestamped from its record clock. */ + WHOOP4_REALTIME(9), + /** WHOOP 4.0 standard BLE 0x2A37 RR, timestamped on notification receipt. */ + WHOOP4_STANDARD(10), ; val isWhoop5Transport: Boolean get() = code in 5..7 @@ -324,7 +328,8 @@ private fun toWall(deviceTs: Int?, deviceClockRef: Int, wallClockRef: Int): Int? * the header checksum, the payload CRC32 and the structural length together, not the payload CRC * alone. */ -fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: Int): Streams { +fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: Int, + family: DeviceFamily? = null): Streams { val out = Streams() for (r in parsed) { if (!r.ok) continue @@ -337,6 +342,7 @@ fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: // Drop RR rows when timestamp is absent (a ts-less RR row is unstorable). p.intArrayOrNull("rr_intervals")?.let { rrs -> val source = RrSourceChannel.fromCode(p.intOrNull("rr_source_channel")) + ?: if (family == DeviceFamily.WHOOP4) RrSourceChannel.WHOOP4_REALTIME else null for (rr in rrs) out.rr.add(RrInterval(ts, rr, source)) } } diff --git a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt index 487e3cc5c2..4579083b19 100644 --- a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt +++ b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt @@ -49,7 +49,7 @@ class HrFingerprintTest { @Test fun analysisFingerprintSqlRunsAndMovesForGravityOnlyCommit() { DriverManager.getConnection("jdbc:sqlite::memory:").use { db -> createFingerprintTables(db) - assertEquals("v4|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) + assertEquals("v5|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) db.createStatement().use { sql -> sql.executeUpdate("INSERT INTO hrSample(ts) VALUES (1000)") @@ -57,7 +57,7 @@ class HrFingerprintTest { sql.executeUpdate("INSERT INTO rrInterval DEFAULT VALUES") sql.executeUpdate("INSERT INTO gravitySample DEFAULT VALUES") } - assertEquals("v4|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) + assertEquals("v5|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) } } diff --git a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt index 607e45febc..963e96bfba 100644 --- a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt +++ b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt @@ -192,7 +192,7 @@ class Whoop5RRSqliteTest { WHOOP4_RR_INTERVALS_SQL, mapOf("deviceId" to id, "from" to base, "to" to base + 7200, "limit" to 100), ) { it.getInt("rrMs") to it.getInt("srcChannel") } - assertEquals(listOf(805 to 8, 815 to 8, 820 to 0, 830 to 0, 840 to 0), selected) + assertEquals(listOf(805 to 8, 815 to 8, 825 to 8), selected) } @After fun close() { db.close() } @@ -704,12 +704,24 @@ class Whoop5RRSqliteTest { assertEquals(listOf(8, 8), rows.map { it.srcChannel }) } - @Test fun whoop4UsesTheFillerSourceForTheHour() = runBlocking { + @Test fun whoop4HistoricalSourceHasPriorityWithinItsHour() = runBlocking { registry("4.0") repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(101, 810))), id) repo.insert(StreamBatch(rr = listOf(RrRow(100, 805, RrSourceChannel.WHOOP4_HISTORICAL))), id) val rows = repo.rrIntervalsForDevice(id, 100, 101, 100) - assertEquals(listOf(800, 810), rows.map { it.rrMs }) + assertEquals(listOf(805), rows.map { it.rrMs }) + } + + @Test fun whoop4RealtimeSourceWinsOverStandardAndLegacyRows() = runBlocking { + registry("4.0") + repo.insert(StreamBatch(rr = listOf( + RrRow(100, 800), + RrRow(100, 810, RrSourceChannel.WHOOP4_STANDARD), + RrRow(100, 820, RrSourceChannel.WHOOP4_REALTIME), + )), id) + val rows = repo.rrIntervalsForDevice(id, 100, 100, 100) + assertEquals(listOf(820), rows.map { it.rrMs }) + assertEquals(listOf(9), rows.map { it.srcChannel }) } @Test fun whoop4FallsBackToUnlabelledRowsWhenNoHistoryExists() = runBlocking { diff --git a/android/app/src/test/java/com/noop/protocol/FramingTest.kt b/android/app/src/test/java/com/noop/protocol/FramingTest.kt index 4c11d40b66..2a834e3fef 100644 --- a/android/app/src/test/java/com/noop/protocol/FramingTest.kt +++ b/android/app/src/test/java/com/noop/protocol/FramingTest.kt @@ -422,6 +422,10 @@ class FramingTest { assertEquals(2, streams.rr.size) assertEquals(RrInterval(ts = 1700000000, rrMs = 850), streams.rr[0]) assertEquals(RrInterval(ts = 1700000000, rrMs = 870), streams.rr[1]) + val whoop4 = extractStreams(listOf(parsed), deviceClockRef = 1700000000, + wallClockRef = 1700000000, family = DeviceFamily.WHOOP4) + assertEquals(listOf(RrSourceChannel.WHOOP4_REALTIME, RrSourceChannel.WHOOP4_REALTIME), + whoop4.rr.map { it.srcChannel }) } // MARK: - WHOOP 5.0/MG REALTIME_DATA (+4) + family-aware reassembly diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt index e716b759b0..6ee1f0e47e 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt @@ -88,7 +88,7 @@ class Whoop5RRTest { } @Test fun transportCodes() { - assertEquals((1..8).toList(), RrSourceChannel.entries.map { it.code }) + assertEquals((1..10).toList(), RrSourceChannel.entries.map { it.code }) } /** From e0515bc684bc4049996ef7a8ac44e92ab2602760 Mon Sep 17 00:00:00 2001 From: Fanboynz Date: Sun, 27 Sep 2026 15:27:26 +1300 Subject: [PATCH 5/5] WHOOP 4 R-R sources: repin the ingress test, pair extractStreams Labelling live WHOOP 4 beats is the point of the change, so the ingress test moves to the new shape rather than losing the assertion: a standard-BLE beat is `whoop4Standard` now where it was left unlabelled, and that NULL is exactly what let a partial type-47 offload hide live intervals. Only a beat with no family stays nil. The test's own name reads truer for it. `extractStreams` gained `family` on both platforms in the same change, which changed the arity on both sides at once and unpaired them in the ledger even though they moved together. Declared twin references pair them again, and the derived authority is refreshed for it: the baseline holds at 292 known findings and no disposition is touched. The Kotlin query now says why it is not `AS MATERIALIZED` like the Swift twin. That hint wants SQLite 3.35 and `minSdk` is 26, so it is a syntax error on most supported devices; the cost is the per-row re-evaluation the Swift comment names, and the note exists so nobody closes the gap by adding the keyword. Worth stating because the PR description says otherwise: the merged policy is provenance order, not the fuller stream. History, then type-40 realtime, then standard BLE, then legacy rows, chosen per UTC hour. Commits one to three selected by count; commit four replaced that with `MIN(rank)` and rewrote the tests to pin it. Keeping it: type-47 carries the strap's own device timestamps where standard BLE is stamped on receipt, which is where #1008's cross-second over-count came from, so the better-timestamped stream winning is the safer read. The residual is that an hour holding sparse history still prefers it over dense live beats. Refs #2496, #1118, #2335 --- .../WhoopProtocol/Sources/WhoopProtocol/Streams.swift | 4 ++++ StrandTests/CollectorStandardHRContactTests.swift | 6 +++++- Tools/parity_twin_map.json | 6 +++--- android/app/src/main/java/com/noop/data/WhoopDao.kt | 11 +++++++++++ .../app/src/main/java/com/noop/protocol/Streams.kt | 3 +++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift index 5dc5aab43d..3280b3b4d4 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift @@ -830,6 +830,10 @@ private func toWall(_ deviceTs: Int?, _ deviceClockRef: Int, _ wallClockRef: Int /// carries an HR byte but streams alongside type-40 during raw collection, so routing both /// would double-count HR for the same instants. Frames whose integrity verdict is negative are /// skipped — no row is derived from a frame that is not intact. +/// +/// Kotlin twin: `extractStreams`. Adding `family` changed the arity on both sides at once, which +/// unpaired them in the ledger even though they moved together; the reference says they are one +/// contract so the next parameter does not have to rediscover that. public func extractStreams(_ parsed: [ParsedFrame], deviceClockRef: Int, wallClockRef: Int, family: DeviceFamily? = nil) -> Streams { diff --git a/StrandTests/CollectorStandardHRContactTests.swift b/StrandTests/CollectorStandardHRContactTests.swift index e8746365fd..0a39a01991 100644 --- a/StrandTests/CollectorStandardHRContactTests.swift +++ b/StrandTests/CollectorStandardHRContactTests.swift @@ -118,7 +118,11 @@ final class CollectorStandardHRContactTests: XCTestCase { await collector.flushStandardHR() XCTAssertTrue(store.inserted.isEmpty) await collector.flushStandardHR() - XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.srcChannel), [.whoop5Standard, nil, nil]) + // A WHOOP 4 standard-BLE beat is labelled now (`whoop4Standard`) where it used to be left + // unlabelled: that NULL is what let a partial type-47 offload hide live intervals, so the + // assertion moves to the new shape. Only a beat with NO family stays nil. + XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.srcChannel), + [.whoop5Standard, .whoop4Standard, nil]) XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.rrMs), [1000, 1001, 1002]) } } diff --git a/Tools/parity_twin_map.json b/Tools/parity_twin_map.json index 88ab06b73c..6abf5498b8 100644 --- a/Tools/parity_twin_map.json +++ b/Tools/parity_twin_map.json @@ -19,15 +19,15 @@ }, "authority": { "files": {"count": 504, "sha256": "fc60ea929a4de0859ebb2fefb0302248f79c712a9f537f6050b6e02a64f485e8"}, - "functions": {"count": 4497, "sha256": "acc9b4ea67a5b44cc1b3497715d0c42bf39d378a703d71e8541a4f3d40131b81"}, + "functions": {"count": 4497, "sha256": "af2aef301c4482689bc5c4704be1f2e0c0688538511d0c8247953a350cf1f70c"}, "properties": {"count": 465, "sha256": "64fcd18c85bf33a4738d154d45ec86967fdc974e78a448d16e9504d9b13609eb"}, "constants": {"count": 1962, "sha256": "be21ed554bc51ed8d3faffce82c131b9d025f2252a8f7c5281c06736a480270a"}, "file_pairs": {"count": 75, "sha256": "d581f60241ec523c595a7e5ff123f7987caee501d8be4ebf6df5ba105e26fe1f"}, - "function_pairs": {"count": 202, "sha256": "face39eac8619a041db0d34d811065ef6d2a83a3b8f0e17b679c27ecd37e7c4a"}, + "function_pairs": {"count": 203, "sha256": "b5647a01ad1b00477ac9dcbb0a7260acf49badf9164f3d7562a0bf4b905d1b8e"}, "property_pairs": {"count": 150, "sha256": "be7168d309c8a9cf5f3f25438c028cc60ab64202bb6e9c09a6c553e3699ad0e2"}, "constant_pairs": {"count": 682, "sha256": "95b64a46a21dec593537c83be57084d2904e2de779a5672d8bdd4cf9da7fdee8"}, "unpaired_files": {"count": 376, "sha256": "654d951b8137dbb2266b83734fa1e3f1cb1648f15c1aab3d015cdef7e79deb75"}, - "unpaired_functions": {"count": 4106, "sha256": "be727bd03462055bec05379807b15686087f43df0ef4706773046fb22ff1e9b1"}, + "unpaired_functions": {"count": 4104, "sha256": "4a7e8aeaf1544374de442570f6eba5a04af48d93459a62c5c9d87311f57b6dbc"}, "unpaired_properties": {"count": 165, "sha256": "fe609dd25384cda6de1616d36753fed68431b1a1f9a73b6fd664c7ff35089955"}, "unpaired_constants": {"count": 598, "sha256": "3d84e1d95ee3a93aad1681443605a641284d894f5e5014669abf7668eb1eb49d"} } diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 22274527e8..4582e39c76 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -135,6 +135,17 @@ internal const val WHOOP5_RR_INTERVALS_SQL = "AND (tsSuspect IS NULL OR tsSuspect <> 1)) " + "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" +/** + * One R-R source per UTC hour for a WHOOP 4, in provenance order: type-47 history, then type-40 + * realtime, then standard BLE, then legacy unlabelled rows. Overlapping transports are never merged, + * and history takes precedence only in the hours it actually covers, so a partial offload no longer + * hides live intervals across the whole interval (#1118, #2335). + * + * Deliberately NOT `AS MATERIALIZED`, unlike the Swift twin. That hint needs SQLite 3.35 and `minSdk` + * is 26, so it is a syntax error on most supported devices. The cost is that SQLite may re-evaluate + * the hour choice per row, which is the quadratic read the Swift side names; do not "fix" the + * asymmetry by adding the keyword. + */ internal const val WHOOP4_RR_INTERVALS_SQL = "WITH whoop4HourChoice AS (" + "SELECT ts / 3600 AS hour, " + diff --git a/android/app/src/main/java/com/noop/protocol/Streams.kt b/android/app/src/main/java/com/noop/protocol/Streams.kt index db4252cfe0..c6bdd73db0 100644 --- a/android/app/src/main/java/com/noop/protocol/Streams.kt +++ b/android/app/src/main/java/com/noop/protocol/Streams.kt @@ -327,6 +327,9 @@ private fun toWall(deviceTs: Int?, deviceClockRef: Int, wallClockRef: Int): Int? * HR for the same instants. Frames whose full integrity verdict is negative are skipped — that is * the header checksum, the payload CRC32 and the structural length together, not the payload CRC * alone. + * + * Swift twin: `extractStreams`. Adding `family` changed the arity on both sides at once, which + * unpaired them in the ledger even though they moved together. */ fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: Int, family: DeviceFamily? = null): Streams {