diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift index d96b387ad7..25315292c7 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift @@ -170,6 +170,7 @@ public func isEmptyRecordFrame(_ frame: [UInt8]) -> Bool { /// CRC-failed and non-ok frames are skipped. public func extractHistoricalStreams(_ parsed: [ParsedFrame], deviceClockRef: Int, wallClockRef: Int, + family: DeviceFamily? = nil, // SESSION-RELATIVE bounds (#547): the strap's own GET_DATA_RANGE // oldest/newest markers for THIS sync. nil on the replay/import/no-range // paths — the gate then falls back to the absolute-only floor (unchanged). @@ -298,6 +299,7 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame], } if let rrs = p["rr_intervals"]?.intArrayValue { let source = p["rr_source_channel"]?.intValue.flatMap(RRSourceChannel.init(rawValue:)) + ?? (family == .whoop4 ? .whoop4Historical : nil) for rr in rrs { out.rr.append(RRInterval(ts: ts, rrMs: rr, srcChannel: source)) } } if let red = p["spo2_red"]?.intValue { diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift index acf0a3493c..c174c914e2 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift @@ -37,7 +37,7 @@ public enum StandardHRContact: String, Equatable, Codable, Sendable { /// The sensor channel or transport that produced an R-R interval. /// /// WHOOP 5 exposes one beat train over multiple transports; codes 5–7 distinguish those observations. -/// WHOOP 4 and unlabelled legacy rows keep nil. An Oura ring has more than one optical channel: +/// WHOOP 4 standard-BLE rows and legacy data keep nil; its type-47 historical rows use code 8. An Oura ring has more than one optical channel: /// the green-quality tag (0x80) and the SpO2 tag (0x6E) both /// decode to R-R and both were stored, so the table held roughly TWO complete copies of every night — /// not duplicate rows to de-duplicate, but the SAME heartbeats measured twice. Labelling the channel is @@ -70,6 +70,9 @@ public enum RRSourceChannel: Int, Equatable, Codable, Sendable, CaseIterable { case whoop5Realtime = 6 /// WHOOP 5 standard BLE 0x2A37 live transport, already converted to milliseconds. case whoop5Standard = 7 + /// WHOOP 4.0 type-47 historical R-R intervals. The 4.0 standard BLE feed has no source label, + /// so this distinguishes the clock-anchored offload from the overlapping live feed at read time. + case whoop4Historical = 8 public var isWhoop5Transport: Bool { (5...7).contains(rawValue) } } @@ -77,8 +80,8 @@ public enum RRSourceChannel: Int, Equatable, Codable, Sendable, CaseIterable { public struct RRInterval: Equatable, Codable { public let ts: Int // wall-clock unix seconds public let rrMs: Int - /// The sensor channel this beat came from, or nil when the source does not distinguish one (every - /// WHOOP 4 row, and legacy unlabelled rows). See `RRSourceChannel`. + /// The sensor channel this beat came from, or nil when the source does not distinguish one (WHOOP 4 + /// standard BLE and legacy unlabelled rows). See `RRSourceChannel`. public let srcChannel: RRSourceChannel? /// #1008 diagnostics: this beat's EMISSION ORDER within the batch that delivered it, as stored in /// `rrInterval.ord`. nil on a decode (nothing has been stored yet) and on rows written before the diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift index 3a2f695481..42c8eb9dfc 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift @@ -50,7 +50,7 @@ final class Whoop5RRTests: XCTestCase { } func testTransportCodes() { - XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...7)) + XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...8)) } /// The table below is the parity contract for the "this night cannot be scored" explanation. The diff --git a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift index 065ac728d2..a4ef2a7285 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift @@ -211,6 +211,7 @@ extension WhoopStore { (SELECT COUNT(*) FROM rrInterval WHERE srcChannel = 7 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AS w7, (SELECT COUNT(*) FROM rrInterval WHERE srcChannel IN (5, 6, 7)) AS w5tagged, + (SELECT COUNT(*) FROM rrInterval WHERE srcChannel = 8) AS w4history, (SELECT COALESCE(GROUP_CONCAT(identity, ';'), '') FROM (SELECT QUOTE(id) || ':' || QUOTE(brand) || ':' || QUOTE(model) || ':' || QUOTE(status) AS identity FROM pairedDevice ORDER BY id)) AS registry, @@ -230,8 +231,9 @@ extension WhoopStore { } let historyCount: Int = row["w5"] let registry: String = row["registry"] - return "v3|h\(hc):\(hm)|" + tails.joined(separator: "|") - + "|w5\(historyCount)|w7\(row["w7"] as Int)|tagged\(row["w5tagged"] as Int)|registry\(registry)" + return "v4|h\(hc):\(hm)|" + tails.joined(separator: "|") + + "|w5\(historyCount)|w7\(row["w7"] as Int)|tagged\(row["w5tagged"] as Int)" + + "|w4history\(row["w4history"] as Int)|registry\(registry)" } } @@ -279,6 +281,8 @@ extension WhoopStore { (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel = 7 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AS w7, EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND srcChannel IN (5, 6, 7)) AS w5owner, + (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t + AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) AS w4h, COALESCE((SELECT QUOTE(brand) || ':' || QUOTE(model) FROM pairedDevice WHERE id = :d), 'absent') AS registry, (SELECT COUNT(*) FROM respSample WHERE deviceId = :d AND ts >= :f AND ts <= :t) AS xc, @@ -296,7 +300,8 @@ extension WhoopStore { (SELECT COUNT(*) FROM event WHERE deviceId = :d AND ts >= :f AND ts <= :t) AS ec, (SELECT COALESCE(MAX(ts), 0) FROM event WHERE deviceId = :d AND ts >= :f AND ts <= :t) AS em """, arguments: ["d": deviceId, "f": from, "t": to, - "rrx": RRSourceChannel.spo2Ibi.rawValue]) else { return "" } + "rrx": RRSourceChannel.spo2Ibi.rawValue, + "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue]) else { return "" } let keys = ["p", "r", "x", "o", "g", "z", "t", "b", "e"] let parts = keys.map { key -> String in let count: Int = row[key + "c"], maxTs: Int = row[key + "m"] @@ -305,7 +310,8 @@ extension WhoopStore { let historyCount: Int = row["w5"] let registry: String = row["registry"] let strictRR = try Self.isWhoop5RRSource(db: db, deviceId: deviceId) - return "s2|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)|ownerTagged\(row["w5owner"] as Int)|registry\(registry)|rr5=\(strictRR)" + return "s3|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)" + + "|w4h\(row["w4h"] as Int)|ownerTagged\(row["w5owner"] as Int)|registry\(registry)|rr5=\(strictRR)" } } @@ -441,11 +447,28 @@ extension WhoopStore { // One transport for the complete requested interval. Legacy WHOOP 5 rows mix units and // origins, so they remain stored but cannot be converted or spliced into a scored beat train. // This subquery uses the SAME time/suspect predicates as the outer read, before LIMIT. + let strictWhoop4History: Bool + if strictWhoop5 { strictWhoop4History = false } + else { + if try Self.isWhoop4RRSource(db: db, deviceId: deviceId) { strictWhoop4History = true } + else { + strictWhoop4History = try Bool.fetchOne(db, + sql: "SELECT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = ? AND srcChannel = ?)", + arguments: [deviceId, RRSourceChannel.whoop4Historical.rawValue]) ?? false + } + } let sourcePredicate = strictWhoop5 ? """ srcChannel = (SELECT MIN(srcChannel) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableWhoop5Channels) AND (tsSuspect IS NULL OR tsSuspect <> 1)) - """ : "1" + """ : (strictWhoop4History ? """ + ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t + AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) + AND srcChannel = :whoop4Historical) + OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t + AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) + AND srcChannel IS NULL)) + """ : "1") return try Row.fetchAll(db, sql: """ SELECT ts, rrMs, srcChannel, ord, seq FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t @@ -454,7 +477,8 @@ extension WhoopStore { AND (tsSuspect IS NULL OR tsSuspect <> 1) -- #1073: exclude future-stamped beats ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :lim """, arguments: ["d": deviceId, "f": from, "t": to, - "rrx": RRSourceChannel.spo2Ibi.rawValue, "lim": limit]) + "rrx": RRSourceChannel.spo2Ibi.rawValue, + "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, "lim": limit]) .map { row in RRInterval(ts: row["ts"], rrMs: row["rrMs"], srcChannel: (row["srcChannel"] as Int?).flatMap(RRSourceChannel.init(rawValue:)), diff --git a/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift b/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift index 59c5386b84..53d11b8002 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift @@ -235,6 +235,11 @@ extension WhoopStore { AND ((:source = 5 AND (srcChannel IS NULL OR srcChannel IN (6, 7))) OR (:source = 7 AND (srcChannel IS NULL OR srcChannel = 6))) """) + let promoteWhoop4History = try db.cachedStatement(sql: """ + UPDATE rrInterval SET srcChannel = :source, ord = :ord + WHERE deviceId = :device AND ts = :ts AND rrMs = :rr AND seq = :seq + AND :source = 8 AND srcChannel IS NULL + """) var seqByTsRr: [RRBatchSecond: [Int: Int]] = [:] var ordByTs: [RRBatchSecond: Int] = [:] for r in streams.rr { @@ -258,6 +263,10 @@ extension WhoopStore { try promote.execute(arguments: ["source": source.rawValue, "ord": ord, "device": deviceId, "ts": r.ts, "rr": r.rrMs, "seq": seq]) } + if inserted == 0, r.srcChannel == .whoop4Historical { + try promoteWhoop4History.execute(arguments: ["source": r.srcChannel!.rawValue, "ord": ord, + "device": deviceId, "ts": r.ts, "rr": r.rrMs, "seq": seq]) + } } } if !streams.events.isEmpty { diff --git a/Packages/WhoopStore/Sources/WhoopStore/Whoop5RRReadPolicy.swift b/Packages/WhoopStore/Sources/WhoopStore/Whoop5RRReadPolicy.swift index e135b30450..82f623fd19 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Whoop5RRReadPolicy.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Whoop5RRReadPolicy.swift @@ -2,6 +2,10 @@ import GRDB import WhoopProtocol extension WhoopStore { + static func isWhoop4RRSource(db: Database, deviceId: String) throws -> Bool { + let row = try Row.fetchOne(db, sql: "SELECT model, brand FROM pairedDevice WHERE id = ?", arguments: [deviceId]) + return DeviceFamily.confirmedRegistryFamily(model: row?["model"], brand: row?["brand"]) == .whoop4 + } /// The transports a WHOOP 5 window may be SCORED through, as a SQL list. /// /// One constant rather than a literal per query. `rrIntervals` pins a window to the lowest of these diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift index f7be3d54f5..da45d56398 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift @@ -18,6 +18,53 @@ import OuraProtocol final class RrSourceChannelTests: XCTestCase { private let ts = 1_750_000_000 + func testWhoop4HistoricalSourceWinsAndPromotesLegacyLiveRows() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( + id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, + capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + + _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), + RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") + _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800, srcChannel: .whoop4Historical), + RRInterval(ts: ts + 1, rrMs: 820, srcChannel: .whoop4Historical)]), + deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts + 1, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [800, 820]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical, .whoop4Historical]) + } + + func testWhoop4FallsBackToUnlabelledRowsWhenNoHistoricalDataExists() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( + id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, + capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800)]), deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [800]) + XCTAssertNil(selected.first?.srcChannel) + } + + func testWhoop4UsesOneHistoricalTransportForTheWholeWindow() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( + id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, + capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), + RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") + _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 805, srcChannel: .whoop4Historical)]), + deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts + 1, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [805]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical]) + } + // MARK: - The label survives the mapping /// A 0x6E record and a 0x80 record covering the same interval must produce rows with DISTINCT @@ -54,7 +101,8 @@ final class RrSourceChannelTests: XCTestCase { /// The two enums are pinned to the same raw values on purpose: they are one durable storage code /// split across two packages only because `OuraProtocol` does not depend on `WhoopProtocol`. func testTheTwoChannelEnumsAgreeCaseForCaseAndCodeForCode() { - XCTAssertEqual(OuraIBIChannel.allCases.count, RRSourceChannel.allCases.filter { !$0.isWhoop5Transport }.count) + XCTAssertEqual(OuraIBIChannel.allCases.count, + RRSourceChannel.allCases.filter { !$0.isWhoop5Transport && $0.rawValue <= 4 }.count) for c in OuraIBIChannel.allCases { let mapped = OuraStreamMapping.rrChannel(c) XCTAssertEqual(mapped?.rawValue, c.rawValue, diff --git a/Strand/Collect/Backfiller.swift b/Strand/Collect/Backfiller.swift index f999439801..3d5aa06adb 100644 --- a/Strand/Collect/Backfiller.swift +++ b/Strand/Collect/Backfiller.swift @@ -36,7 +36,7 @@ final class Backfiller { /// (parsed frames, deviceClockRef, wallClockRef, sessionOldestUnix?, sessionNewestUnix?) → Streams. /// The trailing session-range markers are the strap's GET_DATA_RANGE oldest/newest for THIS sync /// (#547 session-relative gate); nil when the range isn't known yet (the absolute-only floor applies). - typealias Extractor = ([ParsedFrame], Int, Int, Int?, Int?) -> Streams + typealias Extractor = ([ParsedFrame], Int, Int, DeviceFamily, Int?, Int?) -> Streams private let store: BackfillStoreWriting /// Device id offloaded chunks persist under. MUTABLE so a WHOOP↔WHOOP switch @@ -289,8 +289,9 @@ final class Backfiller { // The default (prod) Extractor reads the opt-in HR-from-PPG sub-lag interpolation flag (Test Centre → // Experimental algorithms) at decode time and threads it into the pure decoder, so the pure package // never reaches for UserDefaults. Default OFF = byte-identical to today. Tests inject their own seam. - extract: @escaping Extractor = { extractHistoricalStreams($0, deviceClockRef: $1, wallClockRef: $2, - sessionOldestUnix: $3, sessionNewestUnix: $4, + extract: @escaping Extractor = { extractHistoricalStreams($0, deviceClockRef: $1, wallClockRef: $2, + family: $3, + sessionOldestUnix: $4, sessionNewestUnix: $5, subLagInterp: PuffinExperiment.ppgHrSubLagInterpEnabled) }) { self.store = store self.deviceId = deviceId @@ -626,7 +627,7 @@ final class Backfiller { let extractFn = extract // keep the injected Extractor seam (tests override it); prod == extractHistoricalStreams let d = await Task.detached(priority: .utility) { () -> DecodedChunk in let parsed = frames.map { parseFrame($0, family: fam) } - let decoded = extractFn(parsed, dev, wall, oldest, newest) + let decoded = extractFn(parsed, dev, wall, fam, oldest, newest) let rejected = rejectedHistoricalRecords(frames, family: fam) return DecodedChunk(parsed: parsed, decoded: decoded, rejected: rejected) }.value diff --git a/Strand/Collect/RawHistoryArchive.swift b/Strand/Collect/RawHistoryArchive.swift index 6ba7365b3b..08be57c928 100644 --- a/Strand/Collect/RawHistoryArchive.swift +++ b/Strand/Collect/RawHistoryArchive.swift @@ -386,6 +386,7 @@ struct RawHistoryArchive { // the opt-in HR-from-PPG sub-lag interpolation flag (Test Centre → Experimental algorithms) so the // archive replay re-derives v26 HR with the same variant the live offload uses. Default OFF. let streams = extractHistoricalStreams(parsed, deviceClockRef: 0, wallClockRef: 0, + family: family, subLagInterp: PuffinExperiment.ppgHrSubLagInterpEnabled) // Count rows ACTUALLY inserted, not decoded: under the per-app-version gate the archive // replays every release, and dedupe makes those re-runs insert 0 — counting decoded rows diff --git a/Strand/Data/Repository.swift b/Strand/Data/Repository.swift index 924c1ae3cf..703fc04982 100644 --- a/Strand/Data/Repository.swift +++ b/Strand/Data/Repository.swift @@ -397,7 +397,7 @@ final class Repository: ObservableObject { out.append(beat) } } - if out.contains(where: { $0.srcChannel?.isWhoop5Transport == true }) { + if out.contains(where: { $0.srcChannel?.isWhoop5Transport == true || $0.srcChannel == .whoop4Historical }) { // Retain the selected stream's emission order within a second. Value sorting corrupts // successive differences; original offsets also keep owner precedence deterministic. return out.enumerated().sorted { diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 17815abaf8..8b6addb1a2 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -12,7 +12,7 @@ import kotlinx.coroutines.flow.Flow /** Kept as one compile-time constant so Room and the plain-JVM SQLite regression test execute the exact * same statement. Swift's twin lives in WhoopStore.analysisFingerprint(). */ internal const val ANALYSIS_FINGERPRINT_SQL = - "SELECT 'v3|' || " + + "SELECT 'v4|' || " + "'h' || (SELECT COUNT(*) FROM hrSample) || ':' || (SELECT COALESCE(MAX(ts), 0) FROM hrSample) || '|' || " + "'p' || (SELECT COALESCE(MAX(rowid), 0) FROM ppgHrSample) || '|' || " + "'r' || (SELECT COALESCE(MAX(rowid), 0) FROM rrInterval) || '|' || " + @@ -26,6 +26,7 @@ internal const val ANALYSIS_FINGERPRINT_SQL = "'|w5' || (SELECT COUNT(*) FROM rrInterval WHERE srcChannel = 5 AND (tsSuspect IS NULL OR tsSuspect <> 1)) || " + "'|w7' || (SELECT COUNT(*) FROM rrInterval WHERE srcChannel = 7 AND (tsSuspect IS NULL OR tsSuspect <> 1)) || " + "'|tagged' || (SELECT COUNT(*) FROM rrInterval WHERE srcChannel IN (5, 6, 7)) || " + + "'|w4history' || (SELECT COUNT(*) FROM rrInterval WHERE srcChannel = 8) || " + "'|registry' || (SELECT COALESCE(GROUP_CONCAT(identity, ';'), '') FROM " + "(SELECT QUOTE(id) || ':' || QUOTE(brand) || ':' || QUOTE(model) || ':' || QUOTE(status) AS identity FROM pairedDevice ORDER BY id))" @@ -56,7 +57,7 @@ internal const val ANALYSIS_FINGERPRINT_SQL = * same statement; unlike [ANALYSIS_FINGERPRINT_SQL] it carries :deviceId/:from/:to binds, so a plain-JVM * SQLite harness could not run it verbatim. Room's KSP verification is what checks it. */ internal const val DAY_STREAM_FINGERPRINT_SQL = - "SELECT 's2|' || " + + "SELECT 's3|' || " + "'p' || (SELECT COUNT(*) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || " + "':' || (SELECT COALESCE(MAX(ts), 0) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || '|' || " + "'r' || (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + @@ -81,6 +82,8 @@ internal const val DAY_STREAM_FINGERPRINT_SQL = "AND srcChannel = 5 AND (tsSuspect IS NULL OR tsSuspect <> 1)) || " + "'|w7' || (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + "AND srcChannel = 7 AND (tsSuspect IS NULL OR tsSuspect <> 1)) || " + + "'|w4h' || (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + + "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) || " + "'|ownerTagged' || EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND srcChannel IN (5, 6, 7)) || " + "'|registry' || COALESCE((SELECT QUOTE(brand) || ':' || QUOTE(model) FROM pairedDevice WHERE id = :deviceId), 'absent')" @@ -110,6 +113,16 @@ internal const val WHOOP5_RR_INTERVALS_SQL = "AND (tsSuspect IS NULL OR tsSuspect <> 1)) " + "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" +internal const val WHOOP4_RR_INTERVALS_SQL = + "SELECT * FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + + "AND (srcChannel IS NULL OR srcChannel <> 2) " + + "AND (tsSuspect IS NULL OR tsSuspect <> 1) " + + "AND ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + + "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel = 8) " + + "OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + + "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel IS NULL)) " + + "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" + /** The earliest beat a device has banked AT ALL, labelled or not, or null when it has none. The lower * bound on the "cannot be scored" explanation: it separates history this strap actually recorded from * history imported from somewhere else, and only the former can have lost anything to a labelling @@ -143,6 +156,10 @@ internal const val PROMOTE_WHOOP5_RR_SOURCE_SQL = "AND ((:source = 5 AND (srcChannel IS NULL OR srcChannel IN (6, 7))) " + "OR (:source = 7 AND (srcChannel IS NULL OR srcChannel = 6)))" +internal const val PROMOTE_WHOOP4_HISTORY_SQL = + "UPDATE rrInterval SET srcChannel = 8, ord = :ord " + + "WHERE deviceId = :deviceId AND ts = :ts AND rrMs = :rrMs AND seq = :seq AND srcChannel IS NULL" + /** * Data-access for the local store. Mirrors the GRDB reads/writes in WhoopStore * (StreamStore.swift, Reads.swift, MetricsCache.swift). @@ -602,9 +619,15 @@ interface WhoopDao : DeviceRegistryDao { @Query(WHOOP5_RR_INTERVALS_SQL) suspend fun whoop5RrIntervals(deviceId: String, from: Long, to: Long, limit: Int): List + @Query(WHOOP4_RR_INTERVALS_SQL) + suspend fun whoop4RrIntervals(deviceId: String, from: Long, to: Long, limit: Int): List + @Query(HAS_WHOOP5_RR_SOURCE_SQL) suspend fun hasWhoop5RrSource(deviceId: String): Boolean + @Query("SELECT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND srcChannel = 8)") + suspend fun hasWhoop4HistoricalRrSource(deviceId: String): Boolean + /** Exact-window twin of Swift `legacyWhoop5RRWithheld`; source-family gating stays in Repository. */ @Query(LEGACY_WHOOP5_RR_WITHHELD_SQL) suspend fun legacyWhoop5RrWithheld(deviceId: String, from: Long, to: Long): Boolean @@ -621,6 +644,9 @@ interface WhoopDao : DeviceRegistryDao { @Query(PROMOTE_WHOOP5_RR_SOURCE_SQL) suspend fun promoteWhoop5RrSource(deviceId: String, ts: Long, rrMs: Int, seq: Int, ord: Int, source: Int) + @Query(PROMOTE_WHOOP4_HISTORY_SQL) + suspend fun promoteWhoop4HistoricalRr(deviceId: String, ts: Long, rrMs: Int, seq: Int, ord: Int) + @Query( "SELECT * FROM event WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + "ORDER BY ts ASC, kind ASC LIMIT :limit" diff --git a/android/app/src/main/java/com/noop/data/WhoopRepository.kt b/android/app/src/main/java/com/noop/data/WhoopRepository.kt index 03d88a75f6..f0a286172f 100644 --- a/android/app/src/main/java/com/noop/data/WhoopRepository.kt +++ b/android/app/src/main/java/com/noop/data/WhoopRepository.kt @@ -580,6 +580,9 @@ class WhoopRepository( // Counts stay separate; the canonical observation owns this key's order and provenance. dao.promoteWhoop5RrSource(row.deviceId, row.ts, row.rrMs, row.seq, row.ord!!, row.srcChannel) } + if (rrIds[index] == -1L && row.srcChannel == RrSourceChannel.WHOOP4_HISTORICAL.code) { + dao.promoteWhoop4HistoricalRr(row.deviceId, row.ts, row.rrMs, row.seq, row.ord!!) + } } val evIds = if (streams.events.isEmpty()) emptyList() else dao.insertEvents(streams.events.map { EventRow(deviceId, it.ts, it.kind, it.payloadJSON) }) @@ -1371,8 +1374,18 @@ class WhoopRepository( suspend fun rrIntervalsForDevice(deviceId: String, from: Long, to: Long, limit: Int = DEFAULT_LIMIT, unlabelledAliasOfWhoop5: Boolean = false): List = transactor.run { - if (isWhoop5RrSource(deviceId, unlabelledAliasOfWhoop5)) dao.whoop5RrIntervals(deviceId, from, to, limit) - else dao.rrIntervals(deviceId, from, to, limit) + when { + isWhoop5RrSource(deviceId, unlabelledAliasOfWhoop5) -> dao.whoop5RrIntervals(deviceId, from, to, limit) + isWhoop4RrSource(deviceId) || dao.hasWhoop4HistoricalRrSource(deviceId) -> + dao.whoop4RrIntervals(deviceId, from, to, limit) + else -> dao.rrIntervals(deviceId, from, to, limit) + } + } + + private suspend fun isWhoop4RrSource(deviceId: String): Boolean { + val owner = dao.pairedDevice(deviceId) + return com.noop.protocol.DeviceFamily.confirmedRegistryFamily(owner?.model, owner?.brand) == + com.noop.protocol.DeviceFamily.WHOOP4 } /** R-R beats over active strap + canonical history. Exact duplicate beats are removed with the @@ -2755,7 +2768,7 @@ class WhoopRepository( for (list in lists) for (beat in list) { byBeat.putIfAbsent(BeatKey(beat.ts, beat.rrMs, beat.seq), beat) } - if (byBeat.values.any { it.srcChannel in 5..7 }) { + if (byBeat.values.any { it.srcChannel in 5..7 || it.srcChannel == 8 }) { // Kotlin's stable sort preserves owner precedence and captured within-second order. return byBeat.values.sortedBy { it.ts } } diff --git a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt index e8a02b996b..238b872821 100644 --- a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt +++ b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt @@ -1010,7 +1010,8 @@ fun extractHistoricalStreams( @Suppress("UNCHECKED_CAST") (p["rr_intervals"] as? List)?.forEach { rrMs -> - rr.add(RrRow(ts, rrMs, RrSourceChannel.fromCode(p.intOrNull("rr_source_channel")))) + rr.add(RrRow(ts, rrMs, RrSourceChannel.fromCode(p.intOrNull("rr_source_channel")) + ?: if (family == DeviceFamily.WHOOP4) RrSourceChannel.WHOOP4_HISTORICAL else null)) } p.intOrNull("spo2_red")?.let { red -> diff --git a/android/app/src/main/java/com/noop/protocol/Streams.kt b/android/app/src/main/java/com/noop/protocol/Streams.kt index 6b56bfef47..88236f9f44 100644 --- a/android/app/src/main/java/com/noop/protocol/Streams.kt +++ b/android/app/src/main/java/com/noop/protocol/Streams.kt @@ -35,7 +35,8 @@ data class HrSample(val ts: Int, val bpm: Int) /** * The sensor channel or transport that produced an R-R interval. * - * WHOOP 5 exposes one beat train over several labelled transports. WHOOP 4 and legacy rows keep null. + * WHOOP 5 exposes one beat train over several labelled transports. WHOOP 4 type-47 history uses code 8; + * its standard BLE and legacy rows keep null. * An Oura ring has more than one optical channel: the green-quality tag (0x80) and the SpO2 tag (0x6E) both * decode to R-R and both were stored, so the table held roughly TWO complete copies of every night — * not duplicate rows to de-duplicate, but the SAME heartbeats measured twice. Labelling the channel is @@ -78,6 +79,8 @@ enum class RrSourceChannel(val code: Int) { WHOOP5_REALTIME(6), /** WHOOP 5 standard BLE 0x2A37, already converted to milliseconds. */ WHOOP5_STANDARD(7), + /** WHOOP 4.0 type-47 historical stream, distinguished from its unlabelled live BLE feed. */ + WHOOP4_HISTORICAL(8), ; val isWhoop5Transport: Boolean get() = code in 5..7 diff --git a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt index c4fbe6a381..487e3cc5c2 100644 --- a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt +++ b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt @@ -49,7 +49,7 @@ class HrFingerprintTest { @Test fun analysisFingerprintSqlRunsAndMovesForGravityOnlyCommit() { DriverManager.getConnection("jdbc:sqlite::memory:").use { db -> createFingerprintTables(db) - assertEquals("v3|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|registry", fingerprint(db)) + assertEquals("v4|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) db.createStatement().use { sql -> sql.executeUpdate("INSERT INTO hrSample(ts) VALUES (1000)") @@ -57,7 +57,7 @@ class HrFingerprintTest { sql.executeUpdate("INSERT INTO rrInterval DEFAULT VALUES") sql.executeUpdate("INSERT INTO gravitySample DEFAULT VALUES") } - assertEquals("v3|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|registry", fingerprint(db)) + assertEquals("v4|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) } } diff --git a/android/app/src/test/java/com/noop/data/RawTimelineRepositoryDaoTest.kt b/android/app/src/test/java/com/noop/data/RawTimelineRepositoryDaoTest.kt index fad01da957..c6983d24a6 100644 --- a/android/app/src/test/java/com/noop/data/RawTimelineRepositoryDaoTest.kt +++ b/android/app/src/test/java/com/noop/data/RawTimelineRepositoryDaoTest.kt @@ -251,6 +251,7 @@ class RawTimelineRepositoryDaoTest { when (method.name) { "pairedDevice", "activeDeviceId" -> null "hasWhoop5RrSource" -> false + "hasWhoop4HistoricalRrSource" -> false else -> call(method.name, args ?: emptyArray()) } } as WhoopDao diff --git a/android/app/src/test/java/com/noop/data/RrSourceChannelTest.kt b/android/app/src/test/java/com/noop/data/RrSourceChannelTest.kt index 2a0eb95deb..a114317d3e 100644 --- a/android/app/src/test/java/com/noop/data/RrSourceChannelTest.kt +++ b/android/app/src/test/java/com/noop/data/RrSourceChannelTest.kt @@ -164,7 +164,8 @@ class RrSourceChannelTest { */ @Test fun theTwoChannelEnumsAgreeCaseForCaseAndCodeForCode() { - assertEquals(OuraIbiChannel.entries.size, RrSourceChannel.entries.count { !it.isWhoop5Transport }) + assertEquals(OuraIbiChannel.entries.size, + RrSourceChannel.entries.count { !it.isWhoop5Transport && it.code <= 4 }) for (c in OuraIbiChannel.entries) { assertEquals( "$c must map to the SAME durable storage code on both sides", diff --git a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt index 3b0b12a562..eefdf6d5ef 100644 --- a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt +++ b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt @@ -119,8 +119,18 @@ class Whoop5RRSqliteTest { .use { it.executeUpdate() } Unit } - "rrIntervals", "whoop5RrIntervals" -> query( - if (method.name == "whoop5RrIntervals") WHOOP5_RR_INTERVALS_SQL else RR_INTERVALS_SQL, + "promoteWhoop4HistoricalRr" -> { + statement(PROMOTE_WHOOP4_HISTORY_SQL, + listOf("deviceId", "ts", "rrMs", "seq", "ord").zip(args.take(5)).toMap()) + .use { it.executeUpdate() } + Unit + } + "rrIntervals", "whoop5RrIntervals", "whoop4RrIntervals" -> query( + when (method.name) { + "whoop5RrIntervals" -> WHOOP5_RR_INTERVALS_SQL + "whoop4RrIntervals" -> WHOOP4_RR_INTERVALS_SQL + else -> RR_INTERVALS_SQL + }, listOf("deviceId", "from", "to", "limit").zip(args.take(4)).toMap(), ) { r -> fun optional(column: String) = r.getInt(column).let { if (r.wasNull()) null else it } @@ -130,6 +140,9 @@ class Whoop5RRSqliteTest { "hasWhoop5RrSource" -> query(HAS_WHOOP5_RR_SOURCE_SQL, mapOf("deviceId" to args[0])) { it.getBoolean(1) }.single() + "hasWhoop4HistoricalRrSource" -> query( + "SELECT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = ? AND srcChannel = 8)", + mapOf("deviceId" to args[0])) { it.getBoolean(1) }.single() "legacyWhoop5RrWithheld" -> query(LEGACY_WHOOP5_RR_WITHHELD_SQL, listOf("deviceId", "from", "to").zip(args.take(3)).toMap()) { it.getBoolean(1) @@ -357,7 +370,7 @@ class Whoop5RRSqliteTest { @Test fun sourceFingerprintQueriesUseCoveringIndex() { val plan = query("EXPLAIN QUERY PLAN $ANALYSIS_FINGERPRINT_SQL") { it.getString("detail") } - assertEquals(3, plan.count { it.contains("USING COVERING INDEX rrInterval_source_suspect") }) + assertEquals(4, plan.count { it.contains("USING COVERING INDEX rrInterval_source_suspect") }) assertFalse(plan.any { it.contains("SCAN rrInterval") }) } @@ -656,6 +669,32 @@ class Whoop5RRSqliteTest { assertEquals(1, dao.rrIntervals(id, 0, 1000, 100).single().srcChannel) } + @Test fun whoop4HistoricalRowsWinPerTimestampAndPromoteLegacyRows() = runBlocking { + registry("4.0") + repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(101, 810))), id) + repo.insert(StreamBatch(rr = listOf( + RrRow(100, 800, RrSourceChannel.WHOOP4_HISTORICAL), + RrRow(101, 820, RrSourceChannel.WHOOP4_HISTORICAL), + )), id) + val rows = repo.rrIntervalsForDevice(id, 100, 101, 100) + assertEquals(listOf(800, 820), rows.map { it.rrMs }) + assertEquals(listOf(8, 8), rows.map { it.srcChannel }) + } + + @Test fun whoop4PrefersOneHistoricalTransportForTheWholeWindow() = runBlocking { + registry("4.0") + repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(101, 810))), id) + repo.insert(StreamBatch(rr = listOf(RrRow(100, 805, RrSourceChannel.WHOOP4_HISTORICAL))), id) + val rows = repo.rrIntervalsForDevice(id, 100, 101, 100) + assertEquals(listOf(805), rows.map { it.rrMs }) + } + + @Test fun whoop4FallsBackToUnlabelledRowsWhenNoHistoryExists() = runBlocking { + registry("4.0") + repo.insert(StreamBatch(rr = listOf(RrRow(100, 800))), id) + assertEquals(listOf(800), repo.rrIntervalsForDevice(id, 0, 1000).map { it.rrMs }) + } + @Test fun firstTagOutsideDayAndSuspectPromotionInvalidateOwnerPolicyCaches() = runBlocking { registry("WHOOP") repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(2000, 900))), id) diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt index fcabe3030a..e716b759b0 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt @@ -88,7 +88,7 @@ class Whoop5RRTest { } @Test fun transportCodes() { - assertEquals((1..7).toList(), RrSourceChannel.entries.map { it.code }) + assertEquals((1..8).toList(), RrSourceChannel.entries.map { it.code }) } /**