diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift index dc73ee5cd2..3bb424bb33 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift @@ -388,13 +388,28 @@ private func decodeWhoop5Historical(_ frame: [UInt8], fb: FieldBuilder, payloadE } } fb.parsed["rr_intervals"] = .intArray(rrs) - // Bytes adjacent to the HR/R-R fields, read off real frames. @36 / 256 tracks the integer hr@22 to - // sub-bpm (corr 0.989 over ~258k records) — a higher-precision heart rate; the others are raw. + // Bytes adjacent to the HR/R-R fields, read off real frames: @36 is a FLAG byte and @37 a duplicate + // heart rate — not the two halves of one fixed-point HR (see below); the others are raw. if let v = readDType(frame, 33, "u8") { fb.add(33, 1, "cardiac_flags", "cardiac", value: .int(v), note: "raw byte near the HR fields") } - if let v = readDType(frame, 36, "u16") { - fb.add(36, 2, "hr_fixed_8_8", "hr", value: .int(v), note: "higher-precision HR: bpm = value/256") + // @36 was read as the low half of a u16 `hr_fixed_8_8` with bpm = value/256. Over 18,650 real v18 + // records that model is false. Bit 4 is NEVER set (0/18,650 — a genuine 8.8 fraction sets it ~50% of + // the time, and it is the ONLY bit never set); 95.02% of values land in 0x80–0x8F where uniform would + // be 6.25%, over just 40 distinct values, sd 26.5 vs 73.9 for a uniform byte. The "corr 0.989 with + // hr@22" that justified the old name was circular — the u16 is literally hr@22 (carried at @37) plus + // this flag byte over 256, so the residual is a flat +0.504 ± 0.189. Bit 7 reads as a VALIDITY bit: + // with it clear (n=748) rr_count == 0 in 70.32% of records vs 19.82% with it set, and the @108/@109 + // sentinel fires in 69.65% vs 1.32%. The remaining bits are unpinned, so the byte ships raw. + if let v = readDType(frame, 36, "u8") { + fb.add(36, 1, "hr_quality_flags", "hr", value: .int(v), + note: "flag byte (raw); bit7 = HR/R-R valid, bit4 never set; NOT a fixed-point HR") + } + // @37 duplicates heart_rate@22 — equal in 99.575% of records (18,523/18,602), differing only by + // -6…+2, and it only tracks HR while @36 bit7 is set (99.74% exact vs 94.12% when clear). + if let v = readDType(frame, 37, "u8") { + fb.add(37, 1, "heart_rate_alt", "hr", value: .int(v), + note: "bpm; duplicate of heart_rate@22 (99.6% exact); trust only when hr_quality_flags bit7 is set") } if let v = readDType(frame, 38, "u16") { fb.add(38, 2, "rr_packed", "rr", value: .int(v), note: "raw u16 near the R-R fields; meaning not pinned") @@ -507,28 +522,45 @@ private func decodeWhoop5Historical(_ frame: [UInt8], fb: FieldBuilder, payloadE // ── The @82–119 "optical/perfusion + tail" span, characterised over 18,602 real v18 records from a // third strap (overnight R22 live stream) + cross-checked on the two fixture devices above. The span // is ~85% ZERO PADDING: bytes 83–103, 110–112 and 117–119 are constant 0x00 on every record, and @104 - // is a constant 0x01 marker (5/5 fixtures, both devices). Only four groups carry data — @106 (u16), - // @108/@109 (a paired channel) and the @113 float — and none is physiologically ground-truth-named - // (no SpO2/respiratory reference exists), so each is carried RAW with its observed behaviour, never + // is a constant 0x01 marker (5/5 fixtures, both devices). Only two byte PAIRS carry data — @106/@107 + // and @108/@109 — plus the @113 float, and none is physiologically ground-truth-named (no + // SpO2/respiratory reference exists), so each is carried RAW with its observed behaviour, never // mapped to a named metric. This documents the region honestly instead of leaving 38 opaque bytes. - if let v = readDType(frame, 106, "u16") { - // An analog u16 that wanders across the night with no clean correlate to HR/motion/skin-temp, and - // reads 0 only when the strap is off-wrist (HR=0). Optical/ADC-baseline-like; raw, not pinned. - fb.add(106, 2, "optical_baseline_106", "optical", value: .int(v), - note: "u16 LE analog optical/ADC baseline; wanders overnight, 0 = off-wrist; raw, not pinned") - } - // @108/@109 are a tightly-coupled PAIR (equal in 23.5% of records, within ±2 in ~80%). Both rise - // monotonically with heart rate (mean ~34 at HR 40–49 → ~58 at HR 80–89) and with motion, and both - // read 128 as a per-channel INVALID sentinel — observed off-wrist AND on some worn records that still - // carry a valid HR (the optical channel can be invalid while HR is derived elsewhere). Amplitude- or + // + // @106/@107 were read as ONE u16 LE. They cannot be: across 18,599 consecutive-second pairs the HIGH + // byte changed while the LOW byte stayed frozen in 3,514 of them (18.89%), and the corpus holds ZERO + // low-byte wrap events — a real u16 cannot step its high byte without a carry. The apparent u16 deltas + // are exactly 256·Δ@107 + Δ@106 (they cluster at 0, ±1, ±255, ±256, ±257, ±513). These are two + // correlated but independent u8 channels (corr +0.73; they move in OPPOSITE directions in 5.8% of the + // pairs where both move), structurally parallel to the @108/@109 pair. 0 — not 128 — is the off-wrist + // marker: both bytes read 0 in exactly the 8 records that also carry HR == 0, while 128 occurs + // unremarkably while worn (@106 in 10 records, @107 in 103). Magnitudes are device-specific (102–255 / + // 119–247 on the R22 strap vs 20–66 / 34–81 on another), so no scale is asserted. + for (name, off) in [("optical_baseline_a", 106), ("optical_baseline_b", 107)] { + if let v = readDType(frame, off, "u8") { + fb.add(off, 1, name, "optical", value: .int(v), + note: "u8 optical/ADC baseline channel; wanders overnight, 0 = off-wrist; raw, not pinned") + } + } + // @108/@109 are a tightly-coupled PAIR (equal in 23.5% of records, within ±2 in ~80%). 128 is a + // RECORD-level sentinel, not a per-channel one: amp_a == 128 in 757 records and amp_b == 128 in 757 — + // the SAME 757, never one without the other. They do NOT rise with heart rate; that reading (mean ~34 + // at HR 40–49 → ~58 at 80–89) was an averaging artifact of counting the 128 sentinel as a number, and + // with sentinels excluded the trend is flat-to-declining (32.45 → 29.52). The real monotone trend is + // with MOTION: ~32.7 while still (dyn_acc < 0.02 g) → 37.4 at 0.05–0.2 g. Amplitude- or // signal-quality-like; carried raw, NOT named SpO2/perfusion without on-device ground truth. + // + // The sentinel itself is a usable per-second SIGNAL-QUALITY flag: it fires on 4.02% of worn seconds + // and predicts the band's own beat-detection failure (rr_count == 0) at 79.44% vs 19.40% — a 4.09× + // lift that SURVIVES holding motion constant (4.11× within dyn_acc < 0.009 g), where shuffled and + // circular-shift nulls all sit at ~1.0×. It is the same quality condition @36 bit7 reports. if let a = readDType(frame, 108, "u8") { fb.add(108, 1, "optical_amp_a", "optical", value: .int(a), - note: "paired optical channel A (≈ optical_amp_b@109); rises with HR/motion; 128 = channel invalid; raw") + note: "paired optical channel A (≈ optical_amp_b@109); tracks motion, not HR; 128 = record-level signal-quality sentinel; raw") } if let b = readDType(frame, 109, "u8") { fb.add(109, 1, "optical_amp_b", "optical", value: .int(b), - note: "paired optical channel B (see optical_amp_a@108); 128 = channel invalid; raw") + note: "paired optical channel B (see optical_amp_a@108); 128 = record-level sentinel, always together with @108; raw") } if let d = readF32(frame, 113), d.isFinite { // A float32 at @113 (observed range ~ -5.3…0, 0 = unset); purpose unknown, carried raw. @@ -543,7 +575,7 @@ private func decodeWhoop5Historical(_ frame: [UInt8], fb: FieldBuilder, payloadE // undecodable records, the raw-capture batch when that toggle is on), so a future re-decode is // lossless. Kept in lockstep with the Android decodeWhoop5Historical provenance note. if let payloadEnd = payloadEnd, 82 < payloadEnd, payloadEnd <= frame.count { - fb.region(82, payloadEnd, "optical/tail (mostly zero padding; see @106/@108/@109/@113)", "unknown") + fb.region(82, payloadEnd, "optical/tail (mostly zero padding; see @106/@107/@108/@109/@113)", "unknown") } } diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Resources/decoder_oracle.json b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Resources/decoder_oracle.json index 10049ae872..92da383d30 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Resources/decoder_oracle.json +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Resources/decoder_oracle.json @@ -91,7 +91,10 @@ "synthetic": false, "hex": "aa01740001003fb12f1280733d8401b69f266a66460066025a0265020000000000007b0a8d656463ff0012163cf6a439bf2924fd3ed763fe3e3200aa000000000000000000f7000901f10b0007010c020c00000000000000000000000000000000000000000000000100656f1e1e0000009d61a7c00000003e862817", "expect": { - "optical_baseline_106": 28517, + "optical_baseline_a": 101, + "optical_baseline_b": 111, + "hr_quality_flags": 141, + "heart_rate_alt": 101, "optical_amp_a": 30, "optical_amp_b": 30, "hist_version": 18, @@ -104,7 +107,6 @@ "motion_wear_quality": 0, "activity_class": 0, "record_index": 25443699, - "hr_fixed_8_8": 25997, "step_cadence": 170, "status_word": 1792, "status_word_1": 3073, @@ -143,7 +145,10 @@ "synthetic": false, "hex": "aa01740001003fb12f12803a3d84018889266a3d0a00000000000000000000000000000000000000000064c33b52b47d3fe1ba1dbda470ecbd000064000000000000000000e500e200c708000c010c020c0000000000000000000000000000000000000000000000010000008080000000000000000000009ffafe6c", "expect": { - "optical_baseline_106": 0, + "optical_baseline_a": 0, + "optical_baseline_b": 0, + "hr_quality_flags": 0, + "heart_rate_alt": 0, "optical_amp_a": 128, "optical_amp_b": 128, "hist_version": 18, @@ -177,7 +182,10 @@ "synthetic": false, "hex": "aa01740001003fb12f128093c47c006dbc296a00600039000000000000000000006137020b610000e1e04c063d8fce36bf7b08233f8fea993e38a50000000000000000000019012101920b5002010c020c0100000000000000000000000000000000000000000005010085808080000000a5538ec000000016d0680d", "expect": { - "optical_baseline_106": 32901, + "optical_baseline_a": 133, + "optical_baseline_b": 128, + "hr_quality_flags": 11, + "heart_rate_alt": 97, "optical_amp_a": 128, "optical_amp_b": 128, "hist_version": 18, @@ -196,7 +204,10 @@ "synthetic": false, "hex": "aa01740001003fb12f128034d67c000ece296a0a77003f01fc0300000000000000203a0302e30000ff00f8093c1f8534bcc3a5473f4819243f85a6000000000000000000001b012601e80b5003010c020c00000000000000000000000000000000000000000000050100ced7241c0000006ead8cc00000008775b70c", "expect": { - "optical_baseline_106": 55246, + "optical_baseline_a": 206, + "optical_baseline_b": 215, + "hr_quality_flags": 2, + "heart_rate_alt": 227, "optical_amp_a": 36, "optical_amp_b": 28, "hist_version": 18, diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5HistoricalTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5HistoricalTests.swift index 2730a4801c..35cd1e0522 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5HistoricalTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5HistoricalTests.swift @@ -107,15 +107,16 @@ final class Whoop5HistoricalTests: XCTestCase { func testHistoricalV18ObservedFields() { // Fields read off this same real worn frame and justified by their observed behaviour: // @11 record_index — a per-record counter (+1/record, independent of unix; seen on two straps) - // @36 hr_fixed_8_8 — value/256 tracks hr@22 to sub-bpm (here 25997/256 ≈ 101.55 ≈ HR 102) + // @36 hr_quality_flags — a flag byte (bit7 = valid), NOT the low half of a fixed-point HR + // @37 heart_rate_alt — a duplicate of heart_rate@22 // @59 step_cadence — a cadence-like byte (never 0; lower when moving faster) // @75 status_word — a 16-bit word that is NOT a deep-sleep marker // @81 sleep_state — high nibble = band state (worn daytime frame = wake) // @33/@38/@40 — raw bytes near the HR/R-R fields; @113 — a float of unknown purpose let p = parseFrame(bytes(historicalHex), family: .whoop5).parsed XCTAssertEqual(p["record_index"]?.intValue, 25443699) - XCTAssertEqual(p["hr_fixed_8_8"]?.intValue, 25997) - XCTAssertEqual((p["hr_fixed_8_8"]?.intValue ?? 0) / 256, 101) // ≈ hr@22 (102) + XCTAssertEqual(p["hr_quality_flags"]?.intValue, 141) // 0x8D + XCTAssertEqual(p["heart_rate_alt"]?.intValue, 101) // ≈ hr@22 (102) XCTAssertEqual(p["step_cadence"]?.intValue, 170) XCTAssertEqual(p["status_word"]?.intValue, 1792) XCTAssertEqual(p["sleep_state"]?.intValue, 0) @@ -128,34 +129,98 @@ final class Whoop5HistoricalTests: XCTestCase { func testHistoricalV18OpticalRegionFields() { // The @82–119 "optical/tail" span, reverse-engineered over 18,602 real v18 records (a third strap's // overnight R22 stream) and cross-checked here on the two fixture devices. It is ~85% zero padding; - // only @106 (u16), @108/@109 (a paired channel) and the @113 float carry data. These are carried - // RAW — none is named to a physiological metric (no SpO2/respiratory ground truth exists). + // only @106/@107 (a byte pair), @108/@109 (a second byte pair) and the @113 float carry data. These + // are carried RAW — none is named to a physiological metric (no SpO2/respiratory ground truth). let worn = parseFrame(bytes(historicalHex), family: .whoop5).parsed - // @106: analog optical baseline (worn nonzero; see the off-wrist case below for the 0 sentinel). - XCTAssertEqual(worn["optical_baseline_106"]?.intValue, 28517) - // @108/@109: a tightly-coupled pair (here 30/30). Both < 128 ⇒ the optical channel is valid. + // @106/@107: two independent u8 baseline channels (worn nonzero; 0 = off-wrist, see below). + XCTAssertEqual(worn["optical_baseline_a"]?.intValue, 101) + XCTAssertEqual(worn["optical_baseline_b"]?.intValue, 111) + // @108/@109: a tightly-coupled pair (here 30/30). Both < 128 ⇒ the optical read is valid. XCTAssertEqual(worn["optical_amp_a"]?.intValue, 30) XCTAssertEqual(worn["optical_amp_b"]?.intValue, 30) - // Off-wrist (HR=0): @106 collapses to 0 and BOTH channels read the 128 invalid sentinel. + // Off-wrist (HR=0): BOTH baseline bytes collapse to 0 — the real off-wrist marker (b106 == 0 in + // exactly the 8 corpus records that also carry HR == 0, and b107 == 0 in those same 8) — and both + // amp bytes read the 128 sentinel. let off = parseFrame(bytes(historicalOffWristHex), family: .whoop5).parsed - XCTAssertEqual(off["optical_baseline_106"]?.intValue, 0) + XCTAssertEqual(off["optical_baseline_a"]?.intValue, 0) + XCTAssertEqual(off["optical_baseline_b"]?.intValue, 0) XCTAssertEqual(off["optical_amp_a"]?.intValue, 128) XCTAssertEqual(off["optical_amp_b"]?.intValue, 128) - // Cross-device: on the SECOND strap, HR=57 decodes fine yet the optical channel is 128/128 — proof - // that 128 is a per-CHANNEL invalid marker independent of HR validity (HR is derived elsewhere), - // while HR=63 on the same strap carries a valid pair (36/28). + // Cross-device: on the SECOND strap, HR=57 decodes fine yet the amp pair is 128/128 — the sentinel + // marks a bad OPTICAL read, not an absent HR (HR is derived elsewhere); HR=63 carries a valid pair. let d2a = parseFrame(bytes(secondDeviceHR57), family: .whoop5).parsed XCTAssertEqual(d2a["heart_rate"]?.intValue, 57) XCTAssertEqual(d2a["optical_amp_a"]?.intValue, 128) XCTAssertEqual(d2a["optical_amp_b"]?.intValue, 128) + // …and that same worn frame reads 128 on the @107 BASELINE byte while carrying a valid HR: 128 is + // not a sentinel there (b106 == 128 in 10 corpus records, b107 == 128 in 103, none of them + // off-wrist), which is why only 0 marks off-wrist on @106/@107. + XCTAssertEqual(d2a["optical_baseline_b"]?.intValue, 128) let d2b = parseFrame(bytes(secondDeviceHR63), family: .whoop5).parsed XCTAssertEqual(d2b["heart_rate"]?.intValue, 63) XCTAssertEqual(d2b["optical_amp_a"]?.intValue, 36) XCTAssertEqual(d2b["optical_amp_b"]?.intValue, 28) } + func testHistoricalV18Byte36IsAFlagByteNotAFixedPointHR() { + // @36/@37 was read as one u16 `hr_fixed_8_8` with bpm = value/256. Over 18,650 real v18 records + // that model is false. @36 behaves as a FLAG byte: bit 4 is NEVER set (0/18,650 — a genuine 8.8 + // fraction sets it ~50% of the time) and is the only bit never set; 95.02% of values fall in + // 0x80–0x8F (uniform would be 6.25%) over just 40 distinct values, sd 26.5 vs 73.9 for a uniform + // byte. @37 is a DUPLICATE heart rate: it equals heart_rate@22 exactly in 99.575% of records + // (18,523/18,602), differing only by -6…+2. The "corr 0.989" that justified the old name is + // circular — the u16 is literally hr@22 plus a flag byte over 256, and the residual + // (value/256 - hr@22) is +0.504 ± 0.189, i.e. @36/256. + let worn = parseFrame(bytes(historicalHex), family: .whoop5).parsed + XCTAssertNil(worn["hr_fixed_8_8"], "the 8.8 fixed-point reading is retired") + XCTAssertEqual(worn["hr_quality_flags"]?.intValue, 141) // 0x8D + XCTAssertEqual(worn["heart_rate_alt"]?.intValue, 101) // hr@22 = 102 + XCTAssertEqual((worn["hr_quality_flags"]?.intValue ?? 0) & 0x10, 0, "bit 4 is never set") + XCTAssertEqual((worn["hr_quality_flags"]?.intValue ?? 0) & 0x80, 0x80, "bit 7 = valid") + + // Bit 7 is a VALIDITY bit: with it CLEAR (n=748 in the corpus) rr_count == 0 in 70.32% of records + // vs 19.82% with it set, and the @108/@109 sentinel fires in 69.65% vs 1.32%. Both second-device + // fixtures have it clear — and there @37 is NOT a heart rate (97 and 227 against hr@22 57 and 63), + // which the retired model would have surfaced as a 227 bpm "high-precision" reading. + for (hex, hr, flags, alt) in [(secondDeviceHR57, 57, 11, 97), (secondDeviceHR63, 63, 2, 227)] { + let p = parseFrame(bytes(hex), family: .whoop5).parsed + XCTAssertEqual(p["heart_rate"]?.intValue, hr) + XCTAssertEqual(p["hr_quality_flags"]?.intValue, flags) + XCTAssertEqual((p["hr_quality_flags"]?.intValue ?? 0) & 0x80, 0, "bit 7 clear ⇒ invalid") + XCTAssertEqual(p["heart_rate_alt"]?.intValue, alt, "not a bpm when the validity bit is clear") + } + } + + func testHistoricalV18OpticalBaselineIsTwoBytesNotAU16() { + // @106 was read as a u16 LE. It cannot be one: across 18,599 consecutive-second pairs the HIGH byte + // changed while the LOW byte stayed frozen in 3,514 pairs (18.89%), and there are ZERO low-byte + // wrap events in the whole corpus — a real u16 cannot step its high byte without a carry. The + // apparent u16 deltas are just 256·Δb107 + Δb106 (they cluster at 0, ±1, ±255, ±256, ±257, ±513). + // @106/@107 are two correlated but independent u8 channels (corr +0.73, moving in OPPOSITE + // directions in 5.8% of pairs), structurally parallel to the @108/@109 pair. + var f = bytes(historicalHex) + f[107] = 0xFF + let m = parseFrame(f, family: .whoop5).parsed + XCTAssertNil(m["optical_baseline_106"], "the u16 reading is retired") + // Moving @107 must leave the @106 channel alone — under the u16 model this byte was worth 256. + XCTAssertEqual(m["optical_baseline_a"]?.intValue, 101) + XCTAssertEqual(m["optical_baseline_b"]?.intValue, 255) + } + + func testHistoricalV18OpticalAmpSentinelIsRecordLevel() { + // The 128 sentinel on @108/@109 was noted as invalidating a single CHANNEL. It is RECORD-level: + // over 18,650 records amp_a == 128 in 757 and amp_b == 128 in 757 — the SAME 757 records, never + // one without the other. All four real fixtures agree, so the two bytes are never split. + for (name, hex) in [("worn", historicalHex), ("offwrist", historicalOffWristHex), + ("device2_hr57", secondDeviceHR57), ("device2_hr63", secondDeviceHR63)] { + let p = parseFrame(bytes(hex), family: .whoop5).parsed + XCTAssertEqual(p["optical_amp_a"]?.intValue == 128, p["optical_amp_b"]?.intValue == 128, + "\(name): the 128 sentinel must fire on both amp bytes or neither") + } + } + func testHistoricalV18OpticalFieldsAreNotNamedPhysiologically() { // Guard the project rule (never invent physiology): the paired optical channel must NOT be surfaced // as SpO2 or a named perfusion metric — those keys stay absent on v18 until real ground truth pins diff --git a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt index e381ad5c32..6b81847d23 100644 --- a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt +++ b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt @@ -302,10 +302,21 @@ private fun decodeWhoop5Historical(frame: ByteArray): Map? { if (v != null && v != 0) rrVals.add(v) } out["rr_intervals"] = rrVals - // Bytes adjacent to the HR/R-R fields. @36/256 tracks hr@22 to sub-bpm (corr 0.989) — a - // higher-precision heart rate; the others are carried raw (meaning not pinned). + // Bytes adjacent to the HR/R-R fields: @36 is a FLAG byte and @37 a duplicate heart rate — not the + // two halves of one fixed-point HR; the others are carried raw (meaning not pinned). frame.histU8(33)?.let { out["cardiac_flags"] = it } - frame.histU16(36)?.let { out["hr_fixed_8_8"] = it } // bpm = value / 256 + // @36 was read as the low half of a u16 `hr_fixed_8_8` with bpm = value/256. Over 18,650 real v18 + // records that model is false. Bit 4 is NEVER set (0/18,650 — a genuine 8.8 fraction sets it ~50% of + // the time, and it is the ONLY bit never set); 95.02% of values land in 0x80–0x8F where uniform would + // be 6.25%, over just 40 distinct values, sd 26.5 vs 73.9 for a uniform byte. The "corr 0.989 with + // hr@22" that justified the old name was circular — the u16 is literally hr@22 (carried at @37) plus + // this flag byte over 256, so the residual is a flat +0.504 ± 0.189. Bit 7 reads as a VALIDITY bit: + // with it clear (n=748) rr_count == 0 in 70.32% of records vs 19.82% with it set, and the @108/@109 + // sentinel fires in 69.65% vs 1.32%. The remaining bits are unpinned, so the byte ships raw. + frame.histU8(36)?.let { out["hr_quality_flags"] = it } // bit7 = valid, bit4 never set + // @37 duplicates heart_rate@22 — equal in 99.575% of records (18,523/18,602), differing only by + // -6…+2, and it only tracks HR while @36 bit7 is set (99.74% exact vs 94.12% when clear). + frame.histU8(37)?.let { out["heart_rate_alt"] = it } frame.histU16(38)?.let { out["rr_packed"] = it } frame.histU8(40)?.let { out["cardiac_status"] = it } frame.histF32(45)?.let { out["gravity_x"] = it } @@ -366,15 +377,33 @@ private fun decodeWhoop5Historical(frame: ByteArray): Map? { } // ── The @82–119 "optical/tail" span, reverse-engineered over 18,602 real v18 records (a third strap's // overnight R22 stream) + cross-checked on two fixture devices: it is ~85% ZERO PADDING (83–103, - // 110–112, 117–119 constant 0x00; @104 a constant 0x01 marker). Only @106 (u16), @108/@109 (a paired - // channel) and the @113 float carry data, and none is physiologically ground-truth-named (no + // 110–112, 117–119 constant 0x00; @104 a constant 0x01 marker). Only the byte pairs @106/@107 and + // @108/@109 plus the @113 float carry data, and none is physiologically ground-truth-named (no // SpO2/respiratory reference), so each is carried RAW. Mirror of Swift decodeWhoop5Historical. - // @106 an analog u16 optical/ADC baseline: wanders overnight, reads 0 only off-wrist; raw, not pinned. - frame.histU16(106)?.let { out["optical_baseline_106"] = it } - // @108/@109 a tightly-coupled PAIR (equal ~24% of records, within ±2 ~80%). Both rise monotonically - // with HR/motion and read 128 as a per-CHANNEL invalid sentinel — seen off-wrist AND on worn records - // that still carry a valid HR. Amplitude/quality-like; carried raw, NOT named SpO2/perfusion without - // on-device ground truth. + // + // @106/@107 were read as ONE u16 LE. They cannot be: across 18,599 consecutive-second pairs the HIGH + // byte changed while the LOW byte stayed frozen in 3,514 of them (18.89%), and the corpus holds ZERO + // low-byte wrap events — a real u16 cannot step its high byte without a carry. The apparent u16 deltas + // are exactly 256·Δ@107 + Δ@106 (they cluster at 0, ±1, ±255, ±256, ±257, ±513). These are two + // correlated but independent u8 channels (corr +0.73; they move in OPPOSITE directions in 5.8% of the + // pairs where both move), structurally parallel to the @108/@109 pair. 0 — not 128 — is the off-wrist + // marker: both bytes read 0 in exactly the 8 records that also carry HR == 0, while 128 occurs + // unremarkably while worn (@106 in 10 records, @107 in 103). Magnitudes are device-specific (102–255 / + // 119–247 on the R22 strap vs 20–66 / 34–81 on another), so no scale is asserted. + frame.histU8(106)?.let { out["optical_baseline_a"] = it } + frame.histU8(107)?.let { out["optical_baseline_b"] = it } + // @108/@109 a tightly-coupled PAIR (equal ~24% of records, within ±2 ~80%). 128 is a RECORD-level + // sentinel, not a per-channel one: amp_a == 128 in 757 records and amp_b == 128 in 757 — the SAME 757, + // never one without the other. They do NOT rise with HR; that reading (~34 at HR 40–49 → ~58 at 80–89) + // was an averaging artifact of counting the 128 sentinel as a number, and with sentinels excluded the + // trend is flat-to-declining (32.45 → 29.52). The real monotone trend is with MOTION: ~32.7 while + // still (dyn_acc < 0.02 g) → 37.4 at 0.05–0.2 g. Amplitude/quality-like; carried raw, NOT named + // SpO2/perfusion without on-device ground truth. + // + // The sentinel itself is a usable per-second SIGNAL-QUALITY flag: it fires on 4.02% of worn seconds + // and predicts the band's own beat-detection failure (rr_count == 0) at 79.44% vs 19.40% — a 4.09× + // lift that SURVIVES holding motion constant (4.11× within dyn_acc < 0.009 g), where shuffled and + // circular-shift nulls all sit at ~1.0×. It is the same quality condition @36 bit7 reports. frame.histU8(108)?.let { out["optical_amp_a"] = it } frame.histU8(109)?.let { out["optical_amp_b"] = it } // @113 a float32 (observed range ~ -5.3..0, 0 = unset); purpose unknown, carried raw. EMPIRICAL. diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5HistoricalDecodeTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5HistoricalDecodeTest.kt index f84a7dc087..8a09d34911 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5HistoricalDecodeTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5HistoricalDecodeTest.kt @@ -87,8 +87,8 @@ class Whoop5HistoricalDecodeTest { // Swift Whoop5HistoricalTests.testHistoricalV18ObservedFields). val p = decodeHistorical(bytes(wornV18), DeviceFamily.WHOOP5)!! assertEquals(25443699, p["record_index"]) // @11 per-record counter - assertEquals(25997, p["hr_fixed_8_8"]) // @36 value/256 ≈ HR (101.55 ≈ 102) - assertEquals(101, (p["hr_fixed_8_8"] as Int) / 256) + assertEquals(141, p["hr_quality_flags"]) // @36 flag byte (0x8D), NOT a fixed-point HR + assertEquals(101, p["heart_rate_alt"]) // @37 duplicate HR (hr@22 = 102) assertEquals(170, p["step_cadence"]) // @59 cadence-like byte (raw) assertEquals(1792, p["status_word"]) // @75 not a deep-sleep marker assertEquals(0, p["sleep_state"]) // @81 worn daytime frame = wake @@ -269,29 +269,95 @@ class Whoop5HistoricalDecodeTest { @Test fun decodesV18OpticalRegionFields() { // Mirror of Swift testHistoricalV18OpticalRegionFields. The @82-119 "optical/tail" span is ~85% - // zero padding; only @106 (u16), @108/@109 (a paired channel) and the @113 float carry data, all + // zero padding; only the byte pairs @106/@107 and @108/@109 plus the @113 float carry data, all // RAW (no SpO2/respiratory ground truth). Cross-checked on both fixture devices. val worn = decodeHistorical(bytes(wornV18), DeviceFamily.WHOOP5)!! - assertEquals(28517, worn["optical_baseline_106"]) + assertEquals(101, worn["optical_baseline_a"]) // @106 + assertEquals(111, worn["optical_baseline_b"]) // @107 assertEquals(30, worn["optical_amp_a"]) assertEquals(30, worn["optical_amp_b"]) - // Off-wrist: @106 collapses to 0 and BOTH channels read the 128 invalid sentinel. + // Off-wrist: BOTH baseline bytes collapse to 0 — the real off-wrist marker (b106 == 0 in exactly + // the 8 corpus records that also carry HR == 0, b107 == 0 in those same 8) — and both amp bytes + // read the 128 sentinel. val off = decodeHistorical(bytes(offWristV18), DeviceFamily.WHOOP5)!! - assertEquals(0, off["optical_baseline_106"]) + assertEquals(0, off["optical_baseline_a"]) + assertEquals(0, off["optical_baseline_b"]) assertEquals(128, off["optical_amp_a"]) assertEquals(128, off["optical_amp_b"]) - // Cross-device: HR=57 decodes fine yet the optical channel is 128/128 (per-channel invalid, - // independent of HR validity); HR=63 on the same strap carries a valid 36/28 pair. + // Cross-device: HR=57 decodes fine yet the amp pair is 128/128 — the sentinel marks a bad OPTICAL + // read, not an absent HR; HR=63 on the same strap carries a valid 36/28 pair. val d57 = decodeHistorical(bytes(secondDeviceHR57), DeviceFamily.WHOOP5)!! assertEquals(57, d57["heart_rate"]) assertEquals(128, d57["optical_amp_a"]) assertEquals(128, d57["optical_amp_b"]) + // …and that same worn frame reads 128 on the @107 BASELINE byte while carrying a valid HR: 128 is + // not a sentinel there (b106 == 128 in 10 corpus records, b107 == 128 in 103, none off-wrist). + assertEquals(128, d57["optical_baseline_b"]) val d63 = decodeHistorical(bytes(secondDeviceHR63), DeviceFamily.WHOOP5)!! assertEquals(63, d63["heart_rate"]) assertEquals(36, d63["optical_amp_a"]) assertEquals(28, d63["optical_amp_b"]) } + @Test + fun v18Byte36IsAFlagByteNotAFixedPointHR() { + // Mirror of Swift testHistoricalV18Byte36IsAFlagByteNotAFixedPointHR. @36/@37 was read as one u16 + // `hr_fixed_8_8` with bpm = value/256. Over 18,650 real v18 records that model is false: bit 4 of + // @36 is NEVER set (0/18,650 — a genuine 8.8 fraction sets it ~50% of the time, and it is the only + // bit never set), 95.02% of values land in 0x80-0x8F (uniform = 6.25%) over just 40 distinct + // values, sd 26.5 vs 73.9. @37 equals heart_rate@22 exactly in 99.575% (18,523/18,602), differing + // only by -6..+2. The "corr 0.989" that justified the old name was circular: the u16 is hr@22 plus + // a flag byte over 256, residual +0.504 ± 0.189. + val worn = decodeHistorical(bytes(wornV18), DeviceFamily.WHOOP5)!! + assertNull(worn["hr_fixed_8_8"]) // the 8.8 fixed-point reading is retired + assertEquals(141, worn["hr_quality_flags"]) // 0x8D + assertEquals(101, worn["heart_rate_alt"]) // hr@22 = 102 + assertEquals(0, (worn["hr_quality_flags"] as Int) and 0x10) // bit 4 is never set + assertEquals(0x80, (worn["hr_quality_flags"] as Int) and 0x80) // bit 7 = valid + + // Bit 7 is a VALIDITY bit: with it CLEAR (n=748 in the corpus) rr_count == 0 in 70.32% of records + // vs 19.82% with it set, and the @108/@109 sentinel fires in 69.65% vs 1.32%. Both second-device + // fixtures have it clear — and there @37 is NOT a heart rate (97 and 227 against hr@22 57 and 63), + // which the retired model would have surfaced as a 227 bpm "high-precision" reading. + for ((hex, expected) in listOf(secondDeviceHR57 to Triple(57, 11, 97), + secondDeviceHR63 to Triple(63, 2, 227))) { + val (hr, flags, alt) = expected + val p = decodeHistorical(bytes(hex), DeviceFamily.WHOOP5)!! + assertEquals(hr, p["heart_rate"]) + assertEquals(flags, p["hr_quality_flags"]) + assertEquals(0, (p["hr_quality_flags"] as Int) and 0x80) // bit 7 clear ⇒ invalid + assertEquals(alt, p["heart_rate_alt"]) // not a bpm when invalid + } + } + + @Test + fun v18OpticalBaselineIsTwoBytesNotAU16() { + // Mirror of Swift testHistoricalV18OpticalBaselineIsTwoBytesNotAU16. @106 was read as a u16 LE. It + // cannot be one: across 18,599 consecutive-second pairs the HIGH byte changed while the LOW byte + // stayed frozen in 3,514 (18.89%), and the corpus holds ZERO low-byte wrap events — a real u16 + // cannot step its high byte without a carry. The apparent u16 deltas are just 256·Δb107 + Δb106. + // The two bytes are independent u8 channels (corr +0.73, moving in OPPOSITE directions in 5.8% of + // pairs), structurally parallel to the @108/@109 pair. + val m = decodeHistorical(mutateAndReCrc(107, 0xFF), DeviceFamily.WHOOP5)!! + assertNull(m["optical_baseline_106"]) // the u16 reading is retired + // Moving @107 must leave the @106 channel alone — under the u16 model it was worth 256. + assertEquals(101, m["optical_baseline_a"]) + assertEquals(255, m["optical_baseline_b"]) + } + + @Test + fun v18OpticalAmpSentinelIsRecordLevel() { + // Mirror of Swift testHistoricalV18OpticalAmpSentinelIsRecordLevel. The 128 sentinel on @108/@109 + // was noted as invalidating a single CHANNEL. It is RECORD-level: over 18,650 records amp_a == 128 + // in 757 and amp_b == 128 in 757 — the SAME 757, never one without the other. All four real + // fixtures agree, so the two bytes are never split. + for (hex in listOf(wornV18, offWristV18, secondDeviceHR57, secondDeviceHR63)) { + val p = decodeHistorical(bytes(hex), DeviceFamily.WHOOP5)!! + assertEquals("the 128 sentinel must fire on both amp bytes or neither", + p["optical_amp_a"] == 128, p["optical_amp_b"] == 128) + } + } + @Test fun v18OpticalFieldsAreNotNamedPhysiologically() { // Guard the project rule: the paired optical channel must NOT be surfaced as SpO2/perfusion diff --git a/android/app/src/test/resources/decoder_oracle.json b/android/app/src/test/resources/decoder_oracle.json index 10049ae872..92da383d30 100644 --- a/android/app/src/test/resources/decoder_oracle.json +++ b/android/app/src/test/resources/decoder_oracle.json @@ -91,7 +91,10 @@ "synthetic": false, "hex": "aa01740001003fb12f1280733d8401b69f266a66460066025a0265020000000000007b0a8d656463ff0012163cf6a439bf2924fd3ed763fe3e3200aa000000000000000000f7000901f10b0007010c020c00000000000000000000000000000000000000000000000100656f1e1e0000009d61a7c00000003e862817", "expect": { - "optical_baseline_106": 28517, + "optical_baseline_a": 101, + "optical_baseline_b": 111, + "hr_quality_flags": 141, + "heart_rate_alt": 101, "optical_amp_a": 30, "optical_amp_b": 30, "hist_version": 18, @@ -104,7 +107,6 @@ "motion_wear_quality": 0, "activity_class": 0, "record_index": 25443699, - "hr_fixed_8_8": 25997, "step_cadence": 170, "status_word": 1792, "status_word_1": 3073, @@ -143,7 +145,10 @@ "synthetic": false, "hex": "aa01740001003fb12f12803a3d84018889266a3d0a00000000000000000000000000000000000000000064c33b52b47d3fe1ba1dbda470ecbd000064000000000000000000e500e200c708000c010c020c0000000000000000000000000000000000000000000000010000008080000000000000000000009ffafe6c", "expect": { - "optical_baseline_106": 0, + "optical_baseline_a": 0, + "optical_baseline_b": 0, + "hr_quality_flags": 0, + "heart_rate_alt": 0, "optical_amp_a": 128, "optical_amp_b": 128, "hist_version": 18, @@ -177,7 +182,10 @@ "synthetic": false, "hex": "aa01740001003fb12f128093c47c006dbc296a00600039000000000000000000006137020b610000e1e04c063d8fce36bf7b08233f8fea993e38a50000000000000000000019012101920b5002010c020c0100000000000000000000000000000000000000000005010085808080000000a5538ec000000016d0680d", "expect": { - "optical_baseline_106": 32901, + "optical_baseline_a": 133, + "optical_baseline_b": 128, + "hr_quality_flags": 11, + "heart_rate_alt": 97, "optical_amp_a": 128, "optical_amp_b": 128, "hist_version": 18, @@ -196,7 +204,10 @@ "synthetic": false, "hex": "aa01740001003fb12f128034d67c000ece296a0a77003f01fc0300000000000000203a0302e30000ff00f8093c1f8534bcc3a5473f4819243f85a6000000000000000000001b012601e80b5003010c020c00000000000000000000000000000000000000000000050100ced7241c0000006ead8cc00000008775b70c", "expect": { - "optical_baseline_106": 55246, + "optical_baseline_a": 206, + "optical_baseline_b": 215, + "hr_quality_flags": 2, + "heart_rate_alt": 227, "optical_amp_a": 36, "optical_amp_b": 28, "hist_version": 18, diff --git a/docs/BLE_REVERSE_ENGINEERING.md b/docs/BLE_REVERSE_ENGINEERING.md index 49107358fb..ddd6cea1cb 100644 --- a/docs/BLE_REVERSE_ENGINEERING.md +++ b/docs/BLE_REVERSE_ENGINEERING.md @@ -428,7 +428,9 @@ garbage (HR `0`, gravity overflow). The fields below were read off real frames a | 22 | `heart_rate` (u8) | **matched the 2A37-verified live HR exactly at all 96 overlapping timestamps** (mean \|Δ\| 0.00 bpm); note this is v24's `21`+1, **not** +4 | | 23 | `rr_count` (u8) | matches #valid R-R intervals 100 % (1141/1143) | | 24 + 2·i | `rr[i]` (u16, ms) | 60000/mean(R-R) ≈ HR for 88 % (rest are HR-averaging) | -| 36 | `hr_fixed_8_8` (u16 LE) — bpm = `value/256` | a **higher-precision heart rate**: `value/256` correlates **0.989** with the integer `heart_rate@22` over ~258k records and carries sub-bpm fractions `@22` can't (e.g. `25997` → 101.55 bpm vs `@22`=102). | +| 36 | `hr_quality_flags` (u8) | a **flag byte**, *not* the low half of a fixed-point HR. Over **18,650** real v18 records bit 4 is **never** set (0/18,650 — a genuine 8.8 fraction sets it ~50 % of the time, and it is the only bit never set), **95.02 %** of values land in `0x80`–`0x8F` (uniform would be 6.25 %) across just **40 distinct values**, and sd = **26.5** vs 73.9 for a uniform byte. **Bit 7 = validity**: with it clear (n=748) `rr_count == 0` in **70.32 %** of records vs **19.82 %** with it set, and the `@108/@109` sentinel fires in **69.65 %** vs 1.32 %. Remaining bits unpinned; carried raw. | +| 37 | `heart_rate_alt` (u8, bpm) | a **duplicate** of `heart_rate@22` — equal in **99.575 %** of records (18,523/18,602), differing only by −6…+2, and it tracks HR only while `@36` bit 7 is set (99.74 % exact vs 94.12 % when clear). | +| ~~36–37~~ | ~~`hr_fixed_8_8` (u16 LE) — bpm = `value/256`~~ | **Retired.** The "corr 0.989 with `heart_rate@22`" that justified this name was **circular**: the u16 is literally `hr@22` (at `@37`) plus the `@36` flag byte over 256, so the residual is a flat **+0.504 ± 0.189** — i.e. `@36/256`, not a sub-bpm fraction. On records where `@36` bit 7 is clear it produced absurd readings (a fixture decodes to **227 bpm**). | | 33 / 38 / 40 | raw bytes near the HR / R-R fields | carried **raw** (meaning not pinned from observation): `@33` a flag-ish byte, `@38` a u16 beside the R-R fields, `@40` a status-like byte. `whoop-local` names `@40` **`signal_quality`** (#715) — plausible for a byte sitting beside HR, but it appears only in that project's code with no stated source and no supporting analysis, so it stays raw here. | | 41 | `dynamic_acceleration` (f32, g) | the strap's own **gravity-removed motion magnitude**, one scalar per second sitting immediately before the gravity triplet. Gated to `[0, 8] g` so a wrong offset stores nothing rather than garbage; reads 0.006–0.033 g across the resting oracle frames. Decoded on both platforms but **not persisted and not scored** — `step_motion_counter@57` and `activity_class@63` are what the motion paths actually consume. See the byte-43 note below. | | 45 / 49 / 53 | `gravity_x/y/z` (f32, g) | \|g\| ≈ 1.0 for 100 % of 500 records; v18 has **one** triplet (not v24's two) | @@ -445,6 +447,8 @@ garbage (HR `0`, gravity overflow). The fields below were read off real frames a | 82 | `aux_byte_82` (u8) | the raw carry of the byte decoded as **`spo2_candidate_82`** — a strap-computed SpO₂ % scalar, tri-mode, sleep-only (#103). Instrumentation only, never a shipped metric; see the note below and [`WHOOP5_DEEP_DATA.md`](WHOOP5_DEEP_DATA.md). | | 83–103 | reserved | observed **constant `0`** on two straps (zero-filled). | | 104 | (const) | observed **constant `1`** on two straps; carried raw, no metric. | +| 106 / 107 | `optical_baseline_a` / `optical_baseline_b` (u8, u8) | two **independent u8** optical/ADC baseline channels — **not** one u16 LE. A u16 is structurally impossible here: across 18,599 consecutive-second pairs the **high byte changed while the low byte stayed frozen in 3,514 (18.89 %)**, and the corpus holds **zero** low-byte wrap events — a real u16 cannot step its high byte without a carry. The apparent u16 deltas are exactly `256·Δ@107 + Δ@106` (clustering at 0, ±1, ±255, ±256, ±257, ±513). Correlated but independent (corr **+0.73**; they move in **opposite** directions in 5.8 % of pairs where both move). **`0` — not `128` — marks off-wrist**: both bytes read 0 in exactly the 8 records that also carry `HR == 0`, while 128 occurs unremarkably while worn (`@106` in 10 records, `@107` in 103). Magnitudes are device-specific (102–255 / 119–247 on one strap vs 20–66 / 34–81 on another), so **no scale is asserted**. | +| 108 / 109 | `optical_amp_a` / `optical_amp_b` (u8, u8) | a tightly-coupled **pair** (equal in 23.5 % of records, within ±2 in ~80 %). **`128` is a RECORD-level sentinel**, not per-channel: `amp_a == 128` in 757 records and `amp_b == 128` in 757 — the **same** 757, never one without the other. They do **not** rise with heart rate; that reading (~34 at HR 40–49 → ~58 at 80–89) was an **averaging artifact** of counting the 128 sentinel as a number — with sentinels excluded the trend is flat-to-declining (**32.45 → 29.52**). The real monotone trend is with **motion**: ~32.7 while still (`dyn_acc` < 0.02 g) → **37.4** at 0.05–0.2 g. The sentinel is a usable per-second **signal-quality** flag: it fires on 4.02 % of worn seconds and predicts the band's own beat-detection failure (`rr_count == 0`) at **79.44 % vs 19.40 %** — a **4.09×** lift that **survives holding motion constant** (4.11× within `dyn_acc` < 0.009 g), where shuffled and circular-shift nulls all sit at ~1.0×. **Not** SpO₂, blood pressure or a perfusion substrate — signal-quality/AGC is the supported reading and the wavelength identity is unknown. | | 113 | `unknown_f32_113` (f32 LE) | a float32 (observed range ~ −5.3…0, `0` = unset); **purpose unknown**, carried raw. | The strongest check on the HR offset: where a historical record and a live `REALTIME_DATA` (§5, 2A37 diff --git a/tools/linux-capture/test_whoop_activity.py b/tools/linux-capture/test_whoop_activity.py index 09a939dc23..a531a22d27 100644 --- a/tools/linux-capture/test_whoop_activity.py +++ b/tools/linux-capture/test_whoop_activity.py @@ -6,7 +6,7 @@ def make_v18(unix=1000, hr=60, motion=0, wear=0, sleep_state=0, - hr_fixed=0, rr_packed=0, cardiac_flags=0, cardiac_status=0, + hr_flags=0, hr_alt=0, rr_packed=0, cardiac_flags=0, cardiac_status=0, record_index=0, step_cadence=0, status_word=0, aux_f32=0.0, temp_aux_1=0, temp_aux_2=0, status_word_1=0, status_word_2=0, onwrist=0, wake_quality=0, aux_byte_82=0, @@ -18,7 +18,8 @@ def make_v18(unix=1000, hr=60, motion=0, wear=0, sleep_state=0, struct.pack_into("