diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager+Trace.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager+Trace.swift index 3d8105a3d9..fff25e0633 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager+Trace.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager+Trace.swift @@ -20,16 +20,18 @@ extension SleepStager { /// The funnel line below only exists when the spine is CALLED, so its absence was ambiguous in /// exactly the way that made the first field report unreadable: it could mean the gate never /// opened, or that the sink was never wired. This says which, on any day with no motion — the - /// only days where the question arises. + /// only days where the question arises. `stored` counts the owner's stored nights that end on the + /// day (W03-052); before that fix it counted every stored night in the read, so logs from either + /// side of it do not compare on this field. public static func hrOnlyGateLine(attempted: Bool, reason: String, gravRows: Int, storedNights: Int) -> String { "[sleep] hr-only gate attempted=\(attempted) reason=\(reason) " + "grav=\(gravRows) stored=\(storedNights)" } - /// The HR-only spine's own funnel line (#1801). Byte-identical twin of the Kotlin - /// `SleepStagerTrace.hrOnlyLine`, so an Android and an Apple log of the same night compare - /// directly — which is the entire reason these formatters are twinned at all. + /// The HR-only spine's own funnel line (#1801). It began as the byte-identical twin of the Kotlin + /// `SleepStagerTrace.hrOnlyLine`; since W03-043 its anchor and band print as `lo..hi`, which the + /// Kotlin line never did, so the two no longer compare field for field. /// /// The path shipped SILENT on both platforms, and a field log then showed `reason=no-motion` /// with no way to tell whether the spine had run and found nothing or never ran. Every number @@ -37,8 +39,11 @@ extension SleepStager { /// from a duration gate eating real runs (`sleepRuns` high, `longestMin` under `minSleepMin`). /// /// `anchorBpm` and `bandBpm` are printed because they are DERIVED, not configured: the anchor is - /// a percentile of THIS window, so the same code gives every wearer a different threshold. - public static func hrOnlyLine(anchorBpm: Double?, bandBpm: Double?, + /// a percentile of the heart rate around each epoch, so the same code gives every wearer a different + /// threshold. Each prints as the lowest and highest value any epoch of the read was held to + /// (`lo..hi`), since there is no single anchor to print (W03-043). `hrP50` and `hrP90` are the whole + /// read's spread. + public static func hrOnlyLine(anchorBpm: ClosedRange?, bandBpm: ClosedRange?, hrP50: Double?, hrP90: Double?, epochs: Int, runs: Int, mergedRuns: Int, sleepRuns: Int, longestSleepMin: Int, staged: Int, kept: Int, @@ -63,6 +68,12 @@ extension SleepStager { return "\(tenths / 10).\(abs(tenths % 10))" } + /// `round1` at both ends, as `lo..hi`. + private static func round1(_ r: ClosedRange?) -> String { + guard let r else { return "nil" } + return "\(round1(r.lowerBound))..\(round1(r.upperBound))" + } + /// One verdict line for a candidate run. `gate` names the constant that decided it /// (minSleepMin, maxMainSleepSpanS, offWrist, daytimeGuard, morningStillness, hrConfirm, /// sparseBridge, accepted); `detail` carries that gate's numbers. `startTs`/`endTs` give the diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift index bf0e8be759..3e7a971bab 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift @@ -598,7 +598,10 @@ public enum SleepStager { return periods } - /// Percentile of the window's bpm that anchors the HR-only sleep band. + /// Percentile of the bpm around an epoch that anchors the HR-only sleep band there: the heart rate of the + /// 24 h around the epoch, the span a decision on the gravity path reads (W03-043). At the data's ends + /// that window slides inward rather than being cut, so it can reach up to 24 h to one side; the gravity + /// path's `LocalContext.baseline` cuts instead (see `hrOnlyLocalAnchors` for why the two differ). /// /// NOT `hrBaseline`. That is the window MEDIAN, and it is the right anchor where it is used — as a /// CONFIRMATION gate on a run gravity stillness already found, where being permissive is deliberate. @@ -622,16 +625,82 @@ public enum SleepStager { /// screen; under-detection leaves "No data", which is the state this feature is trying to improve on /// and therefore a safe place to fail. p10 x 1.05 measured 8.1 h against a truth of 8 on a /// field-shaped 24 h window, and under-reads a long multi-night window rather than over-reading it - /// (8.7 h of 16 h across two nights in 54 h) — pinned in `SleepStagerHrOnlyAnchorTests`. + /// (8.7 h of 16 h across two nights in 54 h) — pinned in `SleepStagerHrOnlyAnchorTests`. Since + /// W03-043 the anchor is taken over the 24 h around each epoch, the window that sweep measured on, so + /// the multi-night figure describes one anchor held across a whole read, which nothing does any more. public static let hrOnlyBandMult: Double = 1.05 /// The `hrOnlyAnchorPercentile` of `hr` by bpm, or nil when empty. Nearest-rank (no interpolation), so /// the value is always one the wearer actually recorded and the two platforms cannot disagree on a - /// rounding rule. + /// rounding rule. `hrOnlyLocalAnchors` applies this rule to the samples around each epoch. static func hrOnlyBaseline(_ hr: [HRSample]) -> Double? { hrPercentile(hr, hrOnlyAnchorPercentile) } + /// For each of `times`, `hrOnlyBaseline` of the samples of `hr` in the 24 h window around it, or nil when + /// none lies there (W03-043). One anchor for the whole read let the heart rate of the next night, which + /// the engine's 66 h read (W03-024) holds, move the band a night was judged against; through this an + /// epoch's anchor depends only on the 24 h around it. + /// + /// The window is [t − `localBaselineRadiusS`, t + `localBaselineRadiusS`], slid inward to stay inside + /// [first sample, last sample] where it would reach past either, and the whole of `hr` when that spans + /// less than 24 h. Sliding rather than cutting keeps every anchor on 24 h of heart rate, the span + /// `hrOnlyBandMult` was measured on. A window cut at the data's end would hold this morning's night, + /// read up to now, to heart rate that is mostly the night itself, and its anchor would rise into the + /// night and split it. A past day's read reaches 12 h past the day once that time has passed, so its + /// nights get the centred window and do not depend on the read's reach. Until then (yesterday's read, + /// before local noon) a night ending late in the day can sit on a slid window; the per-day cache keys on + /// the heart rate read, so the next pass after more data lands scores it again. + /// + /// `hr` and `times` must be sorted ascending. The window slides once over `hr` (both of its ends only + /// move forward as `t` grows), and each anchor is read from a histogram of the window's bpm, so a read + /// costs one pass plus a histogram scan per epoch rather than a sort per epoch. A bpm outside + /// 0 ... `LocalContext.maxHistogramBpm` is kept in a sorted side list instead, so a corrupt value + /// neither sizes the histogram nor changes the rank (W03-051's rule). + static func hrOnlyLocalAnchors(_ hr: [HRSample], at times: [Int]) -> [Double?] { + guard let first = hr.first?.ts, let last = hr.last?.ts else { return times.map { _ in nil } } + let r = localBaselineRadiusS + let maxBpm = LocalContext.maxHistogramBpm + var counts = [Int](repeating: 0, count: maxBpm + 1) + var outside: [Int] = [] + var n = 0 + func insertionIndex(_ b: Int) -> Int { + var l = 0, h = outside.count + while l < h { let m = (l + h) / 2; if outside[m] < b { l = m + 1 } else { h = m } } + return l + } + func add(_ b: Int) { + if b >= 0 && b <= maxBpm { counts[b] += 1 } else { outside.insert(b, at: insertionIndex(b)) } + n += 1 + } + func remove(_ b: Int) { + if b >= 0 && b <= maxBpm { counts[b] -= 1 } else { outside.remove(at: insertionIndex(b)) } + n -= 1 + } + func kthSmallest(_ k: Int) -> Int { + let negatives = outside.prefix { $0 < 0 }.count + if k < negatives { return outside[k] } + var seen = negatives + for b in 0...maxBpm where counts[b] > 0 { + seen += counts[b] + if k < seen { return b } + } + return outside[negatives + (k - seen)] + } + var out: [Double?] = [] + out.reserveCapacity(times.count) + var lo = 0, hi = 0 + for t in times { + let start = min(max(t - r, first), max(first, last - 2 * r)) + while hi < hr.count && hr[hi].ts <= start + 2 * r { add(hr[hi].bpm); hi += 1 } + while lo < hi && hr[lo].ts < start { remove(hr[lo].bpm); lo += 1 } + if n == 0 { out.append(nil); continue } + let k = min(max(Int(Double(n - 1) * hrOnlyAnchorPercentile), 0), n - 1) + out.append(Double(kthSmallest(k))) + } + return out + } + /// The `p` percentile of `hr` by bpm, nearest-rank. Shared with `hrOnlyBaseline` so the spread the /// trace reports is measured by the SAME rule as the anchor it is meant to be judged against. static func hrPercentile(_ hr: [HRSample], _ p: Double) -> Double? { @@ -694,10 +763,24 @@ public enum SleepStager { /// /// Bucket order does not depend on sort stability: samples are grouped by epoch and reduced with a /// median, so their order within an epoch cannot change the result. + /// + /// This form holds every epoch to one `baseline`; `hrOnlySessions` passes each epoch its own anchor + /// through the form below (W03-043). static func hrOnlySleepRuns(_ hr: [HRSample], baseline: Double?, epochS: Int = hrOnlyEpochS, maxGapMinutes: Int = maxGapMin) -> [Period] { guard let baseline = baseline, baseline > 0 else { return [] } + return hrOnlySleepRuns(hr, epochS: epochS, maxGapMinutes: maxGapMinutes, + anchors: { $0.map { _ in baseline } }) + } + + /// The spine with an anchor per epoch: `anchors` receives the epoch start times, ascending, and returns + /// the anchor each is held to, index for index. An epoch with no anchor, or a non-positive one, is not + /// in the band. + static func hrOnlySleepRuns(_ hr: [HRSample], + epochS: Int = hrOnlyEpochS, + maxGapMinutes: Int = maxGapMin, + anchors: ([Int]) -> [Double?]) -> [Period] { if hr.isEmpty || epochS <= 0 { return [] } var byEpoch: [Int: [Double]] = [:] var lastTs: [Int: Int] = [:] @@ -714,7 +797,12 @@ public enum SleepStager { // understatement would then be weighed against the caller's minimum-duration gate. let times = keys.map { $0 * epochS } let ends = keys.map { lastTs[$0]! } - let flags = keys.map { HRVAnalyzer.median(byEpoch[$0]!) <= baseline * hrOnlyBandMult } + let anchorAt = anchors(times) + precondition(anchorAt.count == keys.count, "one anchor per epoch") + let flags = keys.indices.map { i -> Bool in + guard let anchor = anchorAt[i], anchor > 0 else { return false } + return HRVAnalyzer.median(byEpoch[keys[i]]!) <= anchor * hrOnlyBandMult + } let maxGapS = maxGapMinutes * 60 var periods: [Period] = [] var runStart = 0 @@ -752,7 +840,8 @@ public enum SleepStager { /// were always V2, under a trace that named the chosen recipe. V1 stages no night without motion, so it /// maps to V2 (`hrOnlyRecipe`). The default keeps pure-function callers on V2. /// - /// The anchor is `hrOnlyBaseline` — the `hrOnlyAnchorPercentile` of the window — and NOT the + /// The anchor is `hrOnlyBaseline`'s rule — the `hrOnlyAnchorPercentile` of the heart rate of the + /// 24 h around each epoch, slid inward at the data's ends (`hrOnlyLocalAnchors`, W03-043) — and NOT the /// `hrBaseline` median the motion path derives. Reusing that median looked like parity and was a bug: /// as a confirmation gate on an already-detected run it is deliberately permissive, but as a primary /// threshold it admits over half of any window by definition. See `hrOnlyAnchorPercentile`. @@ -773,9 +862,9 @@ public enum SleepStager { minMinutes: Int = minSleepMin, traceSink: ((String) -> Void)? = nil) -> [SleepSession] { let hrS = hr.sorted { $0.ts < $1.ts } - // ONE sort of the bpm axis, reused for the anchor and for the spread the trace reports. + // ONE sort of the bpm axis, for the spread the trace reports. let sortedBpm = hrS.map { Double($0.bpm) }.sorted() - guard let baseline = percentileOfSorted(sortedBpm, hrOnlyAnchorPercentile) else { + guard !hrS.isEmpty else { traceSink?(GateTrace.hrOnlyLine(anchorBpm: nil, bandBpm: nil, hrP50: nil, hrP90: nil, epochs: 0, runs: 0, mergedRuns: 0, sleepRuns: 0, longestSleepMin: 0, @@ -784,12 +873,19 @@ public enum SleepStager { } let rrS = rr.sorted { $0.ts < $1.ts } var out: [SleepSession] = [] + // Each epoch is held to the anchor of the heart rate around it (W03-043); the trace prints their range. + var anchors: [Double?] = [] + let rawRuns = hrOnlySleepRuns(hrS) { epochStarts in + anchors = hrOnlyLocalAnchors(hrS, at: epochStarts) + return anchors + } + let held = anchors.compactMap { $0 } + let anchorRange = held.min().flatMap { lo in held.max().map { lo...$0 } } // mergePeriods for the same reason the motion path calls it: a run boundary is a threshold // crossing, and a sleeping heart rate oscillates across the band all night. Without this the // spine returns the night's minutes correctly but shredded into sub-mergeMin fragments, every one // of which then fails the minimum-duration gate below — 8 h of detected sleep yielding zero // sessions. Absorbing the short runs first is what turns a spine into a night. - let rawRuns = hrOnlySleepRuns(hrS, baseline: baseline) let merged = mergePeriods(rawRuns) var staged = 0 var longestSleepS = 0 @@ -815,8 +911,8 @@ public enum SleepStager { hrOnly: true)) } traceSink?(GateTrace.hrOnlyLine( - anchorBpm: baseline, - bandBpm: baseline * hrOnlyBandMult, + anchorBpm: anchorRange, + bandBpm: anchorRange.map { ($0.lowerBound * hrOnlyBandMult)...($0.upperBound * hrOnlyBandMult) }, // The wearer's own spread. An anchor alone cannot be judged: p10 of 60 means one thing when // the median is 63 and quite another when it is 74, and only the second leaves a night the // band can separate. diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepHrOnlyWindowInvarianceTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepHrOnlyWindowInvarianceTests.swift new file mode 100644 index 0000000000..cea6a24f2c --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepHrOnlyWindowInvarianceTests.swift @@ -0,0 +1,113 @@ +import XCTest +@testable import StrandAnalytics +import WhoopProtocol + +/// W03-043: a night found from heart rate alone (a strap that banks no motion, #1801) must depend only on +/// the heart rate around it, not on how far the read reaches. +/// +/// The HR-only spine held every epoch to one anchor, the tenth percentile of the whole read. Since W03-024 +/// the engine reads each day over 66 h, three nights' worth, so the next night's heart rate, 16 h after the +/// night being scored, moved the band that night was judged against: the gravity path's W03-023, on the +/// path W03-023 did not touch. +/// +/// The fixture is the engine's read for the day the night ends on (tz 0): from 30 h before its midnight to +/// 36 h after, heart rate every 30 s, three nights at 23:00 to 07:00 around 56 bpm and the day around 74. +/// Two reads hold the same data except for the third night, which starts 16 h after the scored night ends. +final class SleepHrOnlyWindowInvarianceTests: XCTestCase { + /// Local midnight (tz 0) the scored night runs across. + private let midnight = 1_700_006_400 + + /// Heart rate every 30 s over [from, to]; `nextNightBpm` is the centre of the night after the scored one. + private func heartRate(from: Int, to: Int, nextNightBpm: Int) -> [HRSample] { + var out: [HRSample] = [] + for ts in stride(from: from, through: to, by: 30) { + let wave = sin(Double(ts) / 500.0) + let bpm: Int + switch ts - midnight { + case (-25 * 3600)..<(-17 * 3600): bpm = 56 + Int(wave * 2) // the night before + case (-1 * 3600)..<(7 * 3600): bpm = 56 + Int(wave * 2) // the scored night + case (23 * 3600)..<(31 * 3600): bpm = nextNightBpm + Int(wave * 2) // the next night + default: bpm = 74 + Int(wave * 11) // awake + } + out.append(HRSample(ts: ts, bpm: bpm)) + } + return out + } + + /// The sessions the engine keeps for the scored day: those ending on it. + private func scoredNight(_ hr: [HRSample]) -> [[Int]] { + SleepStager.hrOnlySessions(hr: hr, rr: [], resp: []) + .filter { $0.end >= midnight && $0.end < midnight + 86_400 } + .map { [$0.start, $0.end] } + } + + func testTheNextNightsHeartRateDoesNotMoveTheScoredNight() { + let usual = scoredNight(heartRate(from: midnight - 30 * 3600, to: midnight + 36 * 3600, nextNightBpm: 56)) + let lower = scoredNight(heartRate(from: midnight - 30 * 3600, to: midnight + 36 * 3600, nextNightBpm: 46)) + XCTAssertFalse(usual.isEmpty, "the fixture must detect the scored night") + XCTAssertEqual(lower, usual) + } + + /// The reach of the read past the night does not move it either: a read ending 12 h after the night + /// covers every epoch's neighbourhood, so it agrees with the engine's full read. + func testAReadEndingTwelveHoursAfterTheNightAgreesWithTheFullRead() { + let full = scoredNight(heartRate(from: midnight - 30 * 3600, to: midnight + 36 * 3600, nextNightBpm: 46)) + let short = scoredNight(heartRate(from: midnight - 30 * 3600, to: midnight + 19 * 3600, nextNightBpm: 46)) + XCTAssertFalse(full.isEmpty, "the fixture must detect the scored night") + XCTAssertEqual(short, full) + } + + /// The sliding anchor is `hrOnlyBaseline` of the samples in each time's 24 h window, slid inside the data, + /// by the brute-force rule, on uneven data with gaps, repeated timestamps and bpm outside the histogram. + func testLocalAnchorIsTheTenthPercentileOfTheTwentyFourHoursAroundEachTime() { + var state: UInt64 = 0x9E37_79B9_7F4A_7C15 + func next(_ bound: Int) -> Int { + state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 + return Int((state >> 33) % UInt64(bound)) + } + var hr: [HRSample] = [] + var ts = 1_000_000 + for _ in 0..<6_000 { + ts += [0, 1, 30, 60, 600, 7_200][next(6)] + let bpm: Int + switch next(100) { + case 0: bpm = -3 + case 1: bpm = 700 + case 2: bpm = 65_000 + default: bpm = 40 + next(80) + } + hr.append(HRSample(ts: ts, bpm: bpm)) + } + let times = Array(stride(from: hr[0].ts - 13 * 3600, through: ts + 13 * 3600, by: 1_777)) + let anchors = SleepStager.hrOnlyLocalAnchors(hr, at: times) + let r = SleepStager.localBaselineRadiusS + let first = hr[0].ts, last = ts + XCTAssertGreaterThan(last - first, 4 * 2 * r, "the data must span several windows") + for (t, anchor) in zip(times, anchors) { + let start = min(max(t - r, first), last - 2 * r) + let window = hr.filter { $0.ts >= start && $0.ts <= start + 2 * r } + XCTAssertEqual(window.last!.ts - window.first!.ts <= 2 * r, true) + XCTAssertEqual(anchor, SleepStager.hrOnlyBaseline(window), "t=\(t)") + } + } + + /// Near either end of the data the window slides inward and stays 24 h long; data shorter than 24 h is + /// one window. + func testTheWindowSlidesInsideTheDataAtEitherEnd() { + let r = SleepStager.localBaselineRadiusS + // 48 h of one sample an hour: bpm 100 for the first 24 h, then 50. + let hr = (0..<48).map { HRSample(ts: $0 * 3600, bpm: $0 < 24 ? 100 : 50) } + let anchors = SleepStager.hrOnlyLocalAnchors(hr, at: [0, 6 * 3600, 47 * 3600, 60 * 3600]) + // At 0 and 6 h the window is [0, 24 h]: 24 samples of 100 and one of 50, so p10 is 100. + XCTAssertEqual(anchors[0], 100) + XCTAssertEqual(anchors[1], 100) + // At 47 h and past the end the window is [23 h, 47 h]: one 100 and 24 of 50. + XCTAssertEqual(anchors[2], 50) + XCTAssertEqual(anchors[3], 50) + XCTAssertEqual(2 * r, 24 * 3600) + let short = Array(hr.prefix(10)) + XCTAssertEqual(SleepStager.hrOnlyLocalAnchors(short, at: [0, 9 * 3600]), + [SleepStager.hrOnlyBaseline(short), SleepStager.hrOnlyBaseline(short)]) + XCTAssertEqual(SleepStager.hrOnlyLocalAnchors([], at: [0]), [nil]) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyTraceTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyTraceTests.swift index a219d301b8..4f4bf1ff47 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyTraceTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyTraceTests.swift @@ -8,10 +8,11 @@ final class SleepStagerHrOnlyTraceTests: XCTestCase { func testLineNamesTheDerivedThresholdAndLongestCandidate() { XCTAssertEqual( - SleepStager.GateTrace.hrOnlyLine(anchorBpm: 61.0, bandBpm: 64.05, hrP50: 74.0, hrP90: 88.0, epochs: 3021, runs: 48, + SleepStager.GateTrace.hrOnlyLine(anchorBpm: 58.0...61.0, bandBpm: 60.9...64.05, hrP50: 74.0, hrP90: 88.0, + epochs: 3021, runs: 48, mergedRuns: 12, sleepRuns: 7, longestSleepMin: 41, staged: 0, kept: 0, minSleepMin: 60), - "[sleep] hr-only spine anchorBpm=61.0 bandBpm=64.1 hrP50=74.0 hrP90=88.0 " + "[sleep] hr-only spine anchorBpm=58.0..61.0 bandBpm=60.9..64.1 hrP50=74.0 hrP90=88.0 " + "epochs=3021 runs=48 merged=12 " + "sleepRuns=7 longestMin=41 staged=0 kept=0 minSleepMin=60" ) @@ -48,10 +49,10 @@ final class SleepStagerHrOnlyTraceTests: XCTestCase { (71.4, "71.4"), (1.05, "1.1"), (0.0, "0.0"), (120.0, "120.0"), ] for (v, expected) in cases { - let line = SleepStager.GateTrace.hrOnlyLine(anchorBpm: v, bandBpm: nil, hrP50: nil, hrP90: nil, epochs: 0, runs: 0, - mergedRuns: 0, sleepRuns: 0, longestSleepMin: 0, + let line = SleepStager.GateTrace.hrOnlyLine(anchorBpm: v...v, bandBpm: nil, hrP50: nil, hrP90: nil, epochs: 0, + runs: 0, mergedRuns: 0, sleepRuns: 0, longestSleepMin: 0, staged: 0, kept: 0, minSleepMin: 60) - XCTAssertTrue(line.contains("anchorBpm=\(expected) "), "\(v) -> expected \(expected), got: \(line)") + XCTAssertTrue(line.contains("anchorBpm=\(expected)..\(expected) "), "\(v) -> expected \(expected), got: \(line)") } } } diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index 8983c9835e..66b4b124ed 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -431,6 +431,31 @@ final class IntelligenceEngine: ObservableObject { return min(dayStart + StreamReadCap.forwardSeconds, now) } + /// Where a day with no motion takes its night from (#804, #1801). The raw values are the gate line's + /// `reason=`, so the log and the branch cannot name two different outcomes. + enum NoMotionNight: String { + /// The owner's own persisted hypnogram (a ring's night, #804), handed to `analyzeDay`. + case storedHypnogram = "stored-hypnogram" + /// A stored night under the WHOOP import namespace: the day is left alone. + case storedSessionsExist = "stored-sessions-exist" + /// Nothing stored: the night is staged from heart rate (#1801). + case hrOnly = "no-motion-no-hypnogram" + } + + /// The `NoMotionNight` for `day`, from the owner's stored sessions in the day's read, and how many of + /// them decided it. + /// + /// Only a stored session that ends on `day` counts (W03-052): that is the rule `analyzeDay` keeps a + /// night by. The read runs from 30 h before the day to 12 h past it (W03-024), so it also holds the + /// nights of the day before and the day after; counting those skipped the HR-only night of a day that + /// had none stored, and the day scored no night at all. + nonisolated static func noMotionNight(owner: String, stored: [SleepSession], day: String, + tzOffsetSeconds: Int) -> (source: NoMotionNight, storedNights: Int) { + let onDay = stored.filter { AnalyticsEngine.dayString($0.end, offsetSec: tzOffsetSeconds) == day }.count + if onDay == 0 { return (.hrOnly, 0) } + return (owner != Repository.whoopSource ? .storedHypnogram : .storedSessionsExist, onDay) + } + /// Counts + a window length only — same privacy class as the sibling `sleep day=` line, no PII. Pure so /// it's unit-tested directly; byte-identical to the Android `sleepDetectNoNightLogLine`. /// Fold the per-day diagnostic lines onto the one channel that already replays them. @@ -1406,31 +1431,28 @@ final class IntelligenceEngine: ObservableObject { // skipped before any heart rate was looked at. // // The stored lookup now runs for every no-motion day, so "nothing else knows about this - // night" is checked rather than assumed. A day that HAS stored sessions is left alone - // whoever owns it. + // night" is checked rather than assumed. A day with a stored night of its own (one ending + // on the day, W03-052) is left alone whoever owns it. let providedSleep: [SleepSession] if grav.count < 2 { let persisted = (try? await store.sleepSessions(deviceId: owner, from: from, to: to, limit: 4000)) ?? [] let stored = persisted.compactMap { AnalyticsEngine.sleepSession(fromProvided: $0) } - if owner != Repository.whoopSource, !stored.isEmpty { - traceSink?(SleepStager.GateTrace.hrOnlyGateLine( - attempted: false, reason: "stored-hypnogram", - gravRows: grav.count, storedNights: stored.count)) + let gate = Self.noMotionNight(owner: owner, stored: stored, day: day, + tzOffsetSeconds: tzOffset) + traceSink?(SleepStager.GateTrace.hrOnlyGateLine( + attempted: gate.source == .hrOnly, reason: gate.source.rawValue, + gravRows: grav.count, storedNights: gate.storedNights)) + switch gate.source { + case .storedHypnogram: providedSleep = stored - } else if !stored.isEmpty { - traceSink?(SleepStager.GateTrace.hrOnlyGateLine( - attempted: false, reason: "stored-sessions-exist", - gravRows: grav.count, storedNights: stored.count)) + case .storedSessionsExist: providedSleep = [] - } else { + case .hrOnly: // Reachable for ANY owner now, which widens this past the 5/MG it was built for: // a WHOOP 4.0 day that banked nothing at all also lands here, where the owner // check previously blocked it. A normal 4.0 day is untouched — it streams // gravity, so it never reaches this gate. - traceSink?(SleepStager.GateTrace.hrOnlyGateLine( - attempted: true, reason: "no-motion-no-hypnogram", - gravRows: grav.count, storedNights: 0)) providedSleep = SleepStager.hrOnlySessions(hr: hr, rr: rr, resp: resp, stager: sleepStager, traceSink: traceSink) diff --git a/StrandTests/NoMotionNightGateTests.swift b/StrandTests/NoMotionNightGateTests.swift new file mode 100644 index 0000000000..cac7ab9b9a --- /dev/null +++ b/StrandTests/NoMotionNightGateTests.swift @@ -0,0 +1,58 @@ +import XCTest +import StrandAnalytics +@testable import Strand + +/// W03-052: a day with no motion takes its night from the owner's stored sessions only when one of them is +/// that day's night. +/// +/// The gate read the owner's stored sessions over the whole sleep read and skipped the HR-only night +/// (#1801) whenever any were there. `analyzeDay` keeps only a session that ENDS on the day, and the read +/// runs from 30 h before the day to 12 h past it (W03-024), so a stored night of the day before or the day +/// after blocked a day that had none of its own: it scored no night at all. +final class NoMotionNightGateTests: XCTestCase { + /// Local midnight (tz 0) that starts `day`. + private let midnight = 1_700_006_400 + private var day: String { AnalyticsEngine.dayString(midnight, offsetSec: 0) } + + private func night(_ start: Int, _ end: Int) -> SleepSession { + SleepSession(start: start, end: end, efficiency: 0.9, + stages: [StageSegment(start: start, end: end, stage: "light")], restingHR: nil, avgHRV: nil) + } + + private func gate(_ owner: String, _ stored: [SleepSession], tz: Int = 0) + -> (source: IntelligenceEngine.NoMotionNight, storedNights: Int) { + IntelligenceEngine.noMotionNight(owner: owner, stored: stored, day: day, tzOffsetSeconds: tz) + } + + /// The next night starts after midnight, inside the day's read, and ends on the day after. + func testTheNextNightDoesNotBlockTheDaysHrOnlyNight() { + let next = night(midnight + 86_400 + 3600, midnight + 86_400 + 8 * 3600) + XCTAssertEqual(gate(Repository.whoopSource, [next]).source, .hrOnly) + XCTAssertEqual(gate("oura-ring", [next]).source, .hrOnly) + } + + /// The previous night ended on the day before, inside the read's 30 h lookback. + func testThePreviousNightDoesNotBlockTheDaysHrOnlyNight() { + let previous = night(midnight - 86_400 - 3600, midnight - 86_400 + 7 * 3600) + XCTAssertEqual(gate(Repository.whoopSource, [previous]).source, .hrOnly) + } + + /// A stored night ending on the day decides it, as before, and only the day's nights are counted. + func testAStoredNightEndingOnTheDayDecidesIt() { + let own = night(midnight - 3600, midnight + 7 * 3600) + let next = night(midnight + 86_400 + 3600, midnight + 86_400 + 8 * 3600) + let imported = gate(Repository.whoopSource, [own, next]) + XCTAssertEqual(imported.source, .storedSessionsExist) + XCTAssertEqual(imported.storedNights, 1) + XCTAssertEqual(gate("oura-ring", [own, next]).source, .storedHypnogram) + XCTAssertEqual(gate(Repository.whoopSource, []).source, .hrOnly) + } + + /// Day membership is `analyzeDay`'s: the session's end in the day under the pass's offset. + func testDayMembershipFollowsTheOffset() { + // Ends 23:00 UTC the day before; at UTC+2 that is 01:00 on the day. + let late = night(midnight - 8 * 3600, midnight - 3600) + XCTAssertEqual(gate(Repository.whoopSource, [late], tz: 0).source, .hrOnly) + XCTAssertEqual(gate(Repository.whoopSource, [late], tz: 2 * 3600).source, .storedSessionsExist) + } +} diff --git a/docs/review/workstreams/W03-analytics.md b/docs/review/workstreams/W03-analytics.md index 75a03c61ac..2944dac46f 100644 --- a/docs/review/workstreams/W03-analytics.md +++ b/docs/review/workstreams/W03-analytics.md @@ -109,7 +109,7 @@ every changed row explained; SleepBench / SleepPSG for any stager change. | W03-040 | S4 | Reported | `RhythmScreener.variationRecurred` is documented as varied windows spread over a sustained span, but the span constant has no reader: three adjacent 5-minute windows set it, and the shared rhythm CSV prints it | `RhythmScreener.swift` `nightMinSpanSeconds`, `summarizeNight` | Phase 2 workflow `wf_b77f01a5-1aa` (W3c-6); P5 confirmed (constant has no reader; `WindowResult` carries no time). V0 plan: `RhythmScreenerTests` clustered windows must not recur | | | W03-041 | S4 | Reported | `WorkoutTypeClassifier`, `WorkoutTypeFeatureExtractor` and `HeuristicWorkoutTypeClassifier` (399 lines) have no caller outside their tests | `WorkoutTypeClassifier.swift` | Phase 2 workflow `wf_b77f01a5-1aa` (W3c-7); P5 confirmed by grep. Decide: delete, or wire as default-off instrumentation | | | W03-042 | S4 | Reported | `DayResult.effortConfidence` is computed on every `analyzeDay` and read by nothing; it is also computed from the night's HR count while strain uses the day's HR | `AnalyticsEngine.swift` `effortConfidence` | Phase 2 workflow `wf_b77f01a5-1aa` (P5 missed finding); V0: git grep finds only the declaration and computation | | -| W03-043 | S3 | Reported | The HR-only path (a strap with no gravity) anchors its sleep band on a percentile of the whole read, the same window dependence W03-023 removed from the gravity path: heart rate half a day after a night moves the band the night is judged against | `SleepStager.hrOnlySessions` (`hrOnlyAnchorPercentile` over the read) | Found while fixing W03-023 (2026-10-03), by reading. V0 plan: the W03-023 invariance fixture with gravity removed and the far data changed in HR instead. V2 of W03-023/024 (2026-10-03): W03-024's 66-hour read reaches this path too. It widens the percentile's read, and the stored-session check before it (`sleepSessions` filters on start in the read) can now find a session that starts after midnight, so a no-gravity day can switch paths; HR-only nights may change on re-score, which the 2026-09-30 replay cannot show (that backup has gravity). Severity stays S3; landing it with or right after W03-024 is the reviewer's recommendation | | +| W03-043 | S3 | Fixed | The HR-only path (a strap with no gravity) anchors its sleep band on a percentile of the whole read, the same window dependence W03-023 removed from the gravity path: heart rate half a day after a night moves the band the night is judged against | `SleepStager.hrOnlySessions` (`hrOnlyAnchorPercentile` over the read) | Found while fixing W03-023 (2026-10-03), by reading. V0 plan: the W03-023 invariance fixture with gravity removed and the far data changed in HR instead. V2 of W03-023/024 (2026-10-03): W03-024's 66-hour read reaches this path too. It widens the percentile's read, and the stored-session check before it (`sleepSessions` filters on start in the read) can now find a session that starts after midnight, so a no-gravity day can switch paths; HR-only nights may change on re-score, which the 2026-09-30 replay cannot show (that backup has gravity). Severity stays S3; landing it with or right after W03-024 is the reviewer's recommendation. V0 (2026-10-08): `SleepHrOnlyWindowInvarianceTests`, the engine's 66 h read with three heart-rate-only nights and no gravity. Lowering only the next night's heart rate (56 to 46 bpm, 16 h after the scored night ends) removed the scored night, and a read ending 12 h after the night found the night the full read did not. Fix: `SleepStager.hrOnlyLocalAnchors`. Each epoch is held to the p10 of the heart rate in a 24 h window (radius `localBaselineRadiusS`), slid inward to stay inside the data at either end. A window cut at the data's end failed `AnalyticsEngineHrOnlyDayTests` and the stream-witness test: their read ends with the night, as this morning's read does, and the late epochs' anchor rose into the night. Sliding keeps every anchor on 24 h, the span `hrOnlyBandMult` was measured on. The trace prints the anchor and band as `lo..hi` over the read's epochs. V1: the invariance tests pass; restoring the read-wide anchor fails both; the sliding anchor equals the brute-force p10 of its window on 6,000 uneven samples with out-of-range bpm. StrandAnalytics 2,224 pass. Real data (probe over the 2026-09-30 backup, gravity withheld, the 12 stored nights of 3 h or more): the night's spine is the same whether the read is cut 12 h after it or not on 12 of 12 nights (old anchor: 3); nights found 12 (old 11), 2,578 minutes inside the gravity-path nights (old 2,639) of 7,269, 2 minutes outside them (old 0); the same at UTC+3 and UTC+5. Not changed: the stored-session gate in front of this path (W03-052). V2 (2026-10-21, independent subagent): holds. `hrOnlyLocalAnchors` against an independent brute force over 43,944 cases (0 to 3,000 samples, repeated timestamps, gaps up to a day, bpm from negative to 1,511, query times at every edge): 0 mismatches; on reads under 24 h the spine equals the old one-anchor spine on 27 of 27; with V2 and V3 the scored night's stages and resting HR, not only its bounds, are identical across the next night's heart rate and the read's reach (6 of 6); both invariance tests fail on `main`. 66 h at 1 Hz: 0.07 s for the anchors, 0.88 s for the spine (debug). Three doc claims it flagged are corrected on the branch: the anchor is the 24 h around the epoch slid inward at the ends, not within the radius (the gravity path's baseline cuts instead); yesterday's read reaches 12 h past the day only after local noon; the trace line is no longer the Kotlin line's byte twin. It added W03-053 … W03-055 | | | W03-044 | S3 | Fixed | The `stagingSparse` verdict stored on every night (the Sleep tab's "may be incomplete" caption), the Rest confidence guard and the sleep-motion trace came from `isGravitySparse` over the whole read, so a dense night was stamped sparse for a gravity gap half a day away | `AnalyticsEngine.analyzeDay` (`gravitySparse`, `stagingSparse`) | Found by the V2 review of W03-023/024 (2026-10-03): on the 2026-09-30 backup 8 of 12 banked sessions were stamped sparse, none with a gravity gap over 20 minutes inside its own span. V0: `SleepDetectWindowInvarianceTests.testADenseNightIsNotStampedSparseByAGapFarAway` failed (stamped true). Fix: each session's verdict is `LocalContext.sparse` at its midpoint (the bridge's ±6 h neighbourhood); the day's value, which Rest confidence and the trace read, is whether any session of its main night is sparse. V1: passes; the read-wide flag is the state V0 failed on; StrandAnalytics 2,215 pass | | | W03-046 | S4 | Fixed | Under W03-024's 66-hour window the per-day steps, respiration, raw SpO₂ and v18 auxiliary (byte 82 candidate) reads kept a literal 200,000 limit, below a full window at one row a second, so a dense window would lose its newest rows silently | `IntelligenceEngine.analyzeRecent` stream reads | Found by the V2 review of W03-023/024: about 230,000 step rows per 66-hour window on the 2026-09-30 backup. Nothing reads the clipped tail today (the steps total uses its own day read), so no number moved. Fix: `StreamReadCap.steps`, `.resp`, `.spo2` and `.v18Aux` (one row a second plus half again, 356,400; densities not measured), used by those reads. V1: `StreamReadCapTests.testOneRowASecondStreamsFitAFullWindow` and the pinned values pass | | | W03-045 | S3 | Fixed | W03-024's 66-hour read disabled the #997 day-slice shortcut: `daySliceFromNight` declines a night read at or above its `limit`, which defaulted to 200,000, below a complete 66-hour read at 1 Hz, so on almost every past day the HR, steps and gravity day streams were read from the store again (about 60 extra reads per pass) | `AnalyticsEngine.daySliceFromNight`; its three `IntelligenceEngine` call sites | Found by the V2 review of W03-023/024: HR, gravity and steps came back with 220,000 to 235,000 rows on 10 of 11 days of the 2026-09-30 backup. Data was never wrong (the shortcut only declines), only slower. V0: `DaySliceFromNightTests.testACompleteSixtySixHourReadAtOneHertzIsSliced` failed (declined). Fix: `limit` is required, and each call passes the cap its night read used (`StreamReadCap.hr`, `.steps`, `.gravity`), so a read at its cap still declines and a complete one slices. V1: six slice tests pass, including the truncated-read case at the HR cap | | @@ -118,6 +118,10 @@ every changed row explained; SleepBench / SleepPSG for any stager change. | W03-050 | S3 | Reported | Since W03-023 the #1657 active-run bridge (a short stir between two sleep runs) runs only within 6 h of where gravity is sparse; with the read-wide flag it ran on 8 of 11 reads of the 2026-09-30 backup because some gap existed somewhere in the read. A dense night with a stir far from any gravity gap now stays split into two sessions | `SleepStager.bridgeSparseSleepTraced`; `SleepStagerActiveBridgeTests` | Found by the V2 review of W03-023/024: bridged pairs over the backup's 11 day-reads fell from 16 to 2; a synthetic night with a dropout and a stir 7 h later gives 2 sessions (old 1). On the replay the stored nights it affects did not change, and the three spans that stopped being bridged scored 0 % band-asleep (W03-023's V3). The function-level #1657 tests call the bridge without a context, so they bypass locality. Owner's call: keep the #308 design (bridge only where gravity is sparse; a dense split night is grouped for the day's totals) or let a dense night bridge a short in-band stir on the HR band alone | | | W03-048 | S4 | Fixed | W03-023's `LocalContext` doc and commit say any read covering a session's 12 h neighbourhood gives the same session; that holds for the decisions it localises, not for detection as a whole: `classifyStill`'s window size, the overnight acceptance chain and the off-wrist gap gate still read the whole window. Comments around the night window also still described it as ending at the next midnight or at dayStart + 12 h | `SleepStager.LocalContext` doc; `AnalyticsEngine.analyzeDay` and `IntelligenceEngine` window comments; `StreamReadCapTests` | Found by the V2 review of W03-023/024 with two synthetic counter-examples (a 30-second-cadence read gaining 1 Hz gravity half a day away; a truncated predecessor changing a daytime block's acceptance); neither shape occurs on the 5/MG data measured. Fix: the doc names what still reads the whole window and why it does not move bounds on 5/MG reads; the stale window comments describe the 66-hour read (and that a span ending at `now` is not whole hours) | | | W03-051 | S4 | Fixed | `LocalContext.baseline` sized its per-call histogram by the largest bpm in the read, so one corrupt value (say 65,000) would make every baseline call allocate that many counters | `SleepStager.LocalContext.baseline(at:)` | Found by the V2 review of W03-023/024 (bpm 45 to 191 on the 2026-09-30 backup, so no effect there). V0 by reading. Fix: the histogram covers 0 to 511; values outside it are counted from a sorted list, as negatives already were, so the median is unchanged. V1: `testAnOutOfRangeBpmIsCountedWithoutSizingTheHistogram` (median equals `hrBaseline`; histogram bounded); the V2 oracle's 20,000-case equivalence is unaffected by construction | | +| W03-052 | S3 | Fixed | A day with no gravity skips the HR-only night whenever the owner has a stored session that STARTS anywhere in the day's read, though `analyzeDay` keeps only sessions that END on the day: since W03-024's 66 h read, a stored next night (starting after midnight, up to noon) blocks the day before from its own HR-only night, and a stored session from the previous day does the same | `IntelligenceEngine.swift` (`grav.count < 2` branch, `store.sleepSessions(deviceId: owner, from:, to:)` → `stored-sessions-exist`); `MetricsCache.sleepSessions` filters on `startTs` | Split out of W03-043's V2 note (2026-10-08), by reading. Reached by an owner whose id holds stored nights (an imported WHOOP export under `my-whoop`, a ring's hypnogram) on a day the strap banked no motion. V0 (2026-10-08): the gate extracted verbatim as `IntelligenceEngine.noMotionNight`; `NoMotionNightGateTests` failed on it: a stored next night (starting after midnight, ending the day after) and a stored previous night each blocked the day's HR-only night, for the import namespace and for a ring, the gate line counted both nights, and day membership ignored the offset. Fix: only a stored session whose end falls on the day under the pass's offset (`analyzeDay`'s rule) decides the gate and is counted in `stored=`; a ring's own night is still handed over with its neighbours, which `analyzeDay` filters. `NoMotionNight`'s raw values are the gate line's `reason=`, so the branch and the log share one name. V1: the four tests pass. V2 (2026-10-21, independent subagent): holds. The gate run standalone reproduces the four scenarios; day membership (`dayString(end, offsetSec:)` against `analyzeDay`'s range rule) agrees on 3,910 cases over ten offsets from −12 h to +14 h, including +5:30 and +5:45, with ends at every quarter hour across ±2 days and ±1 s at the edges. No sibling: the other readers of `sleepSessions(deviceId:)` gate no per-day decision on the read. The gate line's `stored=` now counts nights ending on the day, which its doc now says | | +| W03-053 | S4 | Reported | `hrOnlyLocalAnchors` keeps out-of-range bpm in a sorted side list whose insert and remove are linear in the window, so a read made mostly of values outside 0 … 511 goes quadratic | `SleepStager.hrOnlyLocalAnchors`; `LocalContext.baseline` (W03-051) has the same pattern | Found by W03-043's V2 (2026-10-21): 237,600 samples all at 65,535 took 5.6 s, random 600 to 1,600 bpm 3.0 s (debug). The strap decoders give byte-range bpm, so likely unreachable; StrandImport's heart-rate constructors not yet checked | | +| W03-054 | S4 | Reported | On a day with no gravity, a stored session ending on the day blocks the HR-only night even when it is only a nap, and on the HR-only branch the neighbouring days' stored nights are not handed to `analyzeDay`, so an HR-only run that overlaps a stored night ending the day before is not suppressed | `IntelligenceEngine.noMotionNight`; the `.hrOnly` branch | Found by W03-052's V2 (2026-10-21) with the gate run standalone (a 14:00 to 15:00 imported nap blocks the day, as before W03-052). The overlap case needs a split or partial stored night next to a strap that banked no motion; before W03-052 that day scored blank, now it could count the overlap on two days. Not reproduced in the app | | +| W03-055 | S4 | Reported | `hrOnlySleepRuns(_:baseline:)` now has only test callers, and `SleepStagerHrOnlyAnchorTests.testLongMultiNightWindowIsUnderReadNotOverRead` pins one anchor over 54 h, which production no longer runs | `SleepStager.hrOnlySleepRuns`; `SleepStagerHrOnlyAnchorTests` | Found by W03-043's V2 (2026-10-21). The `hrOnlyBandMult` doc already says so. Decide: re-point the test at `hrOnlyLocalAnchors` or delete the helper | | ## Log @@ -144,3 +148,6 @@ every changed row explained; SleepBench / SleepPSG for any stager change. - 2026-10-03 — W03-047 fixed on `review/w03-detection`; W03-049 and W03-050 recorded for the owner (behaviour the local rule changed). - 2026-10-03 — W03-048 fixed on `review/w03-detection` (doc). - 2026-10-03 — W03-051 fixed on `review/w03-detection`. +- 2026-10-08 — W03-043 fixed on `review/w03-hr-only` (the HR-only anchor is the p10 of the 24 h around each epoch); W03-052 split out of its V2 note. +- 2026-10-08 — W03-052 fixed on `review/w03-hr-only` (the no-motion gate counts only stored nights ending on the day). +- 2026-10-21 — V2 of W03-043 and W03-052 (both hold); three doc claims corrected on `review/w03-hr-only`; W03-053 … W03-055 recorded.