Analytics (W3): leave low-quality SpO₂ candidate readings out of the night, one read for every surface

Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift

@@ -1208,8 +1208,10 @@ public enum AnalyticsEngine {
     }
 
     /// Nightly gated mean of the 5/MG SpO2 **candidate** byte (`@82`) over the detected in-bed
-    /// `sessions`, with the sample count it rests on — or nil when no in-band reading fell inside any
-    /// span. (#112, tracking #103.)
+    /// `sessions`, with the sample count it rests on — or nil when the night has no reliable reading:
+    /// no in-band reading fell inside any span, or every reading with one failed the quality check
+    /// (W03-007, W03-019). nil therefore does not mean the strap reported nothing; the full
+    /// `nightlySpo2CandidateNight` result says which. (#112, tracking #103.)
     ///
     /// WHY THIS EXISTS. The candidate is decoded and stored but deliberately never scored: `@82` looks
     /// like a strap-computed SpO2 %, and on one independent 8-night check it tracked the WHOOP app almost
@@ -1244,10 +1246,16 @@ public enum AnalyticsEngine {
     /// is the failure AGENTS.md's "two readouts of one fact" rule is about. The same in-band gate, the
     /// same inclusive session bounds and the same rounding; since W03-003 the mean averages measurement
     /// WINDOWS rather than seconds (see `Spo2CandidateNight`), and `samples` is still in-band seconds.
-    public static func nightlySpo2CandidateMean(_ sessions: [SleepSession],
-                                         aux: [V18AuxSample]) -> (mean: Int, samples: Int)? {
-        guard let night = nightlySpo2CandidateNight(sessions, aux: aux) else { return nil }
-        return (mean: night.meanRounded, samples: night.samples)
+    public static func nightlySpo2CandidateMean(
+        _ sessions: [SleepSession],
+        aux: [V18AuxSample],
+        minimumSeconds: Int = AnalyticsEngine.spo2CandidateMinReadingSeconds
+    ) -> (mean: Int, samples: Int)? {
+        guard let night = nightlySpo2CandidateNight(sessions, aux: aux, minimumSeconds: minimumSeconds),
+              let mean = night.meanRounded else {
+            return nil
+        }
+        return (mean: mean, samples: night.samples)
     }
 
     /// The plausible range for a raw Oura `0x6F` SpO2 sample before the ceiling transform below.

Packages/StrandAnalytics/Tests/StrandAnalyticsTests/Spo2CandidateNightlyTests.swift

@@ -18,8 +18,14 @@ final class Spo2CandidateNightlyTests: XCTestCase {
 
     private func aux(_ ts: Int, _ v: Int?) -> V18AuxSample { V18AuxSample(ts: ts, auxByte82: v) }
 
+    /// The mean with the in-band floor lowered to one second (W03-013): these cases pin the gate on
+    /// single readings, which the default ten-second floor would turn into low-quality readings.
+    private func mean(_ sessions: [SleepSession], aux: [V18AuxSample]) -> (mean: Int, samples: Int)? {
+        AnalyticsEngine.nightlySpo2CandidateMean(sessions, aux: aux, minimumSeconds: 1)
+    }
+
     func testMeanAndSampleCountOverTheSession() {
-        let r = AnalyticsEngine.nightlySpo2CandidateMean(
+        let r = mean(
             [session(1000, 600)], aux: [aux(1100, 94), aux(1200, 96), aux(1300, 95)])
         XCTAssertEqual(r?.mean, 95)
         XCTAssertEqual(r?.samples, 3)
@@ -29,37 +35,37 @@ final class Spo2CandidateNightlyTests: XCTestCase {
     /// and a volunteer's correlation built on the first would be worthless.
     func testSampleCountTravelsWithTheMean() {
         let many = (0..<50).map { aux(1000 + $0, 93) }
-        XCTAssertEqual(AnalyticsEngine.nightlySpo2CandidateMean([session(900, 600)], aux: many)?.samples, 50)
+        XCTAssertEqual(mean([session(900, 600)], aux: many)?.samples, 50)
     }
 
     /// Out-of-band values are DIAGNOSTIC CODES and SATURATION SENTINELS, not low blood oxygen. Averaging
     /// them in would produce a number that is not a percentage of anything — the single most damaging
     /// thing this helper could do, because it would look plausible.
     func testSubSeventyAndSentinelValuesAreExcluded() {
-        let r = AnalyticsEngine.nightlySpo2CandidateMean(
+        let r = mean(
             [session(1000, 600)],
             aux: [aux(1100, 94), aux(1150, 3), aux(1200, 0), aux(1250, 0x80), aux(1300, 96)])
         XCTAssertEqual(r?.mean, 95, "only the two in-band readings may count")
         XCTAssertEqual(r?.samples, 2)
     }
 
     func testDaytimeSamplesOutsideTheSessionAreExcluded() {
-        let r = AnalyticsEngine.nightlySpo2CandidateMean(
+        let r = mean(
             [session(1000, 600)], aux: [aux(500, 99), aux(1100, 94), aux(5000, 88)])
         XCTAssertEqual(r?.samples, 1)
         XCTAssertEqual(r?.mean, 94)
     }
 
     func testNilWhenNothingInBandFallsInsideASession() {
-        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean([session(1000, 600)], aux: [aux(1100, 5)]))
-        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean([session(1000, 600)], aux: [aux(9999, 95)]))
-        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean([], aux: [aux(1100, 95)]))
-        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean([session(1000, 600)], aux: []))
+        XCTAssertNil(mean([session(1000, 600)], aux: [aux(1100, 5)]))
+        XCTAssertNil(mean([session(1000, 600)], aux: [aux(9999, 95)]))
+        XCTAssertNil(mean([], aux: [aux(1100, 95)]))
+        XCTAssertNil(mean([session(1000, 600)], aux: []))
     }
 
     /// A record with no @82 at all is not a zero reading.
     func testMissingBytesAreSkippedNotCountedAsZero() {
-        let r = AnalyticsEngine.nightlySpo2CandidateMean(
+        let r = mean(
             [session(1000, 600)], aux: [aux(1100, nil), aux(1200, 96)])
         XCTAssertEqual(r?.mean, 96)
         XCTAssertEqual(r?.samples, 1)
@@ -69,16 +75,16 @@ final class Spo2CandidateNightlyTests: XCTestCase {
     /// helper shape. `sum / kept` on two `Int`s truncates toward zero; three samples averaging
     /// 97.666... must round to 98, not floor to 97.
     func testMeanRoundsRatherThanFloors() {
-        let r = AnalyticsEngine.nightlySpo2CandidateMean(
+        let r = mean(
             [session(1000, 600)], aux: [aux(1100, 98), aux(1200, 98), aux(1300, 97)])
         XCTAssertEqual(r?.mean, 98, "97.666... must round to 98, not floor to 97")
     }
 
     /// The boundaries are inclusive, matching the decoder's own `70...100` emit gate exactly.
     func testBandBoundariesMatchTheDecoderGate() {
-        XCTAssertEqual(AnalyticsEngine.nightlySpo2CandidateMean(
+        XCTAssertEqual(mean(
             [session(1000, 600)], aux: [aux(1100, 70), aux(1200, 100)])?.samples, 2)
-        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean(
+        XCTAssertNil(mean(
             [session(1000, 600)], aux: [aux(1100, 69), aux(1200, 101)]))
     }
 }

Packages/StrandAnalytics/Tests/StrandAnalyticsTests/Spo2CandidateQualityTests.swift

@@ -0,0 +1,110 @@
+import XCTest
+import WhoopProtocol
+@testable import StrandAnalytics
+
+/// W03-007 — a reading is only as good as the seconds behind it.
+///
+/// On a night the strap sat loose, byte 82's 30-second readings come back mixed with the strap's own
+/// non-percentage codes, and the few in-band seconds left between them are fragments that jump. Taken at
+/// face value those readings became the night's low and its dips. These cases are synthetic, shaped like
+/// the owner's 2026-09-29 and 2026-09-30 nights (no values copied from them).
+final class Spo2CandidateQualityTests: XCTestCase {
+
+    private func sess(_ s: Int, _ d: Int) -> SleepSession {
+        SleepSession(start: s, end: s + d, efficiency: 0.9, stages: [], restingHR: 50, avgHRV: 60)
+    }
+    private func aux(_ ts: Int, _ v: Int?) -> V18AuxSample { V18AuxSample(ts: ts, auxByte82: v) }
+    /// One reading: `values[i]` at `start + i`, 30 seconds for the strap's real window shape.
+    private func reading(_ start: Int, _ values: [Int]) -> [V18AuxSample] {
+        values.enumerated().map { aux(start + $0.offset, $0.element) }
+    }
+
+    /// A clean reading, as a well-fitted strap reports one.
+    private let clean = [95, 95, 96, 96, 96, 95, 95, 95, 96, 96, 96, 96, 95, 95, 95,
+                         96, 96, 96, 95, 95, 95, 96, 96, 96, 96, 95, 95, 95, 96, 96]
+    /// Half codes: a steady low run broken up by the codes a loose strap interleaves.
+    private let fragmented = [32, 32, 32, 160, 74, 74, 72, 74, 74, 75, 74, 74, 73, 73, 71,
+                              160, 160, 160, 74, 32, 32, 32, 75, 128, 128, 128, 72, 73, 128, 128]
+    /// Enough seconds, but they sweep across the band instead of agreeing on a value.
+    private let sweep = [128, 1, 128, 128, 128, 128, 128, 128, 84, 83, 88, 32, 83, 32, 75,
+                         72, 71, 72, 81, 90, 96, 99, 100, 100, 99, 100, 100, 100, 100, 100]
+
+    /// V0: the loose-strap night. Two of its readings are low-quality; neither may become a dip or the
+    /// night's low, and the average rests on the clean readings alone.
+    func testLowQualityReadingsAreNotDipsAndDoNotMoveTheNight() {
+        let aux = reading(0, clean) + reading(1200, fragmented) + reading(2400, sweep) + reading(3600, clean)
+        let n = AnalyticsEngine.nightlySpo2CandidateNight([sess(0, 5000)], aux: aux)
+        XCTAssertEqual(n?.events.count, 0)
+        XCTAssertEqual(n?.minimum, 96)
+        XCTAssertEqual(n?.windows, 2)
+        XCTAssertEqual(n?.windowsAttempted, 4)
+        XCTAssertEqual(n?.windowsLowCoverage, 1)
+        XCTAssertEqual(n?.windowsUnsettled, 1)
+    }
+
+    /// The gate judges what the strap reported, never whether the value is low. A steady, well-covered
+    /// low reading stays a low reading: a filter tuned on the outcome would hide the nights it is for.
+    func testASteadyWellCoveredLowReadingStaysALowReading() {
+        let steadyLow = (0..<30).map { 86 + $0 % 3 }
+        let n = AnalyticsEngine.nightlySpo2CandidateNight(
+            [sess(0, 3000)], aux: reading(0, clean) + reading(1200, steadyLow))
+        XCTAssertEqual(n?.events.count, 1)
+        XCTAssertEqual(n?.minimum, 87)
+        XCTAssertEqual(n?.windowsLowQuality, 0)
+    }
+
+    /// The two gates at the strap's real 30-second shape, at and one second past each boundary: 20 of
+    /// 30 in band passes and 19 fails; 18 of 30 agreeing seconds passes and 17 fails.
+    func testTheGatesAtTheirBoundariesOnAThirtySecondReading() {
+        func quality(_ values: [Int]) -> (low: Int, windows: Int)? {
+            AnalyticsEngine.nightlySpo2CandidateNight([sess(0, 600)], aux: reading(0, values))
+                .map { ($0.windowsLowQuality, $0.windows) }
+        }
+        let codes = { (n: Int) in Array(repeating: 128, count: n) }
+        XCTAssertEqual(quality(codes(10) + Array(repeating: 95, count: 20))?.windows, 1)
+        XCTAssertEqual(quality(codes(11) + Array(repeating: 95, count: 19))?.low, 1)
+        let spread = { (agree: Int) in Array(repeating: 95, count: agree) + (0..<(30 - agree)).map { $0 % 2 == 0 ? 80 : 99 } }
+        XCTAssertEqual(quality(spread(18))?.windows, 1)
+        XCTAssertEqual(quality(spread(17))?.low, 1)
+    }
+
+    /// W03-013: a ratio cannot judge a fragment. A reading cut to a few seconds by a session edge passes
+    /// both ratios trivially (one in-band second of one measured is full coverage and full agreement),
+    /// so it needs a floor; below it, it is low quality, never the night's low.
+    func testAFewSecondFragmentIsNotAReliableReading() {
+        let fragment = [aux(1190, 74)]
+        let n = AnalyticsEngine.nightlySpo2CandidateNight([sess(0, 1190)], aux: reading(0, clean) + fragment)
+        XCTAssertEqual(n?.windows, 1)
+        XCTAssertEqual(n?.windowsLowCoverage, 1)
+        XCTAssertEqual(n?.minimum, 96)
+        XCTAssertTrue(n!.events.isEmpty)
+    }
+
+    /// A night on which the strap attempted readings and none was reliable is NOT nil: it is a night
+    /// with no reliable reading, which is exactly what a surface needs to say.
+    func testANightOfFailedReadingsIsStatedNotDropped() {
+        let n = AnalyticsEngine.nightlySpo2CandidateNight(
+            [sess(0, 3000)], aux: reading(0, fragmented) + reading(1200, sweep))
+        XCTAssertNotNil(n)
+        XCTAssertNil(n?.mean)
+        XCTAssertNil(n?.minimum)
+        XCTAssertEqual(n?.windowsAttempted, 2)
+        XCTAssertEqual(n?.windowsLowQuality, 2)
+        XCTAssertTrue(n!.events.isEmpty)
+        XCTAssertNil(AnalyticsEngine.nightlySpo2CandidateMean([sess(0, 3000)],
+                                                              aux: reading(0, fragmented)),
+                     "the legacy integer mean has no answer either")
+    }
+
+    /// The chart's seconds and the night's figures are cut by the same windows and the same verdict: the
+    /// trace holds exactly the reliable readings' in-band seconds, so it cannot draw a low the tiles left
+    /// out.
+    func testTheTraceHoldsOnlyTheReliableReadingsSeconds() {
+        let aux = reading(0, clean) + reading(1200, fragmented) + reading(2400, sweep)
+        let trace = AnalyticsEngine.spo2CandidateReliableSeconds([sess(0, 3000)], aux: aux)
+        let night = AnalyticsEngine.nightlySpo2CandidateNight([sess(0, 3000)], aux: aux)
+        XCTAssertEqual(trace.count, night?.samples)
+        XCTAssertEqual(trace.map(\.ts), Array(0..<30))
+        XCTAssertEqual(trace.map(\.value).min(), 95)
+    }
+}

Strand/Data/IntelligenceEngine.swift

@@ -223,8 +223,9 @@ final class IntelligenceEngine: ObservableObject {
         let hrvDiag: String?
         /// #103/queue-11a: this night's whole SpO₂ candidate result — mean, range and below-threshold
         /// dips — computed off the main actor when the SpO₂ candidate display toggle is ON from the
-        /// `spo2_candidate_82` V18Aux byte. nil when the toggle is OFF or the night has no in-band
-        /// reading. Written to metricSeries under the "-noop" device ID in pass 2.
+        /// `spo2_candidate_82` V18Aux byte. nil when the toggle is OFF or the strap measured nothing in
+        /// the night; a night whose readings all failed the quality check is non-nil with no mean
+        /// (W03-007). Written to metricSeries under the "-noop" device ID in pass 2.
         ///
         /// Carries the whole `Spo2CandidateNight` rather than the mean it used to: the dips are shown
         /// beside that mean now, and re-resolving them in pass 2 would mean two loops over one night's
@@ -1679,8 +1680,9 @@ final class IntelligenceEngine: ObservableObject {
                 // #103/queue-11a: the night's SpO₂ candidate result — the in-band (70–100)
                 // `spo2_candidate_82` V18Aux readings resolved per measurement window (W03-003) into a
                 // mean, a range, the night's dip readings and its window counts. Only computed when the
-                // display toggle is ON; nil when there is no v18 aux stream, no in-band reading, or the
-                // toggle is OFF.
+                // display toggle is ON; nil when there is no v18 aux stream, no nonzero byte-82 second,
+                // or the toggle is OFF. A night whose readings all failed the quality check is non-nil
+                // with no mean (W03-007).
                 //
                 // Resolved ONCE, here, for everything pass 2 writes: the mean, the minimum, the dip count,
                 // the dip span and the window counts are all read off this single value rather than
@@ -2034,6 +2036,10 @@ final class IntelligenceEngine: ObservableObject {
         // Rest composite (0–100) per computed night, persisted as the `sleep_performance` metric
         // series so the dashboard's Rest score reflects the new composite, not raw efficiency.
         var restPoints: [MetricPoint] = []
+        // W03-007: days whose SpO₂ candidate night had readings but none reliable. Their mean and minimum
+        // rows are deleted after the persist, so a night that re-scores to "no reliable reading" cannot
+        // keep showing the average an earlier pass stored.
+        var spo2NoReliableReadingDays: [String] = []
         // User-corrected sleep windows override the detected sleep when scoring a day's sleep aggregates,
         // so Rest + recovery honor the edit , not just the Sleep tab's session view. An edited block
         // substitutes its detected twin (matched by the stable detected startTs) before totals recompute.
@@ -2286,11 +2292,22 @@ final class IntelligenceEngine: ObservableObject {
             // Every key comes from `Spo2CandidateSeries`, never a literal spelled here — the reader looks
             // these up optionally, so a key spelled twice could drift on one side and bank nothing for a
             // field nobody notices is missing (the `V18AuxSlot.decoderKey` lesson, one layer up).
+            //
+            // W03-007: the figures rest on RELIABLE readings only, and a night whose readings all failed
+            // the quality check has no mean and no minimum. Its counts are still written, with the
+            // low-quality key as the marker of a gated night, and a mean or minimum an earlier pass
+            // stored for that day is deleted after the persist below (`spo2NoReliableReadingDays`).
             if let cand = spo2CandidateByDay[daily.day] {
-                restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.meanKey,
-                                              value: cand.mean))
-                restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.minimumKey,
-                                              value: Double(cand.minimum)))
+                if let mean = cand.mean, let minimum = cand.minimum {
+                    restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.meanKey,
+                                                  value: mean))
+                    restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.minimumKey,
+                                                  value: Double(minimum)))
+                } else {
+                    spo2NoReliableReadingDays.append(daily.day)
+                }
+                restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.lowQualityKey,
+                                              value: Double(cand.windowsLowQuality)))
                 restPoints.append(MetricPoint(day: daily.day, key: Spo2CandidateSeries.samplesKey,
                                               value: Double(cand.samples)))
                 // W03-003: the window counts. Their presence is also what tells a reader that this
@@ -2535,7 +2552,7 @@ final class IntelligenceEngine: ObservableObject {
             }
             markerSources = sourceIds
         }
-        try? await store.persistComputedScores(
+        let persistedScores = (try? await store.persistComputedScores(
             dailyMetrics: persistedDailies,
             metricPoints: restPoints,
             provenance: Array(provenanceByCell.values),
@@ -2545,7 +2562,7 @@ final class IntelligenceEngine: ObservableObject {
             replaceMetricKeys: markerKeys,
             additionalMetricPoints: markerPoints,
             replaceMetricSourceIds: markerSources
-        )
+        )) != nil
 
         // Now evict only the STALE computed rows in the window , those a prior (e.g. UTC-keyed) run left
         // behind that the current local-keyed run no longer produces. Read the window, diff against the
@@ -2568,6 +2585,13 @@ final class IntelligenceEngine: ObservableObject {
                 _ = try? await store.deleteMetricSeriesPoint(deviceId: computedId, day: day,
                                                             key: "sleep_performance")
             }
+            // Only after a persist that landed: deleting the mean and minimum when the gated night's
+            // counts were never written would leave a half-written night, which the funnel skips.
+            for day in spo2NoReliableReadingDays where persistedScores {
+                for key in [Spo2CandidateSeries.meanKey, Spo2CandidateSeries.minimumKey] {
+                    _ = try? await store.deleteMetricSeriesPoint(deviceId: computedId, day: day, key: key)
+                }
+            }
         }
         markPostLoopPhase("persist")
         // ── Fitness Age (Phase 2) , weekly, keyed to the week's Saturday ────────────────────────────