From 635ed3d41636f58f982981bf6530fcef000e1163 Mon Sep 17 00:00:00 2001 From: Fanboynz Date: Thu, 20 Aug 2026 21:42:12 +1200 Subject: [PATCH 1/2] Retire two superseded R-R shadow measurements (#1331/#1118) The shadow de-dup block ran 13 full passes over a night's R-R rows, six of them collapseOverCount, and each collapse SORTS the night's ~50-70k intervals. It runs for EVERY night of an affected strap, and analyzeRecent re-scores ~21 days every 15 minutes - so the phones least able to spare the work were paying the most, for instrumentation that feeds no stored value. Two of those measurements have finished their job. The same-second TOLERANCE SWEEP (20/34/60) was sizing a fix already ruled out. Every affected night reads crossSecondOverCount, so no same-second tolerance can reach duplicates that straddle the boundary - the conclusion recorded on #1331 before this change. `xsec`, the 40 ms collapse widened to a 1-second window, was documented from the start as a strict UPPER BOUND that over-merges real beats, kept only to size how far a cross-second collapse COULD get. It has now produced that number in the field: covXsec 0.80 with beatAccXsec 0.26 - coverage driven BELOW 1.0 and accuracy collapsed, eating real beats exactly as its own comment predicted. What survives is the honest floor (`ex`, exact duplicates only, provably no real-beat loss) and the incumbent candidate (`dd`, 40 ms same-second). The delivery histogram merged in #1486 supersedes what both retired measurements reached for, and costs one pass rather than nine. 13 passes per night became 6, and six sorts became two, identically on both platforms - verified by counting call sites in each rather than by reading the diff. Neither platform still emits `hrv sweep` or the xsec fields, and the replacement fragment in the surviving `hrv dedup` line is byte-identical across the two. An earlier revision of this commit said six sorts "became three". It is two: ex and dd. Corrected in the code rather than left as a plausible-sounding number, since that is the whole failure mode this block exists to avoid. Verification: compileFullDebugKotlin clean, full Android unit suite green, doc_comment_lint clean. --- Strand/Data/IntelligenceEngine.swift | 46 +++++++------------ .../com/noop/analytics/IntelligenceEngine.kt | 46 +++++++------------ 2 files changed, 32 insertions(+), 60 deletions(-) diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index 7889df2642..0451e28d62 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -1103,30 +1103,32 @@ final class IntelligenceEngine: ObservableObject { // same-second collapse is the aggressive UPPER BOUND (it also catches the two-channel // twins but can over-merge two real neighbours whose values sit within 40 ms). The // real de-dup lives between them; the log shows both so we can see where. - // #1331: a THIRD candidate, `xsec` — the 40 ms collapse widened to a 1-second WINDOW. - // Every night here reads `crossSecondOverCount`, meaning the same-second collapses - // above CAN'T reach the duplicates (they straddle the second boundary), so `cov40` - // stays ~1.7-2.0 on the heavy nights and respiratory stays blanked. `xsec` measures - // how far a cross-second collapse WOULD get (does coverage fall to ~1.0, does - // beat-accuracy clear #1127's 0.5 gate?). It is a strict UPPER BOUND, not a shippable - // de-dup: a steady real HR has ~identical intervals one second apart, so this - // over-merges real beats. Sizing only; the real fix is density/timeline-based + - // ground-truth-validated (@artemc's H10). Instrumentation, shipped path unchanged. + // #1331 RETIRED (two of them). `xsec` — the 40 ms collapse widened to a 1-second + // window — was a strict UPPER BOUND that over-merges real beats, kept only to size + // how far a cross-second collapse COULD get. It has now produced that number in the + // field: covXsec 0.80 with beatAccXsec 0.26, i.e. it eats real beats exactly as its + // own comment predicted. And the same-second TOLERANCE SWEEP (20/34/60) was sizing a + // fix already ruled out — every affected night reads `crossSecondOverCount`, so no + // same-second tolerance can reach duplicates that straddle the boundary. + // + // Both cost real work on the phones least able to spare it: this block runs for + // EVERY night of an affected strap, `analyzeRecent` re-scores ~21 days every 15 + // minutes, and each `collapseOverCount` SORTS the night's ~50-70k intervals. Six + // sorts per night became two. What survives is the honest floor (`ex`, exact + // duplicates only — provably no real-beat loss) and the incumbent candidate (`dd`, + // 40 ms same-second). The delivery histogram below supersedes what both retired + // measurements were reaching for, and costs one pass instead of nine. let ex = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr, rrTolMs: 0) let dd = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr) - let xs = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr, rrTolMs: 40, windowSec: 1) let hDd = HRVAnalyzer.analyze(rawRR: dd.rrMs) let covEx = HRVAnalyzer.rrCoverage(tsSec: ex.tsSec, rrMs: ex.rrMs) let covDd = HRVAnalyzer.rrCoverage(tsSec: dd.tsSec, rrMs: dd.rrMs) let accDd = HRVAnalyzer.beatAccurateFraction(tsSec: dd.tsSec, rrMs: dd.rrMs) - let covXs = HRVAnalyzer.rrCoverage(tsSec: xs.tsSec, rrMs: xs.rrMs) - let accXs = HRVAnalyzer.beatAccurateFraction(tsSec: xs.tsSec, rrMs: xs.rrMs) diagLine += "\nhrv dedup day=\(res.daily.day) exactN=\(ex.rrMs.count)/\(sleepRr.count) " + "covExact=\(String(format: "%.2f", covEx)) | ch40N=\(dd.rrMs.count) " + "cov40=\(String(format: "%.2f", covDd)) beatAcc40=\(String(format: "%.2f", accDd)) " + "rmssd40=\(ms(hDd.rmssd))ms sdnn40=\(ms(hDd.sdnn))ms meanNN40=\(ms(hDd.meanNN))ms " - + "| xsecN=\(xs.rrMs.count) covXsec=\(String(format: "%.2f", covXs)) " - + "beatAccXsec=\(String(format: "%.2f", accXs)) (1s upper bound)" + + "| collapse candidates only; the DELIVERY histogram below sizes the fix" // #1118 sweep: the same-second collapse at a range of tolerances, so a capture shows // WHICH tolerance the over-count actually responds to instead of only the one 40 ms // point. 34 ms is the two-optical-channel twin spacing; 0 is exact-duplicates-only. @@ -1135,22 +1137,6 @@ final class IntelligenceEngine: ObservableObject { // intervals — ~50k on an over-count night. Reusing them keeps the sweep to three extra // passes instead of five on a block that runs for EVERY night of an affected strap, // inside the per-day rescore loop #836 already had to slim down. Twin of Kotlin. - let accEx = HRVAnalyzer.beatAccurateFraction(tsSec: ex.tsSec, rrMs: ex.rrMs) - func sweepPoint(_ tol: Int) -> (cov: Double, acc: Double) { - let c = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr, rrTolMs: Double(tol)) - return (HRVAnalyzer.rrCoverage(tsSec: c.tsSec, rrMs: c.rrMs), - HRVAnalyzer.beatAccurateFraction(tsSec: c.tsSec, rrMs: c.rrMs)) - } - let p20 = sweepPoint(20), p34 = sweepPoint(34), p60 = sweepPoint(60) - let points: [(tol: Int, cov: Double, acc: Double)] = [ - (0, covEx, accEx), (20, p20.cov, p20.acc), (34, p34.cov, p34.acc), - (40, covDd, accDd), (60, p60.cov, p60.acc), - ] - let sweep = points.map { p -> String in - "t\(p.tol)=\(String(format: "%.2f", p.cov))/\(String(format: "%.2f", p.acc))" - }.joined(separator: " ") - diagLine += "\nhrv sweep day=\(res.daily.day) n=\(sleepRr.count) " - + "cov/acc by same-second tol: \(sweep)" } hrvDiag = diagLine // #1118: flag this night's HRV as over-counted (same verdict the diag logs) so the diff --git a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt index 588090db0b..3f98d41589 100644 --- a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt @@ -909,29 +909,33 @@ object IntelligenceEngine { // ts AND value, no real-beat loss) is the safe floor; the ~40 ms collapse is the // aggressive UPPER BOUND (catches the two-channel twins but can over-merge two real // neighbours within 40 ms). Log both so we can see where the real de-dup sits. Twin. - // #1331: a THIRD candidate, `xsec` — the 40 ms collapse widened to a 1-second WINDOW. - // crossSecondOverCount means the same-second collapses above can't reach the boundary- - // straddling twins (cov40 stays ~1.7-2.0 on heavy nights, resp blanked); `xsec` sizes how - // far a cross-second collapse WOULD get (coverage → ~1.0? beat-accuracy clears the 0.5 - // gate?). Strict UPPER BOUND, not shippable (a steady HR has ~identical intervals one - // second apart, so it over-merges real beats); the real fix is density/timeline-based + - // H10-validated. Instrumentation only, shipped path unchanged. Twin of the Swift line. + // #1331 RETIRED (two of them). `xsec` — the 40 ms collapse widened to a 1-second + // window — was a strict UPPER BOUND that over-merges real beats, kept only to size how + // far a cross-second collapse COULD get. It has now produced that number in the field: + // covXsec 0.80 with beatAccXsec 0.26, i.e. it eats real beats exactly as its own + // comment predicted. And the same-second TOLERANCE SWEEP (20/34/60) was sizing a fix + // already ruled out — every affected night reads CROSS_SECOND_OVER_COUNT, so no + // same-second tolerance can reach duplicates that straddle the boundary. + // + // Both cost real work on the phones least able to spare it: this block runs for EVERY + // night of an affected strap, analyzeRecent re-scores ~21 days every 15 minutes, and + // each collapseOverCount SORTS the night's ~50-70k intervals. Six sorts per night + // became two. What survives is the honest floor (`ex`, exact duplicates only — + // provably no real-beat loss) and the incumbent candidate (`dd`, 40 ms same-second). + // The delivery histogram below supersedes what both retired measurements reached for, + // and costs one pass instead of nine. Twin of the Swift block. val ex = HrvAnalyzer.collapseOverCount(ts, sleepRr, 0.0) val dd = HrvAnalyzer.collapseOverCount(ts, sleepRr) - val xs = HrvAnalyzer.collapseOverCount(ts, sleepRr, 40.0, 1L) val hDd = HrvAnalyzer.analyzeRaw(dd.second) val covEx = HrvAnalyzer.rrCoverage(ex.first, ex.second) val covDd = HrvAnalyzer.rrCoverage(dd.first, dd.second) val accDd = HrvAnalyzer.beatAccurateFraction(dd.first, dd.second) - val covXs = HrvAnalyzer.rrCoverage(xs.first, xs.second) - val accXs = HrvAnalyzer.beatAccurateFraction(xs.first, xs.second) dayDiag("hrv dedup day=${res.daily.day} exactN=${ex.second.size}/${sleepRr.size} " + "covExact=${String.format(java.util.Locale.US, "%.2f", covEx)} | ch40N=${dd.second.size} " + "cov40=${String.format(java.util.Locale.US, "%.2f", covDd)} " + "beatAcc40=${String.format(java.util.Locale.US, "%.2f", accDd)} " + "rmssd40=${ms(hDd.rmssd)}ms sdnn40=${ms(hDd.sdnn)}ms meanNN40=${ms(hDd.meanNN)}ms " + - "| xsecN=${xs.second.size} covXsec=${String.format(java.util.Locale.US, "%.2f", covXs)} " + - "beatAccXsec=${String.format(java.util.Locale.US, "%.2f", accXs)} (1s upper bound)") + "| collapse candidates only; the DELIVERY histogram below sizes the fix") // #1118 sweep: the same-second collapse at a range of tolerances, so a capture shows // WHICH tolerance the over-count actually responds to instead of only the one 40 ms // point. 34 ms is the two-optical-channel twin spacing; 0 is exact-duplicates-only. @@ -939,24 +943,6 @@ object IntelligenceEngine { // (collapseOverCount's default tolerance is 40), and each collapse sorts the night's // intervals — ~50k on an over-count night. Reusing them keeps the sweep to three extra // passes instead of five on a block that runs for EVERY night of an affected strap. - val accEx = HrvAnalyzer.beatAccurateFraction(ex.first, ex.second) - fun sweepPoint(tol: Int): Pair { - val c = HrvAnalyzer.collapseOverCount(ts, sleepRr, tol.toDouble()) - return HrvAnalyzer.rrCoverage(c.first, c.second) to - HrvAnalyzer.beatAccurateFraction(c.first, c.second) - } - val p20 = sweepPoint(20) - val p34 = sweepPoint(34) - val p60 = sweepPoint(60) - val sweep = listOf( - Triple(0, covEx, accEx), Triple(20, p20.first, p20.second), - Triple(34, p34.first, p34.second), Triple(40, covDd, accDd), - Triple(60, p60.first, p60.second), - ).joinToString(" ") { (tol, cov, acc) -> - "t$tol=${String.format(java.util.Locale.US, "%.2f", cov)}/" + - String.format(java.util.Locale.US, "%.2f", acc) - } - dayDiag("hrv sweep day=${res.daily.day} n=${sleepRr.size} cov/acc by same-second tol: $sweep") } } else if (res.sleepSessions.isEmpty()) { // #1244: no in-sleep R-R AND no detected session (past the >=200-HR gate) = the "HR tracked, From 9416b92985d73511d5fef06ec6e61c8866f3987c Mon Sep 17 00:00:00 2001 From: Fanboynz Date: Thu, 20 Aug 2026 21:44:55 +1200 Subject: [PATCH 2/2] Point at the histogram in the direction it actually appears Re-review. The retirement comment and the surviving `hrv dedup` line both told the reader the delivery histogram is "below". It is emitted BEFORE the dedup line - Swift appends it at 1092 against ~1120, Kotlin at 901 against ~928 - so in the log it sits ABOVE. A pointer in a diagnostic is only worth anything if it points the right way, and this one would have sent someone reading a strap log looking past the thing it was recommending. Four references, two per platform, now say above. Nothing else changed. Verified this pass: no orphaned variables survive the deletion on either platform, nothing anywhere consumes `hrv sweep` or the xsec fields (no readout, test or doc parses them), the delivery histogram is still wired on both, and the nine surviving fields of the `hrv dedup` line match across the two platforms name for name. Verification: compileFullDebugKotlin clean, full Android unit suite green, doc_comment_lint clean, emitted fragment byte-identical across platforms. --- Strand/Data/IntelligenceEngine.swift | 4 ++-- .../src/main/java/com/noop/analytics/IntelligenceEngine.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index 0451e28d62..89aa667644 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -1116,7 +1116,7 @@ final class IntelligenceEngine: ObservableObject { // minutes, and each `collapseOverCount` SORTS the night's ~50-70k intervals. Six // sorts per night became two. What survives is the honest floor (`ex`, exact // duplicates only — provably no real-beat loss) and the incumbent candidate (`dd`, - // 40 ms same-second). The delivery histogram below supersedes what both retired + // 40 ms same-second). The delivery histogram above supersedes what both retired // measurements were reaching for, and costs one pass instead of nine. let ex = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr, rrTolMs: 0) let dd = HRVAnalyzer.collapseOverCount(tsSec: ts, rrMs: sleepRr) @@ -1128,7 +1128,7 @@ final class IntelligenceEngine: ObservableObject { + "covExact=\(String(format: "%.2f", covEx)) | ch40N=\(dd.rrMs.count) " + "cov40=\(String(format: "%.2f", covDd)) beatAcc40=\(String(format: "%.2f", accDd)) " + "rmssd40=\(ms(hDd.rmssd))ms sdnn40=\(ms(hDd.sdnn))ms meanNN40=\(ms(hDd.meanNN))ms " - + "| collapse candidates only; the DELIVERY histogram below sizes the fix" + + "| collapse candidates only; the DELIVERY histogram above sizes the fix" // #1118 sweep: the same-second collapse at a range of tolerances, so a capture shows // WHICH tolerance the over-count actually responds to instead of only the one 40 ms // point. 34 ms is the two-optical-channel twin spacing; 0 is exact-duplicates-only. diff --git a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt index 3f98d41589..32aff84c0a 100644 --- a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt @@ -922,7 +922,7 @@ object IntelligenceEngine { // each collapseOverCount SORTS the night's ~50-70k intervals. Six sorts per night // became two. What survives is the honest floor (`ex`, exact duplicates only — // provably no real-beat loss) and the incumbent candidate (`dd`, 40 ms same-second). - // The delivery histogram below supersedes what both retired measurements reached for, + // The delivery histogram above supersedes what both retired measurements reached for, // and costs one pass instead of nine. Twin of the Swift block. val ex = HrvAnalyzer.collapseOverCount(ts, sleepRr, 0.0) val dd = HrvAnalyzer.collapseOverCount(ts, sleepRr) @@ -935,7 +935,7 @@ object IntelligenceEngine { "cov40=${String.format(java.util.Locale.US, "%.2f", covDd)} " + "beatAcc40=${String.format(java.util.Locale.US, "%.2f", accDd)} " + "rmssd40=${ms(hDd.rmssd)}ms sdnn40=${ms(hDd.sdnn)}ms meanNN40=${ms(hDd.meanNN)}ms " + - "| collapse candidates only; the DELIVERY histogram below sizes the fix") + "| collapse candidates only; the DELIVERY histogram above sizes the fix") // #1118 sweep: the same-second collapse at a range of tolerances, so a capture shows // WHICH tolerance the over-count actually responds to instead of only the one 40 ms // point. 34 ms is the two-optical-channel twin spacing; 0 is exact-duplicates-only.