From decd9f16fe1f28441c6b4be951ac97b4ad1c8f90 Mon Sep 17 00:00:00 2001 From: Fanboynz Date: Wed, 9 Sep 2026 21:24:15 +1200 Subject: [PATCH] perf(steps): persist the steps-calibration motion folds across launches The calibration fits one coefficient from sixty days of strap motion, and StepsMotionCache already stops it re-folding all sixty on every pass within a process. The cache was in-memory only, so a relaunch re-paid the whole fold: 32864ms cold against 3330ms warm on a worn 60-day library, once per launch. Persist it. dayMotionIntensity is a pure fold over one day's gravity stream, so unlike the day-scan cache there is no pass-config to carry: that one keys on dayScanCacheConfigSig, which folds in baselines1 and the habitual-sleep terms, and sleepConsistency (1-CV over 28 nights) and habitualMidsleepSec (a circular mean) both shift with any night moving, so a persisted day scan would invalidate wholesale on exactly the passes it would need to survive. Adds a fold version to the payload, because cacheKey witnesses the inputs only: a day whose gravity has not moved keys identically across an app update that changed the fold, and without the version check the pre-update volume would be served until that day's stream happened to change. Bump it on any change to dayMotionIntensity. The payload stays a derived cache that nothing else reads, so the whole failure surface is one re-fold. A payload that is missing, truncated, over-long or written by an older fold is discarded rather than repaired, and a malformed line costs only its own day. Storage is UserDefaults / SharedPreferences, not the record schema, so there is no migration. Android keeps the analytics layer Context-free by threading accessors down from the caller, mirroring manualStepCoefficient and persistStepsCalibration; they default to no-ops, so an unwired caller keeps today's per-process behaviour. Load and save sit in the analyzeRecent wrapper under analyzeGate, never in analyzeRecentOnCpu, so the #1524 method budget is untouched. The store is written only when the payload moved. A pass that reused every day renders the string already stored, and those are the back-to-back passes an offload storm is made of, so the write is skipped rather than relying on the store to discard it. The Android accessors are nullable and the one-shot load latches only when a store was actually consulted, so a caller that does not wire them cannot spend the single load on nothing and leave the rest of the process unpersisted. Both platforms pin the same rendered payload as a literal, so a one-sided change to the header, the separators or the number rendering fails a test rather than quietly ending reuse after an update. --- .../StrandAnalytics/StepsMotionCache.swift | 69 +++++++++- .../StepsMotionCacheTests.swift | 106 ++++++++++++++ Strand/Data/IntelligenceEngine.swift | 32 +++++ .../com/noop/analytics/IntelligenceEngine.kt | 47 ++++++- .../com/noop/analytics/StepsMotionCache.kt | 78 ++++++++++- .../main/java/com/noop/ble/WhoopBleClient.kt | 6 + .../src/main/java/com/noop/ui/AppViewModel.kt | 12 ++ .../src/main/java/com/noop/ui/MainActivity.kt | 12 ++ .../noop/analytics/StepsMotionCacheTest.kt | 130 ++++++++++++++++++ 9 files changed, 485 insertions(+), 7 deletions(-) diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/StepsMotionCache.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/StepsMotionCache.swift index 0a9a6b79d8..36809e2811 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/StepsMotionCache.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/StepsMotionCache.swift @@ -14,9 +14,22 @@ import Foundation /// there is no pass-config signature to invalidate against — the value changes exactly when that one day's /// gravity changes, and the key below is the whole story. /// -/// Like the day-scan cache this is in-memory and per-process. It never persists, never crosses the -/// `.noopbak` boundary, and a relaunch pays the full fold once. That is deliberate: the cost being closed is -/// the repeat within a process, where an offload storm fires passes back to back. +/// Unlike the day-scan cache this one PERSISTS across launches, and the paragraph above is why it can. A +/// persisted day scan would have to carry `dayCacheConfigSig`, which folds in baselines1 and the habitual +/// sleep terms — and `sleepConsistency` (1-CV over 28 nights) and `habitualMidsleepSec` (a circular mean) +/// shift with ANY night moving, so it would invalidate wholesale on exactly the passes it would need to +/// survive. Nothing pass-global reaches this fold, so the payload stays valid while gravity stands still and +/// the sixty-day fold is paid once per install instead of once per launch: measured 32.9 s -> 3.3 s on a +/// worn 60-day library, which the cold pass otherwise repaid after every relaunch. +/// +/// It stays a DERIVED cache and nothing else reads it, so the whole failure surface is one re-fold: a +/// payload that is missing, unreadable, or written by an older fold is discarded rather than repaired. +/// +/// It still does NOT cross the `.noopbak` boundary, and must not start: the key is deliberately absent from +/// both `BackupSettings` whitelists. A restore carries the settings and the record store, and this describes +/// neither — it describes gravity rows AS THEY WERE ON ONE DEVICE. Shipping it to another device would be +/// the one way to serve a fold whose key no longer witnesses anything, which is the failure every other +/// guard here exists to prevent. A restored device simply re-folds once. public enum StepsMotionCache { /// The per-day reuse key. Reuse a cached motion volume iff this string is unchanged. /// @@ -39,4 +52,54 @@ public enum StepsMotionCache { public static func logLine(reused: Int, folded: Int, size: Int) -> String { "analyzeRecent stepsMotion reused=\(reused)/\(reused + folded) size=\(size)" } + + /// The version of the FOLD the persisted values were produced by. Bump on any change to + /// `StepsEstimateEngine.dayMotionIntensity` that moves what it returns for the same samples. + /// + /// In memory this could not exist: a process cannot outlive the binary that filled it, so the fold that + /// produced a cached value is always the fold that would reproduce it. A persisted entry outlives its + /// build, and `cacheKey` witnesses the INPUTS only — a day whose gravity has not moved keys identically + /// across an app update, so without this the old volume would be served until that day's stream happened + /// to change. A bump discards every entry and costs one full re-fold, once, which is the cheap side. + public static let foldVersion = 1 + + /// Render the cache for storage. Days are emitted in sorted order so an unchanged cache renders to an + /// identical payload and the write is a no-op rather than churn. + /// + /// One line per day, `day\tkey\tmotion`, under a header naming `foldVersion`. Tab-separated because the + /// key itself contains `|`; the motion is written by raw bit pattern so it round-trips exactly and + /// locale-free, the same reason `cacheKey` encodes the skin anchor that way. + public static func serialize(_ entries: [String: (key: String, motion: Double)]) -> String { + var out = header + for day in entries.keys.sorted() { + guard let e = entries[day] else { continue } + out += "\n\(day)\t\(e.key)\t\(e.motion.bitPattern)" + } + return out + } + + /// Parse a stored payload. Returns empty on anything it cannot vouch for — a wrong/absent header (an + /// older `foldVersion` included), a malformed line, or an implausible entry count. Every rejection costs + /// one re-fold, so this discards rather than salvages: a half-trusted cache is the one failure mode that + /// could feed a stale volume into the calibration fit. + public static func deserialize(_ raw: String) -> [String: (key: String, motion: Double)] { + var lines = raw.split(separator: "\n", omittingEmptySubsequences: false) + guard lines.first == Substring(header) else { return [:] } + lines.removeFirst() + guard lines.count <= maxEntries else { return [:] } + var out: [String: (key: String, motion: Double)] = [:] + for line in lines { + let f = line.split(separator: "\t", omittingEmptySubsequences: false) + guard f.count == 3, !f[0].isEmpty, !f[1].isEmpty, let bits = UInt64(f[2]) else { continue } + out[String(f[0])] = (key: String(f[1]), motion: Double(bitPattern: bits)) + } + return out + } + + /// Payload header. Carries `foldVersion` so a fold change invalidates by failing the equality check. + private static var header: String { "stepsMotion v\(foldVersion)" } + + /// Upper bound on entries a payload may declare. The writer prunes to the calibration window every pass, + /// so a payload far above it did not come from this cache and is not worth parsing. + private static let maxEntries = 512 } diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/StepsMotionCacheTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/StepsMotionCacheTests.swift index b09c8ce0e5..22aa3dc320 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/StepsMotionCacheTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/StepsMotionCacheTests.swift @@ -50,4 +50,110 @@ final class StepsMotionCacheTests: XCTestCase { XCTAssertEqual(StepsMotionCache.logLine(reused: 0, folded: 60, size: 60), "analyzeRecent stepsMotion reused=0/60 size=60") } + + // MARK: - Persistence + + /// The exact payload the cache renders, pinned as a literal. This is the cross-platform contract that + /// matters most now the cache is stored: the Kotlin twin pins the SAME string, so a one-sided change to + /// the header, the separators or the number rendering is caught here rather than by a user whose folds + /// silently stopped being reused after an update. + static let vector = """ + stepsMotion v1 + 2026-09-01\tmy-whoop|8640|1757000000\t4659742922898407424 + 2026-09-02\tmy-whoop|0|0\t0 + """ + + static let vectorEntries: [String: (key: String, motion: Double)] = [ + "2026-09-01": (key: "my-whoop|8640|1757000000", motion: 3421.75), + "2026-09-02": (key: "my-whoop|0|0", motion: 0), + ] + + func testSerializeRendersThePinnedPayload() { + XCTAssertEqual(StepsMotionCache.serialize(Self.vectorEntries), Self.vector) + } + + func testRoundTripPreservesKeysAndVolumes() { + let back = StepsMotionCache.deserialize(StepsMotionCache.serialize(Self.vectorEntries)) + XCTAssertEqual(back.count, 2) + for (day, want) in Self.vectorEntries { + XCTAssertEqual(back[day]?.key, want.key) + XCTAssertEqual(back[day]?.motion, want.motion) + } + } + + /// A ZERO fold is a cached VALUE, not a missing one — the engine caches a zero from a read that + /// succeeded so unworn gaps stop re-reading their whole stream every pass. A round trip that dropped it + /// would quietly reintroduce exactly the cost the cache exists to remove, on the sparse libraries it is + /// worst for, so the zero day is asserted present rather than merely equal. + func testZeroFoldSurvivesTheRoundTrip() { + let back = StepsMotionCache.deserialize(StepsMotionCache.serialize(Self.vectorEntries)) + XCTAssertNotNil(back["2026-09-02"]) + XCTAssertEqual(back["2026-09-02"]?.motion, 0) + } + + /// Rendering is sorted, so an unchanged cache re-renders byte-identically and the store write coalesces + /// instead of churning a 60-entry payload on every pass. + func testRenderingIsStableAcrossDictionaryOrder() { + var shuffled: [String: (key: String, motion: Double)] = [:] + for (day, e) in Self.vectorEntries.sorted(by: { $0.key > $1.key }) { shuffled[day] = e } + XCTAssertEqual(StepsMotionCache.serialize(shuffled), StepsMotionCache.serialize(Self.vectorEntries)) + } + + /// An older fold's payload must be DISCARDED, not read. `cacheKey` witnesses the inputs only, so a day + /// whose gravity has not moved keys identically across an app update — without the header check the + /// pre-update volume would be served until that day's stream happened to change. + func testOlderFoldVersionIsDiscarded() { + let stale = Self.vector.replacingOccurrences(of: "stepsMotion v1", with: "stepsMotion v0") + XCTAssertTrue(StepsMotionCache.deserialize(stale).isEmpty) + } + + /// Anything that is not a payload this cache wrote yields an empty cache and one re-fold, never a + /// partially-trusted one. + func testUnreadablePayloadsYieldNothing() { + for raw in ["", "stepsMotion", "garbage", "\n", "v1\n2026-09-01\tk\t0"] { + XCTAssertTrue(StepsMotionCache.deserialize(raw).isEmpty, "expected empty for \(raw.debugDescription)") + } + } + + /// A malformed line is skipped and its neighbours survive: a truncated write should cost the days it + /// truncated, not the whole window. + func testMalformedLinesAreSkippedIndividually() { + let raw = """ + stepsMotion v1 + 2026-09-01\tmy-whoop|8640|1757000000\t4659742922898407424 + 2026-09-02\tmissing-a-field + 2026-09-03\tmy-whoop|1|2\tnot-a-number + \tempty-day\t0 + 2026-09-05\t\t0 + 2026-09-06\tmy-whoop|3|4\t4623226492472524800 + """ + let back = StepsMotionCache.deserialize(raw) + XCTAssertEqual(Set(back.keys), ["2026-09-01", "2026-09-06"]) + XCTAssertEqual(back["2026-09-06"]?.motion, 12.5) + } + + /// The writer prunes to the calibration window every pass, so a payload far above it did not come from + /// this cache. Rejecting it whole keeps a hand-edited or corrupt store from being parsed at length. + func testImplausiblyLargePayloadIsRejected() { + var raw = "stepsMotion v1" + for i in 0...513 { raw += "\nday-\(i)\tmy-whoop|1|2\t0" } + XCTAssertTrue(StepsMotionCache.deserialize(raw).isEmpty) + } + + /// Rendering is a FIXPOINT over a payload this build wrote. The engine skips the store write when the + /// rendered cache equals what is stored, so a pass that reused every day must produce the string it + /// read: if this ever stopped holding, every pass of an offload storm would write ~4 KB to say nothing. + func testRenderingIsAFixpoint() { + let once = StepsMotionCache.serialize(Self.vectorEntries) + XCTAssertEqual(StepsMotionCache.serialize(StepsMotionCache.deserialize(once)), once) + } + + /// The other half of that guard: a payload carrying a line this build DROPS must not re-render to + /// itself, so the cleaned version is written back once rather than being re-parsed every launch. + func testDroppedLinesReRenderDifferentlyAndAreRewritten() { + let dirty = Self.vector + "\n2026-09-03\tmy-whoop|1|2\tnot-a-number" + let cleaned = StepsMotionCache.serialize(StepsMotionCache.deserialize(dirty)) + XCTAssertNotEqual(cleaned, dirty) + XCTAssertEqual(cleaned, Self.vector) + } } diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index abddbeb297..987f8cb0a2 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -64,6 +64,18 @@ final class IntelligenceEngine: ObservableObject { /// never crosses `.noopbak`. The engine is a single long-lived instance (AppModel), so this survives the /// storm's back-to-back passes the drain is made of. See `AnalyzeRecentDayCache` (StrandAnalytics). private var dayScanCache: [String: (key: String, scan: DayScan)] = [:] + /// UserDefaults key holding the persisted `stepsMotionCache` payload. Versioned in the key as well as + /// in the payload header so a format change cannot even be read, let alone half-parsed. + private static let stepsMotionCacheDefaultsKey = "analyzeRecent.stepsMotionCache.v1" + /// Whether this process has already seeded `stepsMotionCache` from `stepsMotionCacheDefaultsKey`. The + /// read happens once per process, not once per pass: after the first pass the in-memory cache is at + /// least as fresh as the payload, so re-reading it could only ever put back what we just pruned. + private var stepsMotionCacheLoaded = false + /// The payload currently in UserDefaults, so an unchanged cache does not rewrite it. A pass that reused + /// every day renders the string it read, and the passes that do so are the back-to-back ones an offload + /// storm is made of — writing ~4 KB on each of them to store what is already there is the cost this + /// avoids. Not an assumption about what the store does with an identical value: it simply is not asked. + private var stepsMotionCachePersisted = "" /// Per-day steps-calibration motion folds, `[day: (key, motion)]`, keyed by `StepsMotionCache.cacheKey`. /// In-memory and per-process exactly like `dayScanCache`; see `StepsMotionCache` for why this one needs /// no config signature. Pruned to the calibration window each pass so it cannot grow without bound. @@ -2374,6 +2386,19 @@ final class IntelligenceEngine: ObservableObject { // `deviceId` (a MainActor instance `let`) to a local Sendable `String` so the @Sendable detached // closure captures the VALUE, never `self`, exactly as FIX 1's `ownerFallbackId`. let stepsFallbackId = deviceId + // Seed the fold cache from storage on the first pass of the process. Without this the sixty-day + // fold is re-paid in full after every relaunch — the cache's whole win is a repeat, and a relaunch + // is a repeat the process boundary hid. Nothing pass-global feeds the fold, so a payload written by + // a previous launch is as good as one written by the previous pass; see `StepsMotionCache`. + if !stepsMotionCacheLoaded { + stepsMotionCacheLoaded = true + if let raw = UserDefaults.standard.string(forKey: Self.stepsMotionCacheDefaultsKey) { + stepsMotionCache = StepsMotionCache.deserialize(raw) + // Seed the write guard with what is actually stored. A payload this build renders identically + // then costs no write at all; one carrying a line we dropped is rewritten clean on this pass. + stepsMotionCachePersisted = raw + } + } let inStepsMotionCache = stepsMotionCache let (refStepsByDay, motionByDay, updatedStepsMotionCache, stepsMotionLogLine): ([String: Double], [String: Double], [String: (key: String, motion: Double)], String) = @@ -2444,6 +2469,13 @@ final class IntelligenceEngine: ObservableObject { return (refSteps, motion, motionCacheLocal, motionLog) }.value stepsMotionCache = updatedStepsMotionCache + // Write the pruned cache back, only when it moved. `serialize` renders sorted, so a pass that reused + // every day produces the string already stored and skips the write entirely. + let stepsMotionPayload = StepsMotionCache.serialize(stepsMotionCache) + if stepsMotionPayload != stepsMotionCachePersisted { + stepsMotionCachePersisted = stepsMotionPayload + UserDefaults.standard.set(stepsMotionPayload, forKey: Self.stepsMotionCacheDefaultsKey) + } diagnosticSink?(stepsMotionLogLine, nil) // #1816: persist whether the strap has banked ANY motion in the calibration scan window, so the // Today tile can distinguish "Need N more phone-step days" (motion exists, phone half missing) diff --git a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt index fae10a92d1..bb74f55aeb 100644 --- a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt @@ -99,6 +99,18 @@ object IntelligenceEngine { private var dayScanCache = HashMap() private var dayScanCacheConfigSig = "" + /** Whether this process has already seeded [stepsMotionCache] from the persisted payload. The read + * happens once per process, not once per pass: after the first pass the in-memory cache is at least as + * fresh as the payload, so re-reading it could only ever put back what the pass just pruned. Latched + * only when a store was actually consulted, so a caller that does not wire one cannot silently spend + * the single load on nothing and leave the rest of the process unpersisted. */ + private var stepsMotionCacheLoaded = false + + /** The payload currently in the store, so an unchanged cache does not rewrite it. A pass that reused + * every day renders the string it read, and those passes are the back-to-back ones an offload storm is + * made of. Not an assumption about what the store does with an identical value: it is not asked. */ + private var stepsMotionCachePersisted = "" + /** Per-day steps-calibration motion folds, `day -> (key, motion)`, keyed by [StepsMotionCache.cacheKey]. * In-memory and per-process exactly like [dayScanCache]; see [StepsMotionCache] for why this one needs no * config signature. Pruned to the calibration window each pass so it cannot grow without bound. */ @@ -461,6 +473,13 @@ object IntelligenceEngine { // and passes it down, keeping this layer Context-free. EDWARDS default = byte-identical. effortMethod: StrainScorer.Method = StrainScorer.Method.EDWARDS, dayCycleMode: DayCycleMode = DayCycleMode.SLEEP_ONSET, + // Persisted backing for [stepsMotionCache]. Context-free like the rest of this layer, mirroring + // manualStepCoefficient / persistStepsCalibration above: the Context-aware caller (AppViewModel) + // reads and writes SharedPreferences and passes the accessors down. The defaults are no-ops, so a + // caller that does not wire them keeps exactly today's per-process-only behaviour, which is what + // every existing test relies on. Read/written under [analyzeGate] with the cache they back. + stepsMotionCacheGet: (() -> String?)? = null, + stepsMotionCacheSet: ((String) -> Unit)? = null, ): List = withContext(Dispatchers.Default) { // #1005: time the whole pass so a re-score STORM is visible in the strap log (the trigger lines // record WHY each pass runs; this records how many nights and how long — the CPU cost per run). @@ -469,12 +488,27 @@ object IntelligenceEngine { // [analyzeGate]). The heavy scoring already ran off the caller's thread via withContext above; the // lock is held only for this engine's own passes, never across an unrelated suspension. val scored = analyzeGate.withLock { + // Seed the fold cache from storage on the first pass of the process. Without this the sixty-day + // fold is re-paid in full after every relaunch — the cache's whole win is a repeat, and a + // relaunch is a repeat the process boundary hid. Nothing pass-global feeds the fold, so a + // payload written by a previous launch is as good as one written by the previous pass; see + // [StepsMotionCache]. Inside the lock because it writes the gate-guarded cache. + if (!stepsMotionCacheLoaded && stepsMotionCacheGet != null) { + stepsMotionCacheLoaded = true + val raw = stepsMotionCacheGet() + if (raw != null) { + stepsMotionCache = StepsMotionCache.deserialize(raw) + // Seed the write guard with what is actually stored. A payload this build renders + // identically then costs no write; one carrying a line we dropped is rewritten clean. + stepsMotionCachePersisted = raw + } + } val (out, healed) = analyzeRecentOnCpu(repo, profile, maxDays, importedDeviceId, maxHROverride, nowSeconds, ownerSource, manualStepCoefficient, persistStepsCalibration, baselineEpoch, recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, sleepTraceSink, recoveryTraceSink, stepsTraceSink, universalSink, workoutsTraceSink, hrvTraceSink, deepHrvWindow, spo2CandidateDisplay, effortMethod, dayCycleMode) - if (healed == 0) out + val result = if (healed == 0) out // #899 heal re-pass: the pass above deleted overlapping duplicate sleep sessions AFTER its days // were scored, and the read-side dedup those days consumed had no bank-recency witness (the fresh // detections weren't banked yet), so its survivor can differ from the heal's. ONE bounded re-pass @@ -485,6 +519,17 @@ object IntelligenceEngine { recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, sleepTraceSink, recoveryTraceSink, stepsTraceSink, universalSink, workoutsTraceSink, hrvTraceSink, deepHrvWindow, spo2CandidateDisplay, effortMethod, dayCycleMode).first + // Write the pruned cache back, after the heal re-pass so the payload reflects whichever pass ran + // last, and only when it moved. `serialize` renders sorted, so a pass that reused every day + // produces the string already stored and skips the write entirely. + if (stepsMotionCacheSet != null) { + val payload = StepsMotionCache.serialize(stepsMotionCache) + if (payload != stepsMotionCachePersisted) { + stepsMotionCachePersisted = payload + stepsMotionCacheSet(payload) + } + } + result } diag("re-score: done — scored ${scored.size} night(s) in ${(System.nanoTime() - reScoreStart) / 1_000_000} ms (#1005)") // #2013: what the pass actually CARRIES, beside how many nights it touched. "scored N nights" is diff --git a/android/app/src/main/java/com/noop/analytics/StepsMotionCache.kt b/android/app/src/main/java/com/noop/analytics/StepsMotionCache.kt index 07ed043499..81cbeaa5a0 100644 --- a/android/app/src/main/java/com/noop/analytics/StepsMotionCache.kt +++ b/android/app/src/main/java/com/noop/analytics/StepsMotionCache.kt @@ -16,9 +16,22 @@ package com.noop.analytics * there is no pass-config signature to invalidate against — the value changes exactly when that one day's * gravity changes, and the key below is the whole story. * - * Like the day-scan cache this is in-memory and per-process. It never persists, never crosses the - * `.noopbak` boundary, and a relaunch pays the full fold once. That is deliberate: the cost being closed is - * the repeat within a process, where an offload storm fires passes back to back. + * Unlike the day-scan cache this one PERSISTS across launches, and the paragraph above is why it can. A + * persisted day scan would have to carry `dayScanCacheConfigSig`, which folds in baselines1 and the habitual + * sleep terms - and `sleepConsistency` (1-CV over 28 nights) and `habitualMidsleepSec` (a circular mean) + * shift with ANY night moving, so it would invalidate wholesale on exactly the passes it would need to + * survive. Nothing pass-global reaches this fold, so the payload stays valid while gravity stands still and + * the sixty-day fold is paid once per install instead of once per launch: measured 32.9 s -> 3.3 s on a + * worn 60-day library, which the cold pass otherwise repaid after every relaunch. + * + * It stays a DERIVED cache and nothing else reads it, so the whole failure surface is one re-fold: a payload + * that is missing, unreadable, or written by an older fold is discarded rather than repaired. + * + * It still does NOT cross the `.noopbak` boundary, and must not start: the key is deliberately absent from + * [com.noop.data.BackupSettingsCodec.WHITELIST] and its Apple twin. A restore carries the settings and the + * record store, and this describes neither - it describes gravity rows AS THEY WERE ON ONE DEVICE. Shipping + * it to another device would be the one way to serve a fold whose key no longer witnesses anything, which is + * the failure every other guard here exists to prevent. A restored device simply re-folds once. */ object StepsMotionCache { /** @@ -44,4 +57,63 @@ object StepsMotionCache { */ fun logLine(reused: Int, folded: Int, size: Int): String = "analyzeRecent stepsMotion reused=$reused/${reused + folded} size=$size" + + /** + * The version of the FOLD the persisted values were produced by. Bump on any change to + * [StepsEstimateEngine.dayMotionIntensity] that moves what it returns for the same samples. + * + * In memory this could not exist: a process cannot outlive the binary that filled it, so the fold that + * produced a cached value is always the fold that would reproduce it. A persisted entry outlives its + * build, and [cacheKey] witnesses the INPUTS only - a day whose gravity has not moved keys identically + * across an app update, so without this the old volume would be served until that day's stream happened + * to change. A bump discards every entry and costs one full re-fold, once, which is the cheap side. + */ + const val FOLD_VERSION: Int = 1 + + /** + * Render the cache for storage. Days are emitted in sorted order so an unchanged cache renders to an + * identical payload and the write is a no-op rather than churn. + * + * One line per day, `day\tkey\tmotion`, under a header naming [FOLD_VERSION]. Tab-separated because the + * key itself contains `|`; the motion is written by raw bit pattern so it round-trips exactly and + * locale-free, the same reason the Swift key encodes its skin anchor that way. The bits are rendered + * UNSIGNED so the payload is byte-identical to the Swift twin's for the same cache, which is what makes + * the shared-vector parity test meaningful - the two sides never read each other's storage. + */ + fun serialize(entries: Map>): String { + val sb = StringBuilder(header()) + for (day in entries.keys.sorted()) { + val e = entries[day] ?: continue + sb.append('\n').append(day).append('\t').append(e.first).append('\t') + .append(e.second.toRawBits().toULong().toString()) + } + return sb.toString() + } + + /** + * Parse a stored payload. Returns empty on anything it cannot vouch for - a wrong/absent header (an + * older [FOLD_VERSION] included), a malformed line, or an implausible entry count. Every rejection costs + * one re-fold, so this discards rather than salvages: a half-trusted cache is the one failure mode that + * could feed a stale volume into the calibration fit. + */ + fun deserialize(raw: String): HashMap> { + val out = HashMap>() + val lines = raw.split('\n') + if (lines.firstOrNull() != header()) return out + if (lines.size - 1 > MAX_ENTRIES) return out + for (i in 1 until lines.size) { + val f = lines[i].split('\t') + if (f.size != 3 || f[0].isEmpty() || f[1].isEmpty()) continue + val bits = f[2].toULongOrNull() ?: continue + out[f[0]] = f[1] to Double.fromBits(bits.toLong()) + } + return out + } + + /** Payload header. Carries [FOLD_VERSION] so a fold change invalidates by failing the equality check. */ + private fun header(): String = "stepsMotion v$FOLD_VERSION" + + /** Upper bound on entries a payload may declare. The writer prunes to the calibration window every pass, + * so a payload far above it did not come from this cache and is not worth parsing. */ + private const val MAX_ENTRIES = 512 } diff --git a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt index 22fda60d81..3225ad8576 100644 --- a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt +++ b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt @@ -3202,6 +3202,12 @@ class WhoopBleClient( profileStore.stepsCalibrationConfidence = cal.confidence profileStore.stepsCalibrationManual = cal.manual }, + // Persisted steps-calibration motion folds. The analytics layer is Context-free, so + // the payload is read and written here; without it the sixty-day fold is re-paid in + // full after every relaunch. A derived cache — a missing or unreadable payload just + // re-folds (see StepsMotionCache). + stepsMotionCacheGet = { NoopPrefs.stepsMotionCache(context) }, + stepsMotionCacheSet = { NoopPrefs.setStepsMotionCache(context, it) }, // Manual "Recalibrate baseline" anchor (noop.hrvBaselineEpoch, whole seconds in a // Long). The analytics layer is Context-free, so read it here and thread it down so // the post-backfill scoring pass honours the recalibration too — not just the UI's diff --git a/android/app/src/main/java/com/noop/ui/AppViewModel.kt b/android/app/src/main/java/com/noop/ui/AppViewModel.kt index 249c618903..87acc98abf 100644 --- a/android/app/src/main/java/com/noop/ui/AppViewModel.kt +++ b/android/app/src/main/java/com/noop/ui/AppViewModel.kt @@ -1144,6 +1144,12 @@ class AppViewModel(app: Application) : AndroidViewModel(app) { profileStore.stepsCalibrationConfidence = cal.confidence profileStore.stepsCalibrationManual = cal.manual }, + // Persisted steps-calibration motion folds. The analytics layer is Context-free, so + // the payload is read and written here; without it the sixty-day fold is re-paid in + // full after every relaunch. A derived cache — a missing or unreadable payload just + // re-folds (see StepsMotionCache). + stepsMotionCacheGet = { NoopPrefs.stepsMotionCache(appContext) }, + stepsMotionCacheSet = { NoopPrefs.setStepsMotionCache(appContext, it) }, // Manual "Recalibrate baseline" anchor (Settings → Charge advanced). The analytics // layer is Context-free, so read the epoch (whole seconds, written as a Long by the // button) here and thread it down — foldHistory drops every HRV night before it. @@ -1856,6 +1862,12 @@ class AppViewModel(app: Application) : AndroidViewModel(app) { profileStore.stepsCalibrationConfidence = cal.confidence profileStore.stepsCalibrationManual = cal.manual }, + // Persisted steps-calibration motion folds. The analytics layer is Context-free, so + // the payload is read and written here; without it the sixty-day fold is re-paid in + // full after every relaunch. A derived cache — a missing or unreadable payload just + // re-folds (see StepsMotionCache). + stepsMotionCacheGet = { NoopPrefs.stepsMotionCache(appContext) }, + stepsMotionCacheSet = { NoopPrefs.setStepsMotionCache(appContext, it) }, baselineEpoch = NoopPrefs.of(appContext) .getLong(Baselines.hrvBaselineEpochKey, 0L).toDouble(), recoveryEpoch = NoopPrefs.of(appContext) diff --git a/android/app/src/main/java/com/noop/ui/MainActivity.kt b/android/app/src/main/java/com/noop/ui/MainActivity.kt index 08e56c0b6b..bbbd9dde74 100644 --- a/android/app/src/main/java/com/noop/ui/MainActivity.kt +++ b/android/app/src/main/java/com/noop/ui/MainActivity.kt @@ -349,6 +349,7 @@ object NoopPrefs { const val KEY_HR_BROADCAST = "noop.hrBroadcast" const val KEY_ANALYZE_WATERMARK = "noop.analyzeWatermark" + const val KEY_STEPS_MOTION_CACHE = "noop.stepsMotionCache.v1" /** "Power saving" (#477): when on, NOOP stretches its periodic strap-sync cadence (15 → 45 min) while * the STRAP is discharging at/below [KEY_POWER_SAVING_BATTERY_PCT]. @@ -495,6 +496,17 @@ object NoopPrefs { of(context).edit().putString(KEY_ANALYZE_WATERMARK, fingerprint).apply() } + /** The persisted steps-calibration motion folds (see `StepsMotionCache`). A derived cache, so a missing + * or unreadable payload costs one re-fold and nothing else; versioned in the key as well as in the + * payload header so a format change cannot even be read. Mirrors the Swift + * `analyzeRecent.stepsMotionCache.v1` UserDefaults key. */ + fun stepsMotionCache(context: Context): String? = + of(context).getString(KEY_STEPS_MOTION_CACHE, null) + + fun setStepsMotionCache(context: Context, payload: String) { + of(context).edit().putString(KEY_STEPS_MOTION_CACHE, payload).apply() + } + /** Whether NOOP should hold the strap connection open via a foreground service. Default true. */ fun backgroundConnection(context: Context): Boolean = of(context).getBoolean(KEY_BACKGROUND_CONNECTION, true) diff --git a/android/app/src/test/java/com/noop/analytics/StepsMotionCacheTest.kt b/android/app/src/test/java/com/noop/analytics/StepsMotionCacheTest.kt index 2082357b3c..b5404896f2 100644 --- a/android/app/src/test/java/com/noop/analytics/StepsMotionCacheTest.kt +++ b/android/app/src/test/java/com/noop/analytics/StepsMotionCacheTest.kt @@ -108,4 +108,134 @@ class StepsMotionCacheTest { 1, matches.size) return matches.single() } + + // ---- Persistence ---- + + /** + * The exact payload the cache renders, pinned as a literal. This is the cross-platform contract that + * matters most now the cache is stored: the Swift twin pins the SAME string, so a one-sided change to + * the header, the separators or the number rendering is caught here rather than by a user whose folds + * silently stopped being reused after an update. + */ + private val vector = + "stepsMotion v1\n" + + "2026-09-01\tmy-whoop|8640|1757000000\t4659742922898407424\n" + + "2026-09-02\tmy-whoop|0|0\t0" + + private val vectorEntries = mapOf( + "2026-09-01" to ("my-whoop|8640|1757000000" to 3421.75), + "2026-09-02" to ("my-whoop|0|0" to 0.0), + ) + + @Test + fun serializeRendersThePinnedPayload() { + assertEquals(vector, StepsMotionCache.serialize(vectorEntries)) + } + + @Test + fun roundTripPreservesKeysAndVolumes() { + val back = StepsMotionCache.deserialize(StepsMotionCache.serialize(vectorEntries)) + assertEquals(2, back.size) + for ((day, want) in vectorEntries) { + assertEquals(want.first, back[day]?.first) + assertEquals(want.second, back[day]?.second!!, 0.0) + } + } + + /** + * A ZERO fold is a cached VALUE, not a missing one — the engine caches a zero from a read that + * succeeded so unworn gaps stop re-reading their whole stream every pass. A round trip that dropped it + * would quietly reintroduce exactly the cost the cache exists to remove, on the sparse libraries it is + * worst for, so the zero day is asserted present rather than merely equal. + */ + @Test + fun zeroFoldSurvivesTheRoundTrip() { + val back = StepsMotionCache.deserialize(StepsMotionCache.serialize(vectorEntries)) + assertTrue(back.containsKey("2026-09-02")) + assertEquals(0.0, back["2026-09-02"]?.second!!, 0.0) + } + + /** + * Rendering is sorted, so an unchanged cache re-renders byte-identically and the store write coalesces + * instead of churning a 60-entry payload on every pass. + */ + @Test + fun renderingIsStableAcrossMapOrder() { + val shuffled = LinkedHashMap>() + for (day in vectorEntries.keys.sortedDescending()) shuffled[day] = vectorEntries.getValue(day) + assertEquals(StepsMotionCache.serialize(vectorEntries), StepsMotionCache.serialize(shuffled)) + } + + /** + * An older fold's payload must be DISCARDED, not read. [StepsMotionCache.cacheKey] witnesses the inputs + * only, so a day whose gravity has not moved keys identically across an app update — without the header + * check the pre-update volume would be served until that day's stream happened to change. + */ + @Test + fun olderFoldVersionIsDiscarded() { + assertTrue(StepsMotionCache.deserialize(vector.replace("stepsMotion v1", "stepsMotion v0")).isEmpty()) + } + + /** + * Anything that is not a payload this cache wrote yields an empty cache and one re-fold, never a + * partially-trusted one. + */ + @Test + fun unreadablePayloadsYieldNothing() { + for (raw in listOf("", "stepsMotion", "garbage", "\n", "v1\n2026-09-01\tk\t0")) { + assertTrue("expected empty for '$raw'", StepsMotionCache.deserialize(raw).isEmpty()) + } + } + + /** + * A malformed line is skipped and its neighbours survive: a truncated write should cost the days it + * truncated, not the whole window. + */ + @Test + fun malformedLinesAreSkippedIndividually() { + val raw = "stepsMotion v1\n" + + "2026-09-01\tmy-whoop|8640|1757000000\t4659742922898407424\n" + + "2026-09-02\tmissing-a-field\n" + + "2026-09-03\tmy-whoop|1|2\tnot-a-number\n" + + "\tempty-day\t0\n" + + "2026-09-05\t\t0\n" + + "2026-09-06\tmy-whoop|3|4\t4623226492472524800" + val back = StepsMotionCache.deserialize(raw) + assertEquals(setOf("2026-09-01", "2026-09-06"), back.keys) + assertEquals(12.5, back["2026-09-06"]?.second!!, 0.0) + } + + /** + * The writer prunes to the calibration window every pass, so a payload far above it did not come from + * this cache. Rejecting it whole keeps a hand-edited or corrupt store from being parsed at length. + */ + @Test + fun implausiblyLargePayloadIsRejected() { + val sb = StringBuilder("stepsMotion v1") + for (i in 0..513) sb.append("\nday-").append(i).append("\tmy-whoop|1|2\t0") + assertTrue(StepsMotionCache.deserialize(sb.toString()).isEmpty()) + } + + /** + * Rendering is a FIXPOINT over a payload this build wrote. The engine skips the store write when the + * rendered cache equals what is stored, so a pass that reused every day must produce the string it + * read: if this ever stopped holding, every pass of an offload storm would write ~4 KB to say nothing. + */ + @Test + fun renderingIsAFixpoint() { + val once = StepsMotionCache.serialize(vectorEntries) + assertEquals(once, StepsMotionCache.serialize(StepsMotionCache.deserialize(once))) + } + + /** + * The other half of that guard: a payload carrying a line this build DROPS must not re-render to + * itself, so the cleaned version is written back once rather than being re-parsed every launch. + */ + @Test + fun droppedLinesReRenderDifferentlyAndAreRewritten() { + val dirty = vector + "\n2026-09-03\tmy-whoop|1|2\tnot-a-number" + val cleaned = StepsMotionCache.serialize(StepsMotionCache.deserialize(dirty)) + assertNotEquals(dirty, cleaned) + assertEquals(vector, cleaned) + } }