diff --git a/.gitignore b/.gitignore index dcc3af7e30..202bbba8b0 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ android/app/release/ # ── Misc / secrets (never commit) ─────────────────────────────── *.log .env +# Oura BYO-app OAuth credentials (never committed) +Strand/Oura/OuraSecrets.xcconfig # Deployment credentials (PAT / signing / endpoints) — never commit. `.env` above # won't match `deploy.env`, so list it (and any `deploy.env.`) explicitly. deploy.env @@ -92,6 +94,12 @@ export.zip __pycache__/ *.pyc +# ── Rust (liters dependency) ──────────────────────────────────── +# Build output only. Rust/Cargo.lock IS tracked: it pins the exact liters +# commit this tree builds against, which a branch-tracking git dependency +# otherwise leaves floating. +Rust/target/ + # Local release artifacts (zips/ipas built for upload; never tracked) dist/ err.txt @@ -102,6 +110,12 @@ Strand/Oura/OuraSecrets.xcconfig # Personal bundle id prefix (BYO; never committed) Config/BundleIdSecrets.xcconfig +# liters linkage, written by Rust/build-ios.sh from the xcframework it just produced. Untracked +# because it names slice directories under Rust/target/, which is itself untracked — committing it +# would point every other checkout at an archive that is not there. Config/Liters.xcconfig is the +# tracked wrapper that optionally includes it. +Config/LitersLocal.xcconfig + # (#979) The linux-capture workbench moved from `tools/` to `Tools/` when the two case-colliding # trees were merged. On a case-SENSITIVE filesystem any capture a contributor already had stays # behind at the old path, where that tool's own .gitignore no longer reaches it — and those files diff --git a/Config/BundleId.xcconfig b/Config/BundleId.xcconfig index e63944ad71..0954d35a2e 100644 --- a/Config/BundleId.xcconfig +++ b/Config/BundleId.xcconfig @@ -16,3 +16,10 @@ BUNDLE_ID_PREFIX = com.noopapp // to stop Xcode from asking you to reselect a team after every `xcodegen generate`. DEVELOPMENT_TEAM = #include? "BundleIdSecrets.xcconfig" + +// The liters (page-replication) linkage seam. Tracked, and inert until Rust/build-ios.sh has been +// run: it defines LITERS_LIB_DIR / LITERS_HEADER_DIR / LITERS_LDFLAGS as empty and then optionally +// includes the untracked Config/LitersLocal.xcconfig that build-ios.sh writes. Included from HERE +// rather than from a target xcconfig because project.yml applies this file project-wide, which is +// the only level every target (macOS Strand, StrandTests, NOOPiOS) already shares. +#include "Liters.xcconfig" diff --git a/Config/Liters.xcconfig b/Config/Liters.xcconfig new file mode 100644 index 0000000000..414522c9de --- /dev/null +++ b/Config/Liters.xcconfig @@ -0,0 +1,37 @@ +// Tracked wrapper for the liters (page-replication) linkage, so a clean checkout that has never run +// Rust/build-ios.sh still generates and builds. Included from Config/BundleId.xcconfig, which +// project.yml applies project-wide, so every target inherits these. +// +// Liters.xcframework is a BUILD ARTIFACT, not a tracked file: the device slice alone is ~83 MB, and +// three slices plus a macOS slice is far past what belongs in git. It is produced by +// `Rust/build-ios.sh`, which ALSO writes the untracked Config/LitersLocal.xcconfig included at the +// bottom of this file. That is deliberate — the search paths below name specific xcframework slice +// directories, and having the build script emit them from the layout `xcodebuild -create-xcframework` +// actually produced is the only way they cannot drift from it. +// +// The three variables default to EMPTY here. project.yml appends them unconditionally +// (LIBRARY_SEARCH_PATHS / SWIFT_INCLUDE_PATHS / OTHER_LDFLAGS), so with no local file the appends +// expand to nothing: no -l flag, no search path, and a default build links exactly what it links +// today. Absence of the xcframework is therefore not a build error for anyone who does not want it. +// +// LITERS_LDFLAGS is what actually pulls the archive in. It is kept separate from the search path so +// that "the paths are configured" and "the library is linked" stay two decisions: a build with the +// paths set but LITERS_LDFLAGS empty compiles the generated bindings' module import and fails at +// link, which is a clearer error than a missing module. +LITERS_LIB_DIR = +LITERS_HEADER_DIR = +LITERS_LDFLAGS = + +// Set by build-ios.sh alongside the paths above. Gates every line of Swift that touches liters +// (Strand/CloudSync/Generated/, and the round-trip tests), so a checkout without the xcframework +// compiles none of it rather than failing to resolve `import liters_ffiFFI`: +// +// OTHER_SWIFT_FLAGS = $(inherited) -D LITERS +// +// NOT SWIFT_ACTIVE_COMPILATION_CONDITIONS, even though that is the idiomatic setting and what +// OuraSecrets.xcconfig uses for OURA_CLOUD_IMPORT / CLOUD_SYNC. Those work because OuraConfig.xcconfig +// is a TARGET-level xcconfig, which outranks the project build settings; this file is included from +// the PROJECT-level xcconfig, which xcodegen's own `SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG` +// (written into the project's Debug config, with no $(inherited)) sits above and silently overwrites. +// Verified with `xcodebuild -showBuildSettings`: the LITERS entry vanishes, DEBUG survives. +#include? "LitersLocal.xcconfig" diff --git a/Packages/NoopLocalAccess/Sources/NoopLocalAccessCore/LocalAccessCore.swift b/Packages/NoopLocalAccess/Sources/NoopLocalAccessCore/LocalAccessCore.swift index 632e181a28..ad4228a505 100644 --- a/Packages/NoopLocalAccess/Sources/NoopLocalAccessCore/LocalAccessCore.swift +++ b/Packages/NoopLocalAccess/Sources/NoopLocalAccessCore/LocalAccessCore.swift @@ -333,6 +333,11 @@ public final class ReadonlyNoopStore { "hrSample", "rrInterval", "event", "battery", "spo2Sample", "skinTempSample", "respSample", "gravitySample", "ppgHrSample", "stepSample", "sleepStateSample", "ppgWaveformSample", "rawImuSample", "v18AuxSample", + // v34: durable `@82` SpO2 percentages. Small (~100k rows/year) but UNBOUNDED — it is the + // one table here deliberately exempt from every prune — so it belongs in the readout for + // exactly the reason the comment above gives: growth that nothing prunes still has to be + // growth somebody can see. + "spo2PctSample", ] var decodedRows = 0 for table in decodedTables where tableNames.contains(table) { diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift index 2532f06584..90c26d2407 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift @@ -279,6 +279,14 @@ public enum AnalyticsEngine { // calibrated blood-oxygen % (that needs WHOOP's proprietary curve). // Default empty keeps pure-function callers/tests + non-4.0 nights nil. spo2: [SpO2Sample] = [], + // Durable 5/MG `@82` SpO2 percentages for the night window (v34). The + // ramp-trimmed nightly MEDIAN is banked on `DailyMetric.spo2Pct` — the + // strap's OWN computed percentage, not a curve NOOP invented. Distinct + // from `spo2:` above, which is the WHOOP 4.0 v24 raw red/IR ADC pair and + // feeds `spo2Red`/`spo2Ir`; the two device generations report different + // physical quantities and neither substitutes for the other. Default + // empty keeps pure-function callers/tests + every non-5/MG night nil. + spo2PctSamples: [Spo2PctSample] = [], profile: UserProfile, baselines: ProfileBaselines = ProfileBaselines(), maxHROverride: Double? = nil, @@ -336,6 +344,14 @@ public enum AnalyticsEngine { // caller/test byte-identical; IntelligenceEngine threads // `PuffinExperiment.motionAwareWakeEnabled`. useMotionAwareWake: Bool = false, + // Wake-gap bridging (experimental, default OFF — see the section + // comment above `SleepStager.wakeBridgeMaxMin`). When true, + // detection assembles one session per sleep period across ≤ 45-min + // wake gaps, so a brief get-up stays inside one session as staged + // wake. Default false keeps every pure-function caller/test + // byte-identical; IntelligenceEngine threads + // `PuffinExperiment.wakeBridgeEnabled`. + useWakeBridge: Bool = false, // Sleep PROVENANCE for the per-day sleep trace (CAPTURE-C / #799). The // measured BLE path is `.measured` (the default); the caller passes // `.imported(...)` when a previously-imported sleep row WON the daily merge, @@ -377,6 +393,7 @@ public enum AnalyticsEngine { tzOffsetSeconds: tzOffsetSeconds, wristOff: wristOff, bandSleepState: bandSleepState, useSleepStagerV2: useSleepStagerV2, + wakeBridge: useWakeBridge, traceSink: traceSink) // Motion-aware wake refinement (#364 follow-up) runs AFTER V1/V2 staging, over every detected // session (naps included — the same eligibility gates apply). `steps` is the SAME calendar-day/ @@ -538,6 +555,18 @@ public enum AnalyticsEngine { return weight > 0 ? total / weight : nil }() + // Daily SDNN (ms) = the 5-min SDNN INDEX (Task Force) over the in-bed R-R across matched sessions — + // the mean of per-5-min-segment SDNN, the BROAD autonomic-variability metric (both branches) and the + // slow twin of the vagal RMSSD `avgHRVDaily` above. The index (not a single whole-night SD) is used + // deliberately: whole-night SD is dominated by the slow HR drift across sleep stages and reads 2-3× + // high, which would mislabel Apple Health (its SDNN samples are short-window) and make any cross-check + // against a watch meaningless. The 5-min index is window-comparable to those. nil when no segment has + // enough clean beats (HRVAnalyzer's own gate). Keeps the R-R timestamps (segmentation needs them). + let avgSDNNDaily: Double? = { + let inBed = rr.filter { r in matched.contains { r.ts >= $0.start && r.ts < $0.end } } + return inBed.isEmpty ? nil : HRVAnalyzer.sdnnIndex(inBed, segmentSec: 300) + }() + // ── HRV & Autonomic nightly trace (#141) ────────────────────────────── // Per-5-min-window RMSSD tagged by the sleep stage at its center, then a night summary comparing // NOOP's whole-night mean (what it reports) against a deep-only mean and a WHOOP-style @@ -608,6 +637,14 @@ public enum AnalyticsEngine { // as-is for the Health "Raw SpO₂" tile — NOT a calibrated blood-oxygen %. (#93) let nightlySpo2Raw = nightlySpo2RawMeans(matched, spo2: spo2) + // ── SpO2 percentage (5/MG @82, durable) ─────────────────────────────── + // Ramp-trimmed nightly median of the strap's OWN computed SpO2 %, over the detected in-bed spans. + // Nil on a WHOOP 4.0, on any night with no in-band reading, and on every window offloaded before + // v34. Unlike the raw red/IR means above this IS a percentage — it is the number the strap + // computed, banked verbatim per-second and aggregated here, not a calibration NOOP applied to an + // ADC. See `nightlySpo2Pct` for the run/ramp policy and why the statistic is a median. + let nightlySpo2PctVal = nightlySpo2Pct(matched, samples: spo2PctSamples) + // ── Recovery / "Charge" ─────────────────────────────────────────────── var recovery: Double? = nil // Ordered "why is Charge what it is" rows, built from the SAME inputs as the score @@ -721,13 +758,26 @@ public enum AnalyticsEngine { recovery: recovery, strain: strain, exerciseCount: workouts.count, - spo2Pct: nil, + // v34: was unconditionally nil here — the on-device engine had no percentage to bank, so the + // Blood Oxygen card read "No Data" on every WHOOP night while the same field rendered fine for + // imported Oura rows. It now carries the 5/MG's own `@82` median when the night has one, and + // stays nil otherwise (WHOOP 4.0, pre-v34 windows, no in-band samples) — an absent reading + // must stay absent rather than become a fabricated number. + // + // NOTE FOR ANYONE CHANGING THIS: `dailyMetric.spo2Pct` is not display-only. `HealthKitBridge` + // writes it to `HKQuantityTypeIdentifier.oxygenSaturation`, so a value banked here leaves the + // app and lands in the user's Apple Health blood-oxygen history alongside clinical readings. + // That is the right home for a device-computed SpO2 % and it is exactly where the Oura import + // already puts its own, but it does mean this field's bar is "the device measured this", + // never "this is our best guess". + spo2Pct: nightlySpo2PctVal?.pct, skinTempDevC: skinTempDevC, respRateBpm: respRateDaily, steps: stepsTotal, activeKcalEst: activeKcalEst, spo2Red: nightlySpo2Raw?.red, - spo2Ir: nightlySpo2Raw?.ir) + spo2Ir: nightlySpo2Raw?.ir, + avgSdnn: avgSDNNDaily) _ = sleepStart; _ = sleepEnd // available for callers wiring sleep_start/end columns // ── Cache rows ──────────────────────────────────────────────────────── @@ -829,6 +879,46 @@ public enum AnalyticsEngine { public static let wRestorative: Double = 0.20 public static let wConsistency: Double = 0.10 + /// Minimum trailing nights before a personal sleep-need estimate is trusted; below this the + /// population default is used (cold-start honesty — never learn a need from a few nights). + public static let minNeedNights: Int = 7 + /// Hard cap on personalized need (h): beyond typical adult need even for genuine long sleepers. + public static let maxNeedHours: Double = 9.5 + + /// Population TARGET nightly sleep need (hours) for an age (NSF/AASM recommended-range + /// midpoint). Used as a FLOOR: the personal estimate can only adjust it UP for genuine long + /// sleepers, NEVER below the population target — so a chronic under-sleeper's need can't drift + /// toward their own deficit (which would falsely erase their sleep debt and read a short night + /// as "Strong"). Deliberately the target, not the minimum: flooring at the low end of the + /// range would understate a sleep-deprived (vs genuinely short-sleeping) user's deficit. + public static func populationNeedFloorHours(age: Int?) -> Double { + guard let age, age > 0 else { return 8.0 } // unknown → adult target + switch age { + case ..<18: return 9.0 // children/teens need more (NSF 8–12; target ~9) + default: return 8.0 // adults (18–64) & older (65+): NSF 7–9, target ~8 (the app default) + } + } + + /// Personalized nightly sleep need (hours), population-ANCHORED and age-floored (T1). The + /// personal component is the user's UPPER-QUARTILE nightly duration over the trailing window — + /// what they sleep on their less-restricted nights, NOT their average (which a chronic deficit + /// drags down) — floored at the age-appropriate population TARGET and capped at `maxNeedHours`. + /// So it only ever ADJUSTS UP for genuine long sleepers, never below the target. Fewer + /// than `minNeedNights` scorable nights → the population default (cold-start). Zero/negative + /// entries (no-data days) are dropped and do not count toward the minimum. + public static func personalizedNeedHours(nightlyHours: [Double], age: Int?) -> Double { + let floor = populationNeedFloorHours(age: age) + let xs = nightlyHours.filter { $0 > 0 }.sorted() + guard xs.count >= minNeedNights else { + return min(max(defaultNeedHours, floor), maxNeedHours) + } + // Linear-interpolated 75th percentile — the "unrestricted" nights. + let pos = 0.75 * Double(xs.count - 1) + let lo = Int(pos), hi = min(lo + 1, xs.count - 1) + let q = xs[lo] + (pos - Double(lo)) * (xs[hi] - xs[lo]) + return min(max(q, floor), maxNeedHours) + } + /// Build the composite. `tstSeconds` = total sleep time, `restorativeSeconds` = deep+REM /// seconds, `deepSeconds` = deep-stage seconds (nil → no deep-adequacy adjustment, pooled /// behaviour). Returns a value in [0,100]. @@ -975,6 +1065,88 @@ public enum AnalyticsEngine { return (mean: sum / kept, samples: kept) } + /// Nightly SpO2 percentage from the DURABLE `@82` stream (`spo2PctSample`, v34) over the detected + /// in-bed `sessions` — the value banked on `DailyMetric.spo2Pct` for a 5/MG night — with the sample + /// count that survived the ramp trim. Nil when no reading fell inside any span. + /// + /// HOW THIS DIFFERS FROM `nightlySpo2CandidateMean`. That one is a naive mean over the raw candidate + /// byte read out of the capped aux table, built as a hand-checkable diagnostic. This is the scored + /// statistic, over the durable stream, and it corrects the two things that make the naive mean wrong. + /// + /// THE ACQUISITION RAMP. The strap does not sample SpO2 continuously: it takes a RUN of ~30 one-second + /// samples roughly once per ~1,200 s while asleep. The first samples of each run are the optical front + /// end settling, and they read LOW — so a mean over every in-band sample is biased downward by a + /// contamination that is entirely one-sided, and the bias grows with how much of the night is ramp + /// (i.e. it is worse on nights with more, shorter runs — precisely when you would least suspect it). + /// Runs are separated here by a gap of more than `spo2RunGapSeconds`; at 1 Hz within a run and ~1,200 s + /// between runs, that boundary is not a close call. The leading `spo2RampSamples` of each run are + /// dropped — but never more than HALF a run, so a short run contributes its settled tail instead of + /// vanishing and silently reweighting the night toward whichever runs happened to be long. + /// + /// WHY THE MEDIAN AND NOT A TRIMMED MEAN. Three reasons, in order of weight: + /// 1. It is the statistic the cloud reader already reports (the per-night 94-97 medians every + /// cross-check against this data has been quoted against). A phone that shipped a trimmed mean + /// would disagree with the server on the same rows for no reason anyone could see, and every + /// future comparison would have to carry a caveat. + /// 2. Ramp removal is a heuristic, not a proof. A partially-settled sample just past the cut, or a + /// motion artifact mid-run, is residual one-sided contamination that a mean absorbs in full and a + /// median very largely ignores. + /// 3. The in-band values live in a 31-wide integer band with a hard mode at 97, so a median is + /// stable and interpretable here in a way it would not be on a sparse continuous quantity. + /// The trim and the median are complementary, not redundant: the trim removes a KNOWN, structured, + /// large block of low readings that would move a median too, and the median absorbs what is left. + /// + /// READ-TIME POLICY, DELIBERATELY. None of this is applied on the way into the store — `spo2PctSample` + /// holds raw in-band bytes. These rules have already changed once on the cloud side, and they will + /// change again if a run's shape turns out to differ by firmware; keeping them here means that costs a + /// re-score, not a re-capture of data the strap has long since trimmed. + public static func nightlySpo2Pct(_ sessions: [SleepSession], + samples: [Spo2PctSample]) -> (pct: Double, samples: Int)? { + guard !sessions.isEmpty, !samples.isEmpty else { return nil } + // In-bed first, so run boundaries are computed over the samples that will actually be used. Note + // what that does and does not mean: a session edge splits a run only when the stretch it excludes + // is itself longer than `spo2RunGapSeconds` — a brief out-of-bed moment mid-run leaves the run + // intact and singly-trimmed. Gap size is the ONLY run signal; session boundaries are not a second + // one. That is deliberate, because the ramp is a property of the sensor restarting, and the sensor + // does not restart because the sleep detector drew a line. + let inBed = samples + .filter { s in sessions.contains(where: { $0.start <= s.ts && s.ts <= $0.end }) } + .sorted { $0.ts < $1.ts } + guard !inBed.isEmpty else { return nil } + + var kept: [Int] = [] + var runStart = 0 + func flushRun(_ range: Range) { + let run = inBed[range] + // Never trim more than half a run: a 4-sample run keeps its last 2 rather than disappearing. + let drop = min(spo2RampSamples, run.count / 2) + kept.append(contentsOf: run.dropFirst(drop).map(\.pct)) + } + for i in 1.. spo2RunGapSeconds { + flushRun(runStart.. STALE_DAYS nights + /// Seeded (nValid ≥ MIN_NIGHTS_SEED) but not updated for > STALE_DAYS nights — so NOT `usable`, and + /// consumers fall back to population norms. This comment used to read "usable but no update for > + /// STALE_DAYS nights", which collides with the `usable` property below (it excludes `.stale`) and + /// reads as a code/comment contradiction. The CODE is the correct half: a personal baseline nobody has + /// confirmed in over two weeks is exactly one not to trust. Pinned by + /// `VitalBandsTests.testStaleBaselineFallsBackToPopulation` on both platforms. + /// (F1, docs/bugs/2026-07-15-strap-battery-backfill-observability.md) + case stale } /// Immutable snapshot of a personal baseline for one metric after N nights. @@ -65,7 +72,10 @@ public struct BaselineState: Equatable, Sendable { /// True iff fully trusted (not calibrating or stale). public var trusted: Bool { status == .trusted } - /// True iff at least provisionally usable (nValid ≥ MIN_NIGHTS_SEED). + /// True iff this personal baseline should be trusted at all. NOT simply "nValid ≥ MIN_NIGHTS_SEED": + /// `.stale` also clears that bar and is still excluded, because it has gone > STALE_DAYS nights with + /// no confirming value. Callers that get `false` here fall back to population norms rather than score + /// off a baseline that may no longer describe this person. public var usable: Bool { status == .provisional || status == .trusted } } @@ -166,14 +176,53 @@ public enum Baselines { halfLifeB: 14.0, halfLifeS: 21.0), "strain": MetricCfg(minVal: 0.0, maxVal: 100.0, floorSpread: 5.0, halfLifeB: 14.0, halfLifeS: 21.0), + + // Daytime/waking-hours configs (DaytimeStress baseline-relative mode, added alongside + // the day-relative default). Distinct from "resting_hr"/"hrv" above, which each fold ONE + // NIGHTLY value (sleep). These fold ONE DAYTIME aggregate per day into a cross-day + // PERSONAL baseline, so a caller can hand the resulting BaselineState to + // `DaytimeStress.analyze(mode: .baselineRelative(hr:rmssd:))` and z-score each waking + // hour against "how MY days usually run" rather than "how today's own calm hours ran". + // + // VALIDATED (26-day Oura-reference correlation, HR-only, r≈0.6): the per-day value to + // fold for "daytime_hr" is that day's 10th-percentile daytime HR (~65 bpm pooled across + // the reference set) — NOT the day-relative calmReference's 25th-percentile quartile. + // The "high stress" cutoff itself is a validated fixed bpm margin over this baseline, + // NOT this config's floorSpread — see DaytimeStress.baselineRelativeHighMarginBPM. + // TUNING SEAM: minVal/maxVal/half-life below (and all of "daytime_rmssd" — RMSSD wasn't + // part of the validated HR-only comparison) are a documented first pass, refinable once + // an HR+HRV comparison exists. floorSpread is wider than the nightly "hrv" config for + // RMSSD: daytime RMSSD carries more incidental noise (posture changes, talking, movement + // between "calm" hours) than overnight recumbent HRV. + "daytime_hr": MetricCfg(minVal: 35.0, maxVal: 160.0, floorSpread: 3.0, + halfLifeB: 14.0, halfLifeS: 21.0), + "daytime_rmssd": MetricCfg(minVal: 5.0, maxVal: 250.0, floorSpread: 7.0, + halfLifeB: 14.0, halfLifeS: 21.0), + + // Readiness HRV baseline, folded in the LOG domain with hard-outlier rejection OFF (RD2 · the + // window-fold spine mode — see Baselines.update's `rejectHardOutliers`). ReadinessEngine + // z-scores lnRMSSD (RD1: HRV is right-skewed), so these bounds/floor are in LN(ms) units — NOT + // the raw-ms "hrv" config above (which the RecoveryScorer folds linearly, untouched). + // minVal/maxVal = ln(8)/ln(250), the plausible HRV band in ln. floorSpread 0.08 (ln) sits just + // below a real wearer's own ln-HRV night-to-night spread (~0.10, measured on real WHOOP nights), + // so personal spread drives the z while an ultra-stable baseline is floored against saturation. + "readiness_hrv_ln": MetricCfg(minVal: 2.079, maxVal: 5.521, floorSpread: 0.08, + halfLifeB: 14.0, halfLifeS: 21.0), ] /// Convenience accessors for the standard configs. public static var hrvCfg: MetricCfg { metricCfg["hrv"]! } public static var restingHRCfg: MetricCfg { metricCfg["resting_hr"]! } public static var respCfg: MetricCfg { metricCfg["resp"]! } + /// Readiness HRV baseline config — folded in the LOG domain (ln ms) with hard-outlier rejection + /// OFF. See the `readiness_hrv_ln` comment above; distinct from the raw-ms `hrvCfg`. + public static var readinessHRVLnCfg: MetricCfg { metricCfg["readiness_hrv_ln"]! } /// Baseline config for the RecoveryScorer Activity-Balance / previous-day-Effort term. public static var strainCfg: MetricCfg { metricCfg["strain"]! } + /// Personal daytime/waking HR baseline config — see the `daytime_hr` comment above. + public static var daytimeHRCfg: MetricCfg { metricCfg["daytime_hr"]! } + /// Personal daytime/waking RMSSD baseline config — see the `daytime_rmssd` comment above. + public static var daytimeRMSSDCfg: MetricCfg { metricCfg["daytime_rmssd"]! } /// Convert a half-life in nights to an EWMA smoothing factor. static func lambda(halfLife: Double) -> Double { @@ -225,7 +274,8 @@ public enum Baselines { /// - `value == nil` or out-of-range: skip-and-hold (carry forward). /// - hard outlier (> HARD_OUTLIER_K × spread): seen but not folded. /// - otherwise: Winsorized EWMA center + EWMA-abs-dev spread update. - public static func update(_ state: BaselineState?, value: Double?, cfg: MetricCfg) -> BaselineState { + public static func update(_ state: BaselineState?, value: Double?, cfg: MetricCfg, + rejectHardOutliers: Bool = true) -> BaselineState { let lb = lambda(halfLife: cfg.halfLifeB) let ls = lambda(halfLife: cfg.halfLifeS) @@ -266,7 +316,17 @@ public enum Baselines { // Suspending this during early life is the core anti-anchoring fix — a high seed with a // floor-tight spread would otherwise reject the user's real, lower readings as "outliers" // (a true 54ms vs an anchored ~90ms baseline is >5× the floor spread). - if state.nValid >= minNightsSeed && !isYoung { + // + // `rejectHardOutliers` (default true) can turn this OFF for a TRAILING-WINDOW re-fold — where + // the same 30-night window is re-folded every call and a *recent sustained* shift lands at the + // window's end (past the young grace period), so rejection would discard a real new normal as a + // string of "outliers" (readiness's device-swap / supplement-onset failure mode). With it off, + // the Winsorization below STILL damps a single freak night (clamped to ±winsorK·spread rather + // than folded raw) — but a sustained shift is followed instead of rejected, because each night + // nudges the center and widens the spread until the new level is no longer an outlier. This is + // exactly the incremental-fold (reject on, RecoveryScorer) vs window-fold (reject off, Readiness) + // distinction; validated on real HRV history (see ReadinessEngine's RD2 note). + if rejectHardOutliers && state.nValid >= minNightsSeed && !isYoung { let dev = abs(value - state.baseline) if dev > hardOutlierK * state.spread { return BaselineState(baseline: state.baseline, spread: state.spread, @@ -304,9 +364,10 @@ public enum Baselines { /// Replay an ordered sequence of nightly values (oldest first) to build state. /// `nil` entries are treated as missing nights (skip-and-hold). - public static func foldHistory(_ values: [Double?], cfg: MetricCfg) -> BaselineState { + public static func foldHistory(_ values: [Double?], cfg: MetricCfg, + rejectHardOutliers: Bool = true) -> BaselineState { var state: BaselineState? = nil - for v in values { state = update(state, value: v, cfg: cfg) } + for v in values { state = update(state, value: v, cfg: cfg, rejectHardOutliers: rejectHardOutliers) } if let s = state { return s } let seed = (cfg.minVal + cfg.maxVal) / 2.0 return BaselineState(baseline: seed, spread: cfg.floorSpread, nValid: 0, @@ -473,12 +534,20 @@ public enum Baselines { // MARK: - Deviation + /// Convert a state's EWMA-abs-dev spread to an approximate Gaussian σ: 1.253 × spread, + /// floored so a caller never divides by ~0. 1.253 is E[|X−μ|] = σ·√(2/π) inverted + /// (E[|X−μ|] ≈ σ/1.253 for a Gaussian). Shared by `deviation()` below and by any other + /// z-scoring caller (e.g. `DaytimeStress`'s baseline-relative mode) so the conversion has + /// exactly one definition. + public static func sigma(_ state: BaselineState) -> Double { + max(1.253 * state.spread, 1e-9) + } + /// Compute z / delta / ratio / in-normal-range for a value vs a baseline. - /// z uses (value − baseline) / (1.253 × spread); 1.253 converts EWMA-abs-dev - /// to an approximate Gaussian σ (E[|X−μ|] = σ·√(2/π) ≈ σ/1.253). + /// z uses (value − baseline) / sigma(state); see `sigma(_:)` for the 1.253 conversion. public static func deviation(_ value: Double, state: BaselineState) -> Deviation { - let sigma = max(1.253 * state.spread, 1e-9) - let z = (value - state.baseline) / sigma + let sig = sigma(state) + let z = (value - state.baseline) / sig let delta = value - state.baseline let ratio = state.baseline != 0 ? (value / state.baseline - 1.0) : 0.0 return Deviation(z: z, delta: delta, ratio: ratio, inNormalRange: abs(z) <= 1.0) diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/ChargeDrivers.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/ChargeDrivers.swift index e79910aae3..acce836edb 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/ChargeDrivers.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/ChargeDrivers.swift @@ -154,6 +154,14 @@ extension RecoveryScorer { var drivers: [ChargeDriver] = [] + // Was the low-HRV penalty eased by the parasympathetic-saturation guard on THIS night? + // Recompute the SAME HRV / resting-HR coupling the score used so the HRV verdict can honestly + // explain a lighter-than-usual penalty (its deltaPoints already reflect the easing, since + // points(...) routes through recovery(...), which applies the guard). + let hrvZFull = zScore(hrv, mean: hrvBaseline.baseline, spread: hrvBaseline.spread) + let rhrZFull: Double? = rhrBaseline.map { zScore($0.baseline, mean: rhr, spread: $0.spread) } + let hrvSaturationEased = parasympatheticSaturation(hrvZ: hrvZFull, rhrZ: rhrZFull).active + // ── HRV (dominant driver; always present once the score exists) ────────── // Higher HRV vs baseline supports recovery. Neutral = HRV at the baseline mean. drivers.append(ChargeDriver( @@ -164,7 +172,8 @@ extension RecoveryScorer { skinTempDev: skinTempDev)), valueText: "\(Int(hrv.rounded())) ms", baselineText: "\(Int(hrvBaseline.baseline.rounded())) ms baseline", - verdict: hrvVerdict(value: hrv, baseline: hrvBaseline.baseline))) + verdict: hrvVerdict(value: hrv, baseline: hrvBaseline.baseline, + saturationEased: hrvSaturationEased))) // ── Resting HR (lower vs baseline supports recovery) ───────────────────── // Neutral = resting HR at the baseline mean. @@ -232,9 +241,16 @@ extension RecoveryScorer { // MARK: - Plain-English verdicts (no fabricated numbers; direction only) - static func hrvVerdict(value: Double, baseline: Double) -> String { + static func hrvVerdict(value: Double, baseline: Double, saturationEased: Bool = false) -> String { if value > baseline { return "above baseline, supporting recovery" } - if value < baseline { return "below baseline, limiting recovery" } + if value < baseline { + // Parasympathetic saturation: resting HR is also low and decoupled from HRV, so the low + // HRV reads as a benign saturated vagal state, not fatigue, and its penalty is eased. + // Say so instead of the plain fatigue read. (Only reached when the guard actually fired.) + return saturationEased + ? "below baseline, but low resting HR signals parasympathetic saturation, so the penalty is eased" + : "below baseline, limiting recovery" + } return "at baseline" } diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/CrossDeviceHRVTrend.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/CrossDeviceHRVTrend.swift new file mode 100644 index 0000000000..2832e2bff6 --- /dev/null +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/CrossDeviceHRVTrend.swift @@ -0,0 +1,103 @@ +import Foundation + +/// Cross-device HRV long-term trend — stitch a wearer's full nightly HRV history across a device switch +/// (e.g. a 2-year Oura ring history continuing onto a WHOOP strap) into ONE continuous long-term view, +/// WITHOUT falsely claiming two brands' incompatible RMSSD scales are the same number (Oura RMSSD +/// ~120–155 ms vs WHOOP ~72–112 ms, with no overlap nights to calibrate against — see +/// `Baselines.deviceEraEpoch`, #459). +/// +/// The honest resolution: each night carries BOTH its RAW value (what that device actually measured) and +/// an ERA-RELATIVE % (how far it sits above/below that device-era's own robust centre). Plotting the +/// era-relative series gives a continuous trend SHAPE across the switch — the Oura decline flows straight +/// into the WHOOP era — while the raw series stays scale-honest per device and boundaries are marked so the +/// user knows the sensor changed. This is a DISPLAY/trend lens only; the 0–3 scores stay strictly +/// per-device (via `deviceEraEpoch`). Pure, deterministic, DB-free. +public enum CrossDeviceHRVTrend { + + /// One night on the stitched trend. + public struct Point: Equatable, Sendable { + /// "yyyy-MM-dd". + public let day: String + /// Absolute nightly HRV (ms) exactly as this device reports it — scale differs across eras. + public let raw: Double + /// % above/below this device-era's own robust (median-of-ln) centre. Cross-era comparable in + /// SHAPE (not absolute level), so the long trend reads continuously through the switch. + public let eraRelativePct: Double + /// Coarse HRV-scale brand ("oura"/"whoop"/…) via `Baselines.brandBucket` — the era this night is in. + public let brand: String + /// True on the first night of a NON-FIRST era — i.e. a device switch — so the UI can draw a + /// boundary marker ("switched to WHOOP"). Never true for the very first era's first night. + public let isEraStart: Bool + + public init(day: String, raw: Double, eraRelativePct: Double, brand: String, isEraStart: Bool) { + self.day = day; self.raw = raw; self.eraRelativePct = eraRelativePct + self.brand = brand; self.isEraStart = isEraStart + } + } + + /// A contiguous single-brand stretch of the history, with its own centre — the unit the trend + /// normalises against, and what a UI legend/segment can label ("Oura · 408 nights"). + public struct Era: Equatable, Sendable { + public let brand: String + public let startDay: String + public let endDay: String + public let nights: Int + /// Geometric (median-of-ln) centre HRV of the era, in ms. + public let centerMs: Double + public init(brand: String, startDay: String, endDay: String, nights: Int, centerMs: Double) { + self.brand = brand; self.startDay = startDay; self.endDay = endDay + self.nights = nights; self.centerMs = centerMs + } + } + + public struct Result: Equatable, Sendable { + public let points: [Point] + public let eras: [Era] + public init(points: [Point], eras: [Era]) { self.points = points; self.eras = eras } + } + + /// Build the continuous cross-device trend from source-tagged nightly HRV — one WINNING source per + /// day (the same per-day merge winner the fold uses; NOT one row per source — mirrors the + /// `deviceEraEpoch` contract). Any order; sorted here. Nights are grouped into CONTIGUOUS single-brand + /// eras (a device switch opens a new era); each era normalises against the median of its own ln(HRV). + /// Single-brand history → one era, every `eraRelativePct` vs that one centre (still valid). Empty → empty. + public static func build(_ sourceDays: [(day: String, sourceId: String, hrv: Double)]) -> Result { + let sorted = sourceDays.filter { $0.hrv > 0 } + .sorted { $0.day != $1.day ? $0.day < $1.day : $0.sourceId < $1.sourceId } + guard !sorted.isEmpty else { return Result(points: [], eras: []) } + + // Segment into contiguous same-brand eras. + var eraRows: [[(day: String, sourceId: String, hrv: Double)]] = [] + var cur: [(day: String, sourceId: String, hrv: Double)] = [] + var curBrand: String? = nil + for row in sorted { + let b = Baselines.brandBucket(row.sourceId) + if let cb = curBrand, cb != b { eraRows.append(cur); cur = [] } + curBrand = b + cur.append(row) + } + if !cur.isEmpty { eraRows.append(cur) } + + var points: [Point] = [] + var eras: [Era] = [] + for (ei, era) in eraRows.enumerated() { + let brand = Baselines.brandBucket(era[0].sourceId) + let centerLn = median(era.map { log($0.hrv) }) + eras.append(Era(brand: brand, startDay: era.first!.day, endDay: era.last!.day, + nights: era.count, centerMs: exp(centerLn))) + for (i, row) in era.enumerated() { + points.append(Point(day: row.day, raw: row.hrv, + eraRelativePct: (log(row.hrv) - centerLn) * 100.0, + brand: brand, + isEraStart: ei > 0 && i == 0)) + } + } + return Result(points: points, eras: eras) + } + + static func median(_ xs: [Double]) -> Double { + let s = xs.sorted(); let n = s.count + guard n > 0 else { return 0 } + return n % 2 == 1 ? s[n / 2] : (s[n / 2 - 1] + s[n / 2]) / 2 + } +} diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/DailyStressEngine.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/DailyStressEngine.swift new file mode 100644 index 0000000000..6776427f71 --- /dev/null +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/DailyStressEngine.swift @@ -0,0 +1,286 @@ +import Foundation +import WhoopStore + +/// Daily autonomic-stress score (0–3) from overnight resting HRV + resting HR, redesigned around the +/// sports-science / wearable-HRV literature (Plews, Buchheit 2014, Altini, Hopkins) rather than a naive +/// mean+SD on raw milliseconds. Pure, deterministic, DB-free. +/// +/// Four evidence-backed departures from the old inline `StressMath`: +/// 1. **Log domain.** HRV (RMSSD) is right-skewed; a symmetric z on raw ms is statistically invalid and +/// outlier-dominated. Everything here z-scores lnRMSSD (Plews/Altini; WHOOP does the same). +/// 2. **Dual baselines.** A SHORT (14-night) adaptive baseline drives today's 0–3, so a sustained shift +/// (a supplement, a training block, altitude) is absorbed into "my recent normal" within ~2 weeks and +/// the daily number self-normalizes instead of reading "high stress" forever. A LONG (60-night) +/// reference powers a SEPARATE `ChronicShift` readout — "your baseline itself has moved down N% for D +/// days" — so the sustained load stays visible instead of being silently defined away. Neither Oura +/// nor WHOOP publicly does this; it is the only clean resolution of the adapt-vs-hide tradeoff. +/// 3. **Coupled RHR+HRV (Buchheit 2014 quadrant).** Low HRV is only "stress" when RHR is ALSO elevated +/// above its own baseline (sympathetic overactivity). Low HRV with LOW RHR is benign parasympathetic +/// saturation (fit / deeply relaxed), not stress — so the HRV penalty is damped there, the exact +/// mirror of the RecoveryScorer's parasympathetic-saturation guard. HRV is the primary weight; RHR +/// confirms. This is what stops a saturated fit night (or a low-HRV-but-calm night) reading as stress. +/// 4. **Smallest-worthwhile-change dead-zone.** A combined deviation under 0.5σ (Hopkins SWC) reads as +/// noise, not stress, so sub-threshold wobble doesn't jitter the daily number. +/// +/// Honest by construction: HRV suppression is *sensitive but not specific* (Altini 2021, n≈9M) — the +/// physiology cannot name the cause (stress vs supplement vs training vs illness). This engine therefore +/// scores the autonomic STATE truthfully and surfaces context (quadrant + chronic shift) rather than +/// pretending to diagnose why. Not medical advice. +public enum DailyStressEngine { + + // MARK: - Tunables (named + auditable; grounded in the design doc / cited literature) + + /// Short adaptive baseline (nights) — drives today's 0–3. 14 sits in the literature's 7–14d "acute" + /// range (ithlete/Plews 7d; a touch longer here to steady a noisy nocturnal wearable signal). + public static let shortWindowDays = 14 + /// Long chronic reference (nights) — powers the sustained-shift readout. 60 matches HRV4Training's + /// normal-values window. + public static let longWindowDays = 60 + /// Minimum baseline nights before the short baseline is trusted enough to score. + public static let minBaselineNights = 7 + + /// HRV carries most of the weight (more sensitive to day-to-day autonomic change — Altini/WHOOP); + /// RHR confirms and, via the coupling below, is decisive when it DIVERGES from HRV. + public static let hrvWeight = 0.6 + public static let rhrWeight = 0.4 + + /// Coupled saturation damp (Buchheit quadrant; mirrors `RecoveryScorer.parasympatheticSaturation`). + /// When HRV is suppressed (zHRV ≥ satEnterZ) but RHR is NOT elevated (zRHR < 0), the low HRV is + /// benign parasympathetic saturation, not stress — damp the HRV term, ramping from no damp at the + /// entry to `satMaxDamp` at `satFullZ` of RHR-below-baseline. Never flips the sign; only reduces. + public static let satEnterZ = 0.5 + public static let satFullZ = 1.5 + public static let satMaxDamp = 0.5 + + /// Smallest-worthwhile-change dead-zone (Hopkins 2000; Buchheit 2014): a combined |raw| under this + /// many σ is noise → reads as baseline (1.5), not stress. + public static let swcBand = 0.5 + + /// A `ChronicShift` is flagged as sustained LOAD only when the recent baseline has moved in the load + /// direction on BOTH channels (HRV down AND RHR up) past these — the coupled sustained-shift gate. + public static let sustainedHRVPctThreshold = 5.0 // recent 14d HRV ≥5% below the 60d normal + public static let sustainedRHRBpmThreshold = 2.0 // recent 14d RHR ≥2 bpm above the 60d normal + + /// Band floor for "high" on the 0–3 scale (matches the app's StressBand.high). + public static let highBandFloor = 2.0 + + // MARK: - Output + + /// Buchheit 2014's HRV×RHR quadrant — the autonomic interpretation behind the number. + public enum Quadrant: String, Sendable, Equatable { + case balanced // no meaningful deviation from the recent baseline + case sympatheticStress // HRV down + RHR up → real autonomic stress / fatigue + case parasympatheticSaturation // HRV down + RHR down → benign (fit / deeply relaxed) + case recovered // HRV up → coping well / well recovered + } + + /// The SEPARATE chronic readout: how far the recent (short) baseline has moved vs the long-term + /// normal, and whether that shift is a coupled sustained LOAD (both channels moved together). + public struct ChronicShift: Sendable, Equatable { + /// + = recent 14-night HRV baseline sits this % BELOW the 60-night normal (suppressed). + public let hrvPctBelowLongTerm: Double + /// + = recent 14-night RHR baseline sits this many bpm ABOVE the 60-night normal (elevated). + public let rhrBpmAboveLongTerm: Double + /// True when BOTH channels have shifted in the load direction past their thresholds — a + /// coupled, physiologically-consistent sustained autonomic load (not one noisy channel). + public let isSustainedLoad: Bool + /// Trailing consecutive nights whose HRV sat below the long-term normal band (a rough "for D + /// days" duration for the readout). + public let daysBelowBand: Int + + public init(hrvPctBelowLongTerm: Double, rhrBpmAboveLongTerm: Double, + isSustainedLoad: Bool, daysBelowBand: Int) { + self.hrvPctBelowLongTerm = hrvPctBelowLongTerm + self.rhrBpmAboveLongTerm = rhrBpmAboveLongTerm + self.isSustainedLoad = isSustainedLoad + self.daysBelowBand = daysBelowBand + } + } + + public struct DailyStress: Sendable, Equatable { + /// 0–3 daily score: today vs the SHORT adaptive baseline, coupled + SWC-damped. 1.5 = baseline. + public let score: Double + /// HRV suppression vs the short baseline (log-domain σ), + = suppressed = toward stress. + public let zHRV: Double + /// RHR elevation vs the short baseline (σ), + = elevated = toward stress. 0 when no RHR. + public let zRHR: Double + /// The Buchheit autonomic quadrant behind the score. + public let quadrant: Quadrant + /// The separate long-term baseline-shift readout (nil when there isn't enough long history). + public let chronicShift: ChronicShift? + /// Baseline-density confidence (calibrating / building / solid). + public let confidence: ScoreConfidence + /// False when RHR was unavailable and the score fell back to HRV-only (no coupling possible). + public let usedRHR: Bool + + public init(score: Double, zHRV: Double, zRHR: Double, quadrant: Quadrant, + chronicShift: ChronicShift?, confidence: ScoreConfidence, usedRHR: Bool) { + self.score = score; self.zHRV = zHRV; self.zRHR = zRHR; self.quadrant = quadrant + self.chronicShift = chronicShift; self.confidence = confidence; self.usedRHR = usedRHR + } + } + + // MARK: - Entry point + + /// Evaluate today's daily stress from the daily-metric history. `days` may be any order; the scored + /// day is `today` (a "yyyy-MM-dd") if given and present, else the latest row carrying HRV/RHR. + /// Returns nil when there is no scorable day or too little baseline. + public static func evaluate(days: [DailyMetric], today: String? = nil) -> DailyStress? { + let sorted = dedupByDay(days.sorted { $0.day < $1.day }) + let idx: Int? + if let today { idx = sorted.firstIndex { $0.day == today && ($0.avgHrv != nil || $0.restingHr != nil) } } + else { idx = sorted.lastIndex { $0.avgHrv != nil || $0.restingHr != nil } } + guard let i = idx else { return nil } + return evaluateSorted(sorted, at: i) + } + + /// Batch-score every scorable day in ONE pass — sorts once, so a caller drawing the whole trend pays + /// O(n · window) folds instead of O(n²·log n) re-sorts. Chronological (day, score) pairs. + public static func evaluateSeries(days: [DailyMetric]) -> [(day: String, stress: DailyStress)] { + let sorted = dedupByDay(days.sorted { $0.day < $1.day }) + var out: [(day: String, stress: DailyStress)] = [] + out.reserveCapacity(sorted.count) + for i in sorted.indices where sorted[i].avgHrv != nil || sorted[i].restingHr != nil { + if let s = evaluateSorted(sorted, at: i) { out.append((day: sorted[i].day, stress: s)) } + } + return out + } + + /// Score `sorted[i]` against the days strictly before it. `sorted` MUST be day-ascending. + static func evaluateSorted(_ sorted: [DailyMetric], at i: Int) -> DailyStress? { + let scored = sorted[i] + let history = Array(sorted[0.. Double { + guard hrvZ >= satEnterZ, rhrZ < 0 else { return 1.0 } + let rhrLow = -rhrZ + let t = max(0.0, min(1.0, (rhrLow - satEnterZ) / (satFullZ - satEnterZ))) + return 1.0 - satMaxDamp * t + } + + /// Smallest-worthwhile-change dead-zone: shrink |raw| toward zero by `band`, preserving sign. + static func deadzone(_ raw: Double, band: Double) -> Double { + raw >= 0 ? max(0, raw - band) : min(0, raw + band) + } + + /// Logistic squash to 0–3 (raw 0 → 1.5), identical scale/curve to the daytime + old daily score. + static func squash(_ raw: Double) -> Double { min(max(3.0 / (1.0 + exp(-raw)), 0), 3) } + + static func classify(zHRV: Double, zRHR: Double, usedRHR: Bool) -> Quadrant { + if zHRV <= -satEnterZ { return .recovered } // HRV meaningfully ABOVE baseline + guard zHRV >= satEnterZ else { return .balanced } // HRV within the SWC band → balanced + if !usedRHR { return .sympatheticStress } // HRV down, no RHR to disambiguate + return zRHR > 0 ? .sympatheticStress : .parasympatheticSaturation + } + + // MARK: - Chronic shift (short baseline vs long reference) + + static func chronicShift(scored: DailyMetric, history: [DailyMetric], + shortHRVln: BaselineState, longHist: [DailyMetric]) -> ChronicShift? { + let longLnHRV = longHist.compactMap { $0.avgHrv.map { log(max($0, 1.0)) } } + guard longLnHRV.count >= minBaselineNights else { return nil } + // The long-term reference is a ROBUST MEDIAN centre over the long window — deliberately NOT the + // recency-weighted EWMA the short baseline uses: with the same short half-life the "60-day" EWMA + // collapses onto the recent level and the shift we exist to surface vanishes (it read 0% on real + // gradually-declining data). The short baseline stays the recent EWMA centre; short-vs-long median + // is the sustained shift. + let longCenterLn = median(longLnHRV) + let hrvPct = (1.0 - exp(shortHRVln.baseline - longCenterLn)) * 100.0 + + let shortRHR = history.suffix(shortWindowDays).compactMap { $0.restingHr.map(Double.init) } + let longRHR = longHist.compactMap { $0.restingHr.map(Double.init) } + let rhrBpm: Double = (shortRHR.count >= 3 && longRHR.count >= minBaselineNights) + ? shortRHR.reduce(0, +) / Double(shortRHR.count) - median(longRHR) : 0.0 + + let sustained = hrvPct >= sustainedHRVPctThreshold && rhrBpm >= sustainedRHRBpmThreshold + // Duration: trailing nights whose HRV sat below the long-term band (median − 0.5·robust σ). + let bandFloorLn = longCenterLn - swcBand * max(1.4826 * mad(longLnHRV), 1e-9) + var d = 0 + for row in (history + [scored]).reversed() { + guard let h = row.avgHrv else { continue } + if log(max(h, 1.0)) < bandFloorLn { d += 1 } else { break } + } + return ChronicShift(hrvPctBelowLongTerm: hrvPct, rhrBpmAboveLongTerm: rhrBpm, + isSustainedLoad: sustained, daysBelowBand: d) + } + + /// Median of an unsorted array (0 for empty). + static func median(_ xs: [Double]) -> Double { + guard !xs.isEmpty else { return 0 } + let s = xs.sorted(); let n = s.count + return n % 2 == 1 ? s[n / 2] : (s[n / 2 - 1] + s[n / 2]) / 2 + } + /// Median absolute deviation (robust spread; ×1.4826 ≈ σ for normal data). + static func mad(_ xs: [Double]) -> Double { + let m = median(xs) + return median(xs.map { abs($0 - m) }) + } + + /// Collapse multiple same-day rows (one per source device — Oura / WHOOP / Apple) into ONE row per + /// calendar day, keeping the richest (HRV beats RHR beats neither). `sorted` MUST be day-ascending. + /// Without this, the trailing-window baselines count DEVICE-ROWS, not days, so a "60-day" window can + /// cover ~30 calendar days and the chronic-shift reference never reaches the pre-shift level. + static func dedupByDay(_ sorted: [DailyMetric]) -> [DailyMetric] { + var out: [DailyMetric] = [] + out.reserveCapacity(sorted.count) + func richness(_ m: DailyMetric) -> Int { (m.avgHrv != nil ? 2 : 0) + (m.restingHr != nil ? 1 : 0) } + for row in sorted { + if let last = out.last, last.day == row.day { + if richness(row) > richness(last) { out[out.count - 1] = row } + } else { + out.append(row) + } + } + return out + } + + // MARK: - Baseline folds (reuse the shared window-fold spine: reject OFF so a sustained shift adapts) + + static func foldLnHRV(_ rows: [DailyMetric]) -> BaselineState? { + let ln = rows.map { $0.avgHrv.map { log(max($0, 1.0)) } } + guard ln.contains(where: { $0 != nil }) else { return nil } + return Baselines.foldHistory(ln, cfg: Baselines.readinessHRVLnCfg, rejectHardOutliers: false) + } + + static func foldRHR(_ rows: [DailyMetric]) -> BaselineState? { + let v = rows.map { $0.restingHr.map(Double.init) } + guard v.contains(where: { $0 != nil }) else { return nil } + return Baselines.foldHistory(v, cfg: Baselines.restingHRCfg, rejectHardOutliers: false) + } +} diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeBaselines.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeBaselines.swift new file mode 100644 index 0000000000..4e8942fe12 --- /dev/null +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeBaselines.swift @@ -0,0 +1,149 @@ +import Foundation +import WhoopProtocol + +// DaytimeBaselines.swift — the CALLER-side folding path that turns a person's past DAYTIME history +// into the personal rolling baselines `DaytimeStress.analyze(mode: .baselineRelative(hr:rmssd:))` +// scores today's waking hours against. +// +// `.baselineRelative` (see DaytimeStress.ScoringMode) is deliberately caller-fed: the scorer takes a +// finished `BaselineState` and never fetches history itself. This file is that missing caller — it +// computes ONE daytime aggregate per past day and folds the ordered series through the SAME +// Winsorized-EWMA machinery (`Baselines.foldHistory`) the nightly resting-HR / HRV baselines use, with +// the daytime-specific `Baselines.daytimeHRCfg` / `daytimeRMSSDCfg` configs. +// +// The per-day aggregate is defined in terms of EXACTLY the hourly means the scorer references: it reuses +// `DaytimeStress`'s own `floorDiv` / `bucketSeconds` / `isWakingHour` / `minHourHRSamples` / +// `HRVAnalyzer` so "the value we fold into the baseline" and "the value we later z-score against that +// baseline" are the same quantity, never two drifting definitions. + +public extension DaytimeStress { + + // MARK: - Per-day daytime aggregate + + /// Percentile of a day's waking-hour mean HRs used as that day's daytime-HR aggregate: the 10th, + /// i.e. the CALM FLOOR ("how low does my HR run when I'm calm and awake"). VALIDATED — this is the + /// ~65 bpm pooled figure from the 26-day Oura-reference correlation behind + /// `baselineRelativeHighMarginBPM`. It is intentionally the floor (not the median) because the HR + /// term pairs it with a FIXED bpm margin (a large effective spread via `marginToSigma`), so anchoring + /// at the floor makes "floor + margin" the exact HIGH cutoff. + static let daytimeHRAggregatePercentile: Double = 0.10 + + /// Percentile of a day's waking-hour RMSSDs used as that day's daytime-RMSSD aggregate: the 50th + /// (MEDIAN / typical), NOT the symmetric calm-ceiling percentile. The asymmetry with HR above is + /// deliberate and load-bearing: the RMSSD term is scaled by the person's OWN spread + /// (`Baselines.sigma`), not a fixed margin, so `(baseline − hourRMSSD) / σ` is only meaningful when + /// `baseline` is the TYPICAL daytime RMSSD — then a typical hour reads neutral and only genuine HRV + /// suppression reads stressed. Anchoring at a calm-ceiling percentile (the direct mirror of HR's + /// floor) would make an ordinary median hour sit ~1–2σ "below baseline" and stack a large false + /// stress term on top of the HR term. TUNING SEAM: like every RMSSD figure in this mode, the median + /// choice is a first pass pending an HR+HRV validation pass (the validated study was HR-only). + static let daytimeRMSSDAggregatePercentile: Double = 0.50 + + /// One local day's waking HR + R-R streams, the input to the daytime-baseline fold. `hr`/`rr` carry + /// wall-clock `ts`; `tzOffsetSeconds` (seconds east of UTC for THAT day) places them on the local + /// clock exactly as `analyze`'s `tzOffsetSeconds` does, so DST-varying days can each carry their own + /// offset. + /// + /// Not `Sendable`: `HRSample` / `RRInterval` (WhoopProtocol) aren't `Sendable`, and this is a plain + /// pure-function input built and consumed within one context — it is never sent across actors. + struct DaytimeDayStreams: Equatable { + public let hr: [HRSample] + public let rr: [RRInterval] + public let tzOffsetSeconds: Int + public init(hr: [HRSample], rr: [RRInterval], tzOffsetSeconds: Int) { + self.hr = hr; self.rr = rr; self.tzOffsetSeconds = tzOffsetSeconds + } + } + + /// One day's daytime aggregates, computed with the SCORER's own hourly bucketing so the folded value + /// matches what `.baselineRelative` later references: + /// - `hr`: the `daytimeHRAggregatePercentile` (P10) of the day's WAKING-hour mean HRs, where each + /// hour's mean HR is gated at `minHourHRSamples` exactly like the scorer (a sparse/imported day + /// whose hours never clear the gate yields `nil` — it contributes no daytime-HR floor, honestly). + /// - `rmssd`: the `daytimeRMSSDAggregatePercentile` (P50) of the day's WAKING-hour RMSSDs, each run + /// through the same `HRVAnalyzer.analyze(rawRR:)` cleaner the scorer uses (so ectopic beats can't + /// fabricate variability); `nil` when no waking hour had enough clean R-R (e.g. an HR-only day). + /// Either field is `nil` independently. Bucketing keys off the HR buckets (like the scorer), so an + /// hour with R-R but no HR contributes neither. + static func dayDaytimeAggregate(hr: [HRSample], rr: [RRInterval], + tzOffsetSeconds: Int) -> (hr: Double?, rmssd: Double?) { + guard !hr.isEmpty else { return (nil, nil) } + + // Bucket HR + R-R into LOCAL hour-of-day buckets, byte-for-byte the scorer's step 1. + var hrByBucket: [Int: [Double]] = [:] + for s in hr { + let local = s.ts + tzOffsetSeconds + let bucket = floorDiv(local, bucketSeconds) * bucketSeconds + hrByBucket[bucket, default: []].append(Double(s.bpm)) + } + var rrByBucket: [Int: [Double]] = [:] + for s in rr { + let local = s.ts + tzOffsetSeconds + let bucket = floorDiv(local, bucketSeconds) * bucketSeconds + rrByBucket[bucket, default: []].append(Double(s.rrMs)) + } + + // Per WAKING hour: the gated mean HR + the cleaned RMSSD, mirroring the scorer's step 2 + waking + // filter. Keyed off HR buckets so an R-R-only hour is ignored exactly as the scorer ignores it. + var wakingMeanHRs: [Double] = [] + var wakingRMSSDs: [Double] = [] + for (bucket, hrs) in hrByBucket where isWakingHour(bucket) { + if hrs.count >= minHourHRSamples, let m = mean(hrs) { wakingMeanHRs.append(m) } + if let rmssd = HRVAnalyzer.analyze(rawRR: rrByBucket[bucket] ?? []).rmssd { + wakingRMSSDs.append(rmssd) + } + } + + let hrAgg = wakingMeanHRs.isEmpty + ? nil : quantile(wakingMeanHRs.sorted(), daytimeHRAggregatePercentile) + let rmssdAgg = wakingRMSSDs.isEmpty + ? nil : quantile(wakingRMSSDs.sorted(), daytimeRMSSDAggregatePercentile) + return (hrAgg, rmssdAgg) + } + + // MARK: - Fold the trailing history into personal baselines + + /// Fold a person's trailing per-day daytime streams (OLDEST → NEWEST, TODAY EXCLUDED — today is the + /// day being scored, not part of its own baseline) into the personal baselines `.baselineRelative` + /// consumes. + /// + /// Each day contributes one `dayDaytimeAggregate`; the ordered series is replayed through + /// `Baselines.foldHistory` (the identical Winsorized-EWMA path as the nightly baselines) with + /// `daytimeHRCfg` / `daytimeRMSSDCfg`. `nil` aggregates (sparse/HR-only days) become skip-and-hold + /// nights, exactly like a missing nightly value. + /// + /// The HR baseline is always returned (it may be `.calibrating`; the caller checks `.usable` to + /// decide whether to use `.baselineRelative` at all). The RMSSD baseline is returned ONLY when it is + /// `.usable` (≥ `Baselines.minNightsSeed` days actually carried a daytime RMSSD); otherwise `nil`, so + /// the scorer honestly runs HR-only rather than z-scoring against a 1–2-day, untrustworthy HRV + /// baseline — the whole-baseline grain of the per-hour graceful-nil already in `rawScore`. + static func foldDaytimeBaselines(days: [DaytimeDayStreams]) -> (hr: BaselineState, rmssd: BaselineState?) { + var hrAggs: [Double?] = [] + var rmssdAggs: [Double?] = [] + hrAggs.reserveCapacity(days.count) + rmssdAggs.reserveCapacity(days.count) + for d in days { + let agg = dayDaytimeAggregate(hr: d.hr, rr: d.rr, tzOffsetSeconds: d.tzOffsetSeconds) + hrAggs.append(agg.hr) + rmssdAggs.append(agg.rmssd) + } + let hrState = Baselines.foldHistory(hrAggs, cfg: Baselines.daytimeHRCfg) + let rmssdState = Baselines.foldHistory(rmssdAggs, cfg: Baselines.daytimeRMSSDCfg) + return (hrState, rmssdState.usable ? rmssdState : nil) + } + + /// The scoring mode to hand `analyze` for TODAY, decided from the trailing daytime `history`: + /// `.baselineRelative` once the personal HR baseline is `.usable` (≥ `Baselines.minNightsSeed` days + /// of real daytime HR aggregates — the Oura-style, validated-r≈0.6 path), else `.dayRelative` (the + /// unchanged default). This is the single graceful-degradation gate: a cold start, or a trailing + /// window that is all sparse/imported days, keeps EXACTLY today's pre-existing day-relative behaviour. + static func scoringMode(history days: [DaytimeDayStreams]) -> ScoringMode { + let baselines = foldDaytimeBaselines(days: days) + guard baselines.hr.usable else { return .dayRelative } + // RMSSD stays out of the LIVE score until it has its own validation pass — see + // `DaytimeStress.daytimeRMSSDScoringEnabled`. The baseline is still folded above (so the + // machinery + tests exercise the real path); it just doesn't reach scoring while gated off. + let rmssd = DaytimeStress.daytimeRMSSDScoringEnabled ? baselines.rmssd : nil + return .baselineRelative(hr: baselines.hr, rmssd: rmssd) + } +} diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeStress.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeStress.swift index 0d09e2814f..729673d793 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeStress.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/DaytimeStress.swift @@ -38,10 +38,116 @@ public enum DaytimeStress { public static let highBandFloor: Double = 2.0 /// Consecutive most-recent covered hours that must all be HIGH to flag sustained stress. public static let sustainedHours: Int = 3 + + // MARK: - Motion gate (MOVE E — the correctness gate) + // + // Cardiac signals cannot tell psychological stress from EXERTION: a brisk walk and an anxious + // meeting both raise HR and drop HRV. Without a motion channel, `analyze` scored an ambulatory + // hour as "stress" — the central correctness bug. When the caller supplies the day's gravity + // (wrist accelerometer), an hour that was substantially ambulatory is MASKED (`level == nil`, + // `HourPoint.maskedForActivity == true`) rather than scored, and it is excluded from the calm + // reference and the coverage/"time in high stress" totals. No gravity → no masking → byte- + // identical prior behaviour (honest: we only gate when we can see motion). + + /// An hour whose gravity-derived activity is ambulatory for at least this fraction of its + /// records is EXERTION, not stress — masked, not scored. "Ambulatory" = a per-record activity + /// intensity above `WorkoutDetector.motionThreshold` (0.20 L2-g, the codebase's calibrated walk + /// floor: desk ≈ 0.05–0.10 g, walking ≈ 0.2–0.4 g). 0.30 ⇒ "≥ 30 % of the hour was walking/ + /// moving"; below it, a stray reach or a single trip to the kitchen doesn't mask a desk hour. + /// First-cut, wants tuning against real worn days (an hourly grain is coarse — this is the gate + /// that later unblocks finer 5-min epochs, at which point the fraction can tighten). Range (0, 1]. + public static let activityMaskFraction: Double = 0.30 + /// Post-exercise shadow: HR stays elevated for a while AFTER exertion ends, so the single hour + /// that immediately FOLLOWS a directly-ambulatory hour is ALSO masked WHILE its mean HR is still + /// above the calm reference by this margin (bpm). A following hour whose HR has already returned + /// to the calm reference is scored normally — the shadow self-limits to genuine cardiac recovery. + /// Deliberately ONE hour deep (keyed on the directly-active hour, not chained through prior + /// shadows) so a genuinely tense afternoon that happens to follow a workout isn't masked away. + /// Range ≥ 0. + public static let postActivityShadowBPM: Double = 8.0 /// First/last local hour-of-day treated as "waking" for the timeline (06:00–22:00). public static let wakingStartHour: Int = 6 public static let wakingEndHour: Int = 22 + /// VALIDATED (26-day Oura-reference correlation, HR-only): a personal daytime-HR elevation + /// of ~15 bpm over a POOLED/ROLLING baseline — the 10th-percentile daytime HR pooled across + /// days, ~65 bpm in the reference set — is where elevated HR starts reading as + /// Oura-comparable "high" stress (r≈0.6 against Oura's own stress signal). A PER-DAY + /// (day-relative) baseline scored WORSE in the same comparison (r 0.43–0.53): an all-day + /// elevated day pulls its own floor up and masks the stress, which is exactly why + /// `.baselineRelative` leans on `Baselines`' cross-day rolling EWMA instead of a day-local + /// reference. TUNING SEAM: this is HR-only; HR+HRV (WHOOP-era, RMSSD included) is expected + /// to beat this r≈0.6 ceiling — re-validate this margin once that comparison exists. See + /// `marginToSigma` for how it's translated onto the shared 0–3 squash curve. + public static let baselineRelativeHighMarginBPM: Double = 15.0 + + /// Gate for whether the personal daytime-RMSSD baseline feeds the live 0–3 score. `false`: + /// `.baselineRelative` scores HR-only, exactly the channel the r≈0.6 margin above was validated + /// on. The RMSSD half of the pipeline — `dayDaytimeAggregate`, `foldDaytimeBaselines`, the + /// `daytime_rmssd` config, and `rawScore`'s HRV term — is built and unit-tested, but stays OUT + /// of the live score until it has its OWN Oura-reference validation pass. + /// + /// WHY OFF (validated against real WHOOP data, 2026-07): daytime RMSSD off the wrist is + /// artifact-dominated — hourly values swing ~40→430 ms as posture / motion / talking break the + /// R-R stream, an order of magnitude noisier than the overnight recumbent HRV the nightly + /// baselines use. `rawScore` sums the HRV z EQUAL-WEIGHT with the HR z, so an artifact hour can + /// swing the combined score by ±3 (the full band) on noise alone. Enabling it before it is shown + /// to IMPROVE the correlation risks pushing the combined score BELOW the HR-only r≈0.6 ceiling — + /// the exact regression `baselineRelativeHighMarginBPM`'s comment warns against. Flip to `true` + /// only once daytime HR+RMSSD is validated to beat HR-only on an Oura-style stress reference. + public static let daytimeRMSSDScoringEnabled: Bool = false + + // MARK: - Scoring mode + + /// WHERE each hour's "calm" reference point + spread come from. Every other step — + /// bucketing, the waking-hour filter, the squash curve, sustained-high, high-stress-minutes + /// — is identical between modes; only the reference differs. + /// + /// Relationship to the rest of the Stress screen (StressView.swift): the DAILY 0–3 score + /// (`StressModel`) already compares last night's NIGHTLY resting-HR/HRV to a plain trailing + /// 30-day mean/SD, computed locally in StressView (not via `Baselines`) — that's a + /// once-a-day number from SLEEP vitals. The Advanced HRV card (`StressIndex`, + /// `HRVFreqDomain`) is a today-only descriptive lens with no baseline at all. `.baselineRelative` + /// is neither: it's an HOURLY breakdown of TODAY from DAYTIME/waking-hours HR+RMSSD against a + /// PERSONAL cross-day rolling baseline. Daytime HR runs warmer than nocturnal resting HR + /// (posture, thermic effect), so it needs its OWN baseline (`daytime_hr`/`daytime_rmssd`, + /// below) rather than reusing the nightly `resting_hr`/`hrv` configs — reusing the nightly + /// ones would systematically over-read stress. The three surfaces are complementary lenses + /// on the same underlying autonomic signal, not competing implementations of one baseline. + public enum ScoringMode: Equatable, Sendable { + /// DEFAULT — unchanged from before this mode existed. Each hour is z-scored against + /// THIS DAY's own calm-hour reference (`calmReference`): the lower quartile of the + /// day's own waking-hour mean HR, the upper quartile of its own waking-hour RMSSD. No + /// personal history needed — the day is its own baseline. Byte-identical output to the + /// pre-existing single-mode `analyze` for the same hr/rr/tzOffsetSeconds. + case dayRelative + + /// Oura-style — each hour is z-scored against the PERSONAL rolling baseline for daytime + /// HR (and, when available, daytime RMSSD): the SAME Winsorized-EWMA machinery + /// (`Baselines.update` / `Baselines.foldHistory`) that backs the nightly HRV / + /// resting-HR baselines elsewhere in the app, using `Baselines.metricCfg["daytime_hr"]` + /// / `["daytime_rmssd"]` (see `Baselines.daytimeHRCfg` / `daytimeRMSSDCfg`). A caller + /// builds `hr` (and, when it has the history, `rmssd`) by folding this person's past + /// daytime aggregates — VALIDATED as each day's 10th-percentile daytime HR (a pooled + /// "how low does my HR run when I'm calm and awake" floor), the same way the nightly + /// baselines are folded from past nights. + /// + /// The HR reference point is `hr.baseline`, but the HIGH-band threshold does NOT scale + /// with this person's own day-to-day `hr.spread` — see `baselineRelativeHighMarginBPM`: + /// the validated model is a roughly FIXED bpm margin over the personal floor, not a + /// variability-scaled one. `hr.spread` still rides along on the passed-in state for + /// other consumers; this mode simply doesn't read it for the HR term. + /// + /// `rmssd` is `nil` when no personal RMSSD baseline exists yet — e.g. an imported, + /// Oura-era day with no R-R stream, so there is no history to fold one from. The + /// stressor then honestly falls back to HR-only scoring for the whole read and flags + /// `Result.hrOnlyFallback`; this mirrors the per-hour graceful-nil already in + /// `rawScore`, just at the whole-baseline grain instead of the single-hour grain. The + /// RMSSD term (when present) DOES still scale by `rmssd.spread` via `Baselines.sigma` — + /// only the HR term has a validated fixed-margin figure so far. + case baselineRelative(hr: BaselineState, rmssd: BaselineState?) + } + // MARK: - Output /// One hour of the daytime timeline. `level` is the shared 0–3 stress proxy, or nil @@ -57,16 +163,23 @@ public enum DaytimeStress { public let meanHR: Double? /// RMSSD over the hour's clean R-R (ms), or nil (too few clean beats). public let rmssd: Double? + /// True when this hour was left unscored because it was AMBULATORY (exertion), not because + /// it lacked HR — so `level == nil` here means "masked as activity", not "no data". Lets the + /// UI distinguish "you were moving" from "no reading" and keeps active hours out of the calm + /// reference and the coverage totals. See the motion-gate constants above. + public let maskedForActivity: Bool /// True when the hour was scored (had enough HR to place on the curve). public var hasData: Bool { level != nil } - public init(hour: Int, startTs: Int, level: Double?, meanHR: Double?, rmssd: Double?) { + public init(hour: Int, startTs: Int, level: Double?, meanHR: Double?, rmssd: Double?, + maskedForActivity: Bool = false) { self.hour = hour self.startTs = startTs self.level = level self.meanHR = meanHR self.rmssd = rmssd + self.maskedForActivity = maskedForActivity } } @@ -82,14 +195,37 @@ public enum DaytimeStress { public let dayMean: Double? /// Peak scored hour (highest `level`), or nil. public let peak: HourPoint? + /// ADDITIVE — total minutes across SCORED waking hours at/above `highBandFloor`, the + /// Oura-comparable "time in high stress" figure. Each scored hour is one `bucketSeconds` + /// bucket, so this is `(# high-band scored hours) * bucketSeconds / 60`. Compare against + /// Oura's `stress_high_s / 60` — NOOP's timeline is hourly-grain vs Oura's ~5-minute + /// grain, so treat this as a coarse approximation, not a precise match. 0 for `.empty` + /// and for any day with no scored hours. + public let highStressMinutes: Int + /// ADDITIVE — true when `.baselineRelative` mode was requested but had no personal RMSSD + /// baseline to score against (e.g. an imported Oura-era day with no R-R history), so the + /// whole read honestly fell back to HR-only scoring. Always false in `.dayRelative` mode + /// (there, a missing RMSSD is already handled per-hour by `rawScore`, not flagged + /// day-wide) and false for `.empty`. + public let hrOnlyFallback: Bool + /// ADDITIVE — count of waking hours left unscored because they were AMBULATORY (the motion + /// gate fired), i.e. `hours.filter { $0.maskedForActivity }.count`. Lets a caller report + /// honest coverage ("N hours excluded — you were moving") instead of a silently short + /// timeline. 0 when no gravity was supplied or nothing was masked; 0 for `.empty`. + public let activityMaskedHours: Int public init(hours: [HourPoint], sustainedHigh: Bool, sustainedRun: Int, - dayMean: Double?, peak: HourPoint?) { + dayMean: Double?, peak: HourPoint?, + highStressMinutes: Int = 0, hrOnlyFallback: Bool = false, + activityMaskedHours: Int = 0) { self.hours = hours self.sustainedHigh = sustainedHigh self.sustainedRun = sustainedRun self.dayMean = dayMean self.peak = peak + self.highStressMinutes = highStressMinutes + self.hrOnlyFallback = hrOnlyFallback + self.activityMaskedHours = activityMaskedHours } /// The scored hours only (level non-nil), in time order. @@ -97,7 +233,9 @@ public enum DaytimeStress { /// Empty read — used when the day had no usable intraday HR at all. public static let empty = Result(hours: [], sustainedHigh: false, sustainedRun: 0, - dayMean: nil, peak: nil) + dayMean: nil, peak: nil, + highStressMinutes: 0, hrOnlyFallback: false, + activityMaskedHours: 0) } // MARK: - Shared stress math (identical formula to the daily StressModel) @@ -135,6 +273,19 @@ public enum DaytimeStress { return min(max(s, 0), 3) } + /// Solve for the z-score spread `sd` such that a raw elevation of exactly `marginBPM` (or any + /// unit — this is unit-agnostic) squashes to exactly `band` on the shared 0–3 curve: + /// `band = 3 / (1 + e^(−marginBPM/sd))`. Used to translate `baselineRelativeHighMarginBPM`'s + /// validated bpm figure into the `sd` the shared `squash` curve expects, so "baseline + + /// margin" lands exactly on `band` by construction rather than by a second, separate + /// threshold check. Defensive fallback (never divides by zero/negative-log) if `band` is + /// ever configured at or outside the curve's open range (0, 3). + static func marginToSigma(marginBPM: Double, atBand band: Double) -> Double { + let ratio = 3.0 / band - 1.0 + guard ratio > 0, marginBPM > 0 else { return max(marginBPM, 1e-9) } + return marginBPM / (-log(ratio)) + } + // MARK: - Public API /// Build the daytime stress timeline from a day's banked HR + R-R. @@ -144,25 +295,64 @@ public enum DaytimeStress { /// - rr: the day's `[RRInterval]`. /// - tzOffsetSeconds: seconds east of UTC, for placing each bucket on the LOCAL /// clock (so "waking hours" and the hour labels are local). Defaults to UTC. + /// - gravity: the day's `[GravitySample]` (wrist accelerometer), for the motion gate. Defaults + /// empty: with no gravity NOTHING is masked and the read is byte-identical to before. When + /// present, ambulatory hours are masked out of the score (see the motion-gate constants). + /// - mode: `.dayRelative` (DEFAULT — unchanged existing behaviour) or + /// `.baselineRelative` (Oura-style, vs a personal rolling baseline). ADDITIVE and + /// opt-in: existing callers that don't pass `mode` keep the exact prior behaviour. /// /// Returns `.empty` when there isn't a single hour with enough HR to score. public static func analyze(hr: [HRSample], rr: [RRInterval], - tzOffsetSeconds: Int = 0) -> Result { + gravity: [GravitySample] = [], + tzOffsetSeconds: Int = 0, + mode: ScoringMode = .dayRelative) -> Result { // v7.0.2 perf (#707): buckets the day's full HR + R-R streams into per-hour aggregates and runs an // RMSSD per hour — invoked from the Stress view, so a `body` re-evaluation re-buckets the whole day. - // Memoize on the streams' fingerprint + tz offset; result is a small `Result`, raw arrays not held. + // Memoize on the streams' fingerprint + tz offset + scoring mode; result is a small `Result`, raw + // arrays not held. The mode key folds in only (baseline, spread) for baseline-relative — the two + // fields that can change the score — not the whole BaselineState (nValid/status/etc. never do). + let modeKey: ModeKey + switch mode { + case .dayRelative: + modeKey = .dayRelative + case .baselineRelative(let hrBaseline, let rmssdBaseline): + modeKey = .baselineRelative(hrBaseline: hrBaseline.baseline, hrSpread: hrBaseline.spread, + rmssdBaseline: rmssdBaseline?.baseline, rmssdSpread: rmssdBaseline?.spread) + } let key = StressKey( hr: StreamFingerprint.of(hr, ts: { $0.ts }, quant: { Int($0.bpm) }), rr: StreamFingerprint.of(rr, ts: { $0.ts }, quant: { Int($0.rrMs) }), - tz: tzOffsetSeconds) - return analyzeCache.value(key) { analyzeUncached(hr: hr, rr: rr, tzOffsetSeconds: tzOffsetSeconds) } + // Fold all three axes into one quant so two different gravity streams can't share a key + // (same hr/rr, different motion must NOT reuse a cached result). + gravity: StreamFingerprint.of(gravity, ts: { $0.ts }, quant: { + Int(($0.x * 128).rounded()) &+ Int(($0.y * 128).rounded()) &* 257 + &+ Int(($0.z * 128).rounded()) &* 66_049 + }), + tz: tzOffsetSeconds, mode: modeKey) + return analyzeCache.value(key) { + analyzeUncached(hr: hr, rr: rr, gravity: gravity, tzOffsetSeconds: tzOffsetSeconds, mode: mode) + } + } + + private struct StressKey: Hashable { + let hr: StreamFingerprint; let rr: StreamFingerprint; let gravity: StreamFingerprint + let tz: Int; let mode: ModeKey + } + + /// Hashable fingerprint of `ScoringMode` for the memo cache — `BaselineState` itself isn't + /// `Hashable`, and only its `baseline`/`spread` can change the score, so those are what get + /// folded in (see the `analyze` doc above). + private enum ModeKey: Hashable { + case dayRelative + case baselineRelative(hrBaseline: Double, hrSpread: Double, rmssdBaseline: Double?, rmssdSpread: Double?) } - private struct StressKey: Hashable { let hr: StreamFingerprint; let rr: StreamFingerprint; let tz: Int } private static let analyzeCache = AnalyticsMemoCache(capacity: 8) private static func analyzeUncached(hr: [HRSample], rr: [RRInterval], - tzOffsetSeconds: Int) -> Result { + gravity: [GravitySample], + tzOffsetSeconds: Int, mode: ScoringMode) -> Result { guard !hr.isEmpty else { return .empty } // 1) Bucket HR + R-R into LOCAL hour-of-day buckets, keyed by the bucket start @@ -194,25 +384,91 @@ public enum DaytimeStress { aggs.append(HourAgg(bucket: b, meanHR: mHR, rmssd: rrRes.rmssd, nHR: hrs.count)) } - // 3) The day's OWN quiet reference: centre on the CALM end (the lower quartile of - // hourly mean HR, the upper quartile of hourly RMSSD), and spread from the - // across-hour SD. This makes a flat day read ~baseline and a spiky day surface - // its tense hours — without any cross-day history. Falls back to the plain mean - // when there are too few scored hours for a quartile. - // - // Built from the WAKING hours only — the same hours scored in step 4. Sleep is the - // calmest, lowest-HR / highest-HRV stretch of the day, and the analysis window - // always begins at local midnight, so the current day routinely carries several - // hours of it. Letting those night hours into the reference drags the "calm" anchor - // far beneath every waking hour, inflating an ordinary calm day toward HIGH and - // falsely tripping the sustained-high Breathe nudge. - let referenceAggs = aggs.filter { isWakingHour($0.bucket) } - let hrMeans = referenceAggs.compactMap { $0.meanHR } - let rmssdVals = referenceAggs.compactMap { $0.rmssd } - let refHR = calmReference(hrMeans, calmIsLow: true) // calm HR is LOW - let refRMSSD = calmReference(rmssdVals, calmIsLow: false) // calm HRV is HIGH - let sdHR = std(hrMeans, mean: mean(hrMeans)) - let sdRMSSD = std(rmssdVals, mean: mean(rmssdVals)) + // 2b) Motion gate: bucket the day's gravity-derived activity by the SAME local hour and mark + // each hour AMBULATORY when at least `activityMaskFraction` of its records clear the + // calibrated walk floor (`WorkoutDetector.motionThreshold`) — reusing the exact activity + // series that SedentaryDetector / WorkoutDetector already trust. Empty gravity → no active + // buckets → nothing masked below (byte-identical to the pre-motion behaviour). + var activeFracByBucket: [Int: Double] = [:] + if !gravity.isEmpty { + var counts: [Int: (active: Int, total: Int)] = [:] + for p in WorkoutDetector.activitySeries(gravity) { + let local = p.ts + tzOffsetSeconds + let bucket = floorDiv(local, bucketSeconds) * bucketSeconds + var e = counts[bucket] ?? (0, 0) + e.total += 1 + if p.intensity > WorkoutDetector.motionThreshold { e.active += 1 } + counts[bucket] = e + } + for (b, e) in counts where e.total > 0 { + activeFracByBucket[b] = Double(e.active) / Double(e.total) + } + } + func isAmbulatory(_ bucket: Int) -> Bool { + (activeFracByBucket[bucket] ?? 0) >= activityMaskFraction + } + + // 3) The reference point + spread for each signal — WHERE they come from depends on + // `mode`. Every other step (bucketing above, the waking-hour filter, the squash + // curve, sustained-high, high-stress-minutes below) is identical between modes. + let refHR: Double? + let sdHR: Double + let refRMSSD: Double? + let sdRMSSD: Double + let hrOnlyFallback: Bool + switch mode { + case .dayRelative: + // The day's OWN quiet reference: centre on the CALM end (the lower quartile of + // hourly mean HR, the upper quartile of hourly RMSSD), and spread from the + // across-hour SD. This makes a flat day read ~baseline and a spiky day surface + // its tense hours — without any cross-day history. Falls back to the plain mean + // when there are too few scored hours for a quartile. + // + // Built from the WAKING hours only — the same hours scored in step 4. Sleep is the + // calmest, lowest-HR / highest-HRV stretch of the day, and the analysis window + // always begins at local midnight, so the current day routinely carries several + // hours of it. Letting those night hours into the reference drags the "calm" anchor + // far beneath every waking hour, inflating an ordinary calm day toward HIGH and + // falsely tripping the sustained-high Breathe nudge. + // Ambulatory hours are excluded from the day's OWN calm reference too: an exertion + // hour's elevated HR / suppressed HRV must not pull the calm anchor up or inflate the + // across-hour spread the z-scores divide by. + let referenceAggs = aggs.filter { isWakingHour($0.bucket) && !isAmbulatory($0.bucket) } + let hrMeans = referenceAggs.compactMap { $0.meanHR } + let rmssdVals = referenceAggs.compactMap { $0.rmssd } + refHR = calmReference(hrMeans, calmIsLow: true) // calm HR is LOW + refRMSSD = calmReference(rmssdVals, calmIsLow: false) // calm HRV is HIGH + sdHR = std(hrMeans, mean: mean(hrMeans)) + sdRMSSD = std(rmssdVals, mean: mean(rmssdVals)) + hrOnlyFallback = false + + case .baselineRelative(let hrBaseline, let rmssdBaseline): + // The PERSONAL cross-day baseline, folded by the caller from past daytime + // aggregates via `Baselines.update`/`foldHistory` (see the `ScoringMode` doc). + refHR = hrBaseline.baseline + // VALIDATED tuning, not `Baselines.sigma(hrBaseline)`: the correlation study behind + // `baselineRelativeHighMarginBPM` found a roughly FIXED bpm margin over the personal + // floor — not one scaled by this person's own day-to-day spread — best matched + // Oura's stress signal. `marginToSigma` solves for the sd that makes exactly + // `refHR + baselineRelativeHighMarginBPM` land on `highBandFloor` on the shared + // squash curve, so the validated margin IS the "high" cutoff by construction. + sdHR = Self.marginToSigma(marginBPM: baselineRelativeHighMarginBPM, atBand: highBandFloor) + if let rmssdBaseline { + refRMSSD = rmssdBaseline.baseline + // No independently validated RMSSD margin yet (see the constant's doc) — this + // term still scales by the person's own spread via the shared σ conversion. + sdRMSSD = Baselines.sigma(rmssdBaseline) + hrOnlyFallback = false + } else { + // No personal RMSSD baseline exists (e.g. an Oura-era day with no R-R history to + // fold one from). `rawScore` already treats a nil meanRMSSD as "skip this term", + // so passing nil here gracefully degrades to HR-only scoring — flagged honestly + // in the output rather than silently. + refRMSSD = nil + sdRMSSD = 0 + hrOnlyFallback = true + } + } // 4) Score each waking-hour bucket on the shared 0–3 curve. var points: [HourPoint] = [] @@ -222,23 +478,38 @@ public enum DaytimeStress { let hourOfDay = floorDiv(a.bucket, bucketSeconds) % 24 // The wall-clock bucket start (undo the local shift applied above). let wallStart = a.bucket - tzOffsetSeconds - // Score only when at least one signal is present AND HR cleared the count gate - // (HR is the always-available anchor; RMSSD enriches it when beats allow). - let level: Double? = a.meanHR != nil + // Motion gate: an AMBULATORY hour — or the post-exercise shadow hour whose HR hasn't yet + // recovered to the calm reference — is EXERTION, so its elevated HR is masked out of the + // score instead of read as stress. The shadow is gated on `refHR` so it self-limits to + // genuine cardiac recovery (a following hour already back at baseline scores normally). + // Only meaningful when the hour actually HAD a reading to withhold — a no-HR hour is plain + // `.noData`, not "masked". + let shadow = isAmbulatory(a.bucket - bucketSeconds) + && a.meanHR != nil && refHR != nil && a.meanHR! > refHR! + postActivityShadowBPM + let masked = a.meanHR != nil && (isAmbulatory(a.bucket) || shadow) + // Score only when at least one signal is present AND HR cleared the count gate AND the + // hour wasn't motion-masked (HR is the always-available anchor; RMSSD enriches it). + let level: Double? = (a.meanHR != nil && !masked) ? squash(rawScore(hr: a.meanHR, meanHR: refHR, sdHR: sdHR, rmssd: a.rmssd, meanRMSSD: refRMSSD, sdRMSSD: sdRMSSD)) : nil points.append(HourPoint(hour: hourOfDay, startTs: wallStart, - level: level, meanHR: a.meanHR, rmssd: a.rmssd)) + level: level, meanHR: a.meanHR, rmssd: a.rmssd, + maskedForActivity: masked)) } + let activityMaskedHours = points.reduce(0) { $0 + ($1.maskedForActivity ? 1 : 0) } let scored = points.compactMap { p -> (HourPoint, Double)? in p.level.map { (p, $0) } } guard !scored.isEmpty else { // No scorable waking hour — still return the (unscored) timeline so the UI can - // show "not enough data" rather than nothing. + // show "not enough data" rather than nothing. `hrOnlyFallback` is a MODE property + // (whether a personal RMSSD baseline existed to score against), so it's still worth + // reporting even though nothing ended up scored. return points.isEmpty ? .empty : Result(hours: points, sustainedHigh: false, sustainedRun: 0, - dayMean: nil, peak: nil) + dayMean: nil, peak: nil, + highStressMinutes: 0, hrOnlyFallback: hrOnlyFallback, + activityMaskedHours: activityMaskedHours) } // 5) Sustained-high flag: walk back from the latest SCORED hour while each is HIGH. @@ -251,8 +522,16 @@ public enum DaytimeStress { let dayMean = mean(scored.map { $0.1 }) let peak = scored.max { $0.1 < $1.1 }?.0 + // 6) Oura-comparable "time in high stress": each scored hour at/above `highBandFloor` + // is one full `bucketSeconds` bucket, converted to minutes. Uses the SAME threshold + // `StressBand.high` (StressView) and the sustained-high check above already use, so + // all three stay in lockstep by construction. + let highStressMinutes = scored.filter { $0.1 >= highBandFloor }.count * (bucketSeconds / 60) + return Result(hours: points, sustainedHigh: sustained, sustainedRun: run, - dayMean: dayMean, peak: peak) + dayMean: dayMean, peak: peak, + highStressMinutes: highStressMinutes, hrOnlyFallback: hrOnlyFallback, + activityMaskedHours: activityMaskedHours) } // MARK: - Helpers diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/HRVAnalyzer.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/HRVAnalyzer.swift index 3dafdbbe75..60a9c515ab 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/HRVAnalyzer.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/HRVAnalyzer.swift @@ -286,6 +286,29 @@ public enum HRVAnalyzer { nInput: nInput, nClean: clean.count) } + /// SDNN index (Task Force 1996): the MEAN of SDNN computed over consecutive fixed-length segments + /// (default 5 min) of the R-R series, each segment cleaned with the SAME range + Malik ectopic + /// rejection the nightly path uses. Unlike whole-night SDNN — which is dominated by the slow HR drift + /// ACROSS sleep stages and can read 2-3× higher — the index reflects SHORT-TERM variability, so it is + /// window-comparable to a wearable's short SDNN reading (e.g. Apple Watch's ~1-min + /// `heartRateVariabilitySDNN` samples) and is the value that can be honestly cross-checked against one. + /// Segments with fewer than `minBeats` clean intervals are skipped; nil when no segment qualifies. + /// Pure, deterministic. Kotlin twin: `HrvAnalyzer.sdnnIndex`. + public static func sdnnIndex(_ rr: [RRInterval], segmentSec: Int = 300) -> Double? { + guard segmentSec > 0, let first = rr.map(\.ts).min(), let last = rr.map(\.ts).max(), + last >= first else { return nil } + var segStart = first + var segmentSDNNs: [Double] = [] + while segStart <= last { + if let sd = analyze(rr, windowStart: segStart, windowEnd: segStart + segmentSec - 1).sdnn { + segmentSDNNs.append(sd) + } + segStart += segmentSec + } + guard !segmentSDNNs.isEmpty else { return nil } + return segmentSDNNs.reduce(0, +) / Double(segmentSDNNs.count) + } + // MARK: - Rolling / windowed rMSSD timeline (#803) /// One windowed rMSSD point: the rMSSD (ms) over the trailing `windowSec` of R-R intervals ending at diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/ReadinessEngine.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/ReadinessEngine.swift index 958082ba0d..5700a7dd08 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/ReadinessEngine.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/ReadinessEngine.swift @@ -57,10 +57,15 @@ public enum ReadinessEngine { public let acwr: Double? /// Foster training monotony over the last week (nil if not enough strain history). public let monotony: Double? + /// How much history backs this read (HRV/RHR baseline density) — so the card can show + /// calibrating / building / solid instead of a confident number off a 7-night baseline. + public let confidence: ScoreConfidence public init(level: Level, headline: String, summary: String, - signals: [Signal], acwr: Double?, monotony: Double?) { + signals: [Signal], acwr: Double?, monotony: Double?, + confidence: ScoreConfidence = .calibrating) { self.level = level; self.headline = headline; self.summary = summary self.signals = signals; self.acwr = acwr; self.monotony = monotony + self.confidence = confidence } } @@ -139,6 +144,8 @@ public enum ReadinessEngine { unit: "ms", decimals: 0, higherIsBetter: true, + logDomain: true, // RD1: lnRMSSD — HRV is right-skewed + cfg: Baselines.readinessHRVLnCfg, // RD2: ln-space spine, reject off goodText: "above your baseline - well recovered", neutralText: "in your normal range", watchText: "a touch below baseline", @@ -153,6 +160,7 @@ public enum ReadinessEngine { unit: "bpm", decimals: 0, higherIsBetter: false, + cfg: Baselines.restingHRCfg, // RD2: raw-bpm spine, reject off goodText: "at or below baseline", neutralText: "in your normal range", watchText: "running a little high", @@ -208,8 +216,16 @@ public enum ReadinessEngine { let (level, headline, summary) = synthesize(signals: signals, hasHistory: !history.isEmpty || acwr != nil) + // RD-confidence: surface how much history backs the read (HRV baseline density, the primary + // readiness driver). A read off a 7-night baseline must not look as certain as one off the full + // 30-night window. Insufficient reads carry .calibrating. + let hrvBaselineNights = history.suffix(baselineWindow).compactMap { $0.avgHrv }.count + let confidence = ScoreConfidence.readiness(hasRead: level != .insufficient, + baselineNights: hrvBaselineNights, + fullWindow: baselineWindow) return Readiness(level: level, headline: headline, summary: summary, - signals: signals, acwr: acwr, monotony: monotony) + signals: signals, acwr: acwr, monotony: monotony, + confidence: confidence) } // MARK: Signal builders @@ -217,13 +233,33 @@ public enum ReadinessEngine { /// Build a z-score signal for a metric where the baseline is the trailing window. private static func zSignal(value: Double?, baseline: [Double], key: String, label: String, unit: String, decimals: Int, - higherIsBetter: Bool, + higherIsBetter: Bool, logDomain: Bool = false, + cfg: MetricCfg, goodText: String, neutralText: String, watchText: String, badText: String) -> Signal? { - guard let v = value, baseline.count >= minBaseline, - let m = mean(baseline), let sd = sampleSD(baseline), sd > 0 else { return nil } + guard let v = value, baseline.count >= minBaseline else { return nil } + // RD1: right-skewed metrics (HRV/RMSSD) are z-scored in the LOG domain — lnRMSSD is closer to + // normal, so a symmetric z is statistically valid, whereas a raw-ms z over-weights the long + // upper tail and misstates tail rarity (Plews/Altini; the app's own HRVReadiness works this + // way). RHR/resp are ~normal and stay linear. Evidence stays in the metric's own units, but the + // baseline shown is then the GEOMETRIC mean (exp of the log-mean) — a typical night, not an + // outlier-inflated arithmetic mean. + let tv = logDomain ? log(max(v, 1.0)) : v + let tb = logDomain ? baseline.map { log(max($0, 1.0)) } : baseline + // RD2: fold the trailing baseline through the shared Winsorized-EWMA spine — recency-weighted, + // σ-floored (a tight baseline can't saturate the z), and Winsor-clamped so a single freak night + // is DAMPED not folded raw — instead of a flat mean + sample SD. Hard-outlier REJECTION is off + // (`rejectHardOutliers: false`): a re-folded trailing window must ADAPT to a recent sustained + // shift (fitness change / device swap) rather than reject the new normal as a run of outliers — + // the window-fold vs incremental-fold distinction, validated on real HRV history. `cfg` is in + // the SAME space as `tb` (ln for HRV, linear for RHR); center + spread come back σ-floored. + let state = Baselines.foldHistory(tb.map { Optional($0) }, cfg: cfg, rejectHardOutliers: false) + guard state.usable else { return nil } + let sigma = Baselines.sigma(state) + guard sigma > 0 else { return nil } + let m = state.baseline // Orient z so positive always means "better". - let z = (higherIsBetter ? (v - m) : (m - v)) / sd + let z = (higherIsBetter ? (tv - m) : (m - tv)) / sigma let flag: Flag let text: String switch z { @@ -233,7 +269,8 @@ public enum ReadinessEngine { default: flag = .bad; text = badText } return Signal(key: key, label: label, - evidence: evidence(value: v, baseline: m, unit: unit, decimals: decimals), + evidence: evidence(value: v, baseline: logDomain ? exp(m) : m, + unit: unit, decimals: decimals), detail: text, flag: flag) } diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer+Trace.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer+Trace.swift index e7184ed350..44452d1cf6 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer+Trace.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer+Trace.swift @@ -71,17 +71,29 @@ extension RecoveryScorer { // pairs in the SAME order recovery(...) appends them so the renormalization below matches. var terms: [(name: String, z: Double, w: Double)] = [] + // Resting-HR z, computed up front (like recovery()) so the saturation guard can read the + // HRV<->RHR coupling before the HRV term is built. nil when there is no RHR baseline. + let rhrZForGuard: Double? = rhrBaseline.map { zScore($0.baseline, mean: rhr, spread: $0.spread) } + // HRV term: higher is better. (Always present once usable; the cold-start guard above returned.) + // The low-HRV penalty is eased when resting HR corroborates parasympathetic saturation, using + // the SAME guard recovery() applies — so the trace's HRV term matches the scored one exactly. // L9: every WEIGHT / SCALE / centre constant goes through r2() too (not just the z-scores), so a // future non-round weight (e.g. 0.333) renders identically on Swift and Kotlin and the parity // fixture cannot silently desync. The values render the same as before today. - let hrvZ = zScore(hrv, mean: hrvBaseline.baseline, spread: hrvBaseline.spread) - terms.append(("hrv", hrvZ, wHRV)) - lines.append("charge term hrv z=\(r2(hrvZ)) w=\(r2(wHRV)) (higher HRV is better)") + let hrvZRaw = zScore(hrv, mean: hrvBaseline.baseline, spread: hrvBaseline.spread) + let sat = parasympatheticSaturation(hrvZ: hrvZRaw, rhrZ: rhrZForGuard) + terms.append(("hrv", sat.effectiveHrvZ, wHRV)) + lines.append("charge term hrv z=\(r2(sat.effectiveHrvZ)) w=\(r2(wHRV)) (higher HRV is better)") + // Name the eased penalty ONLY when the guard fired, so a "Charge looks high for low HRV" + // report shows exactly why: low HRV + low resting HR = benign vagal saturation, penalty eased. + if sat.active { + lines.append("charge saturation active hrvZraw=\(r2(hrvZRaw)) rhrZ=\(r2(rhrZForGuard ?? 0)) " + + "damp=\(r2(sat.dampFraction)) (low HRV + low resting HR: parasympathetic saturation, HRV penalty eased)") + } - // RHR term: lower is better -> (mu - x) / sigma. - if let b = rhrBaseline { - let z = zScore(b.baseline, mean: rhr, spread: b.spread) + // RHR term: lower is better -> (mu - x) / sigma. (Reuses the z computed for the guard.) + if let z = rhrZForGuard { terms.append(("rhr", z, wRHR)) lines.append("charge term rhr z=\(r2(z)) w=\(r2(wRHR)) (lower RHR is better)") } else { diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer.swift index e61d1f0fb2..5a119dc95e 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/RecoveryScorer.swift @@ -95,6 +95,94 @@ public enum RecoveryScorer { /// Sleep-performance scale (±2 z spans the normal range). public static let sleepPerfScale: Double = 0.12 + // MARK: - Parasympathetic-saturation guard (Charge) + // + // A low nightly HRV is normally a fatigue signal, so the HRV term (wHRV = 0.55, the dominant + // driver) pulls Charge DOWN. But there is a known BENIGN pattern the raw z-score mis-reads: + // parasympathetic (vagal) saturation. In a very fit or very relaxed state the RMSSD-HRV response + // saturates and reads LOW while resting HR reads LOW too, and the two DECOUPLE. That is not poor + // recovery — it is a saturated rest state — yet the dominant HRV penalty tanks Charge anyway. + // + // The discriminator is the SIGN of the resting-HR term versus the HRV term. recovery() builds + // the HRV term as (hrv - mu)/sigma (NEGATIVE when HRV is below baseline = the penalty) and the + // RHR term as (mu - rhr)/sigma (POSITIVE when resting HR is below baseline = recovery-good). In a + // normally-coupled autonomic response a low HRV travels WITH a HIGH resting HR (sympathetic + // drive), so both terms point the SAME way (down). When instead the HRV term points "bad" while + // the resting-HR term points "good", the expected HRV<->RHR coupling has broken — the signature + // of vagal saturation, not fatigue. In that regime the guard EASES (never removes) the low-HRV + // penalty; real fatigue (low HRV + HIGH resting HR) is left exactly as it was. + + /// Personal-sigma units BOTH the low-HRV arm and the low-resting-HR arm must clear before the + /// guard engages. 0.5 sigma keeps it OFF marginal / noise nights: a night must be clearly low on + /// HRV AND clearly low on resting HR (each at least half a personal standard deviation into its + /// saturation direction) before any easing applies. + public static let satEnterZ: Double = 0.5 + + /// Personal-sigma units at which the saturation signal reaches FULL strength. Between satEnterZ + /// and this the easing ramps linearly from 0 to satMaxDampFraction; at/above it the easing is + /// capped. 1.5 sigma means "both HRV and resting HR a clear 1.5 SD below baseline" is treated as + /// unambiguous saturation. + public static let satFullZ: Double = 1.5 + + /// Maximum fraction of the low-HRV penalty the guard will ever credit back. 0.5 = at most HALF + /// the penalty is eased, so a genuinely low-HRV night is NEVER scored as if HRV sat at baseline: + /// at least half of the low-HRV penalty always survives. This is the core conservatism knob. + public static let satMaxDampFraction: Double = 0.5 + + /// Outcome of the parasympathetic-saturation check for one night. + struct ParasympatheticSaturation: Equatable, Sendable { + /// Effective HRV z after easing. Equals the input hrvZ when the guard is inactive; when + /// active it is shrunk toward 0 (never past it), lightening the low-HRV penalty. + let effectiveHrvZ: Double + /// True iff the saturation regime was detected AND the penalty was actually eased. + let active: Bool + /// Fraction of the HRV penalty credited back, in [0, satMaxDampFraction]. 0 when inactive. + let dampFraction: Double + } + + /// Detect parasympathetic saturation from tonight's HRV and resting-HR z-scores and, when the + /// signature is present, ease the low-HRV penalty by shrinking its (negative) z toward 0. + /// + /// - Parameters: + /// - hrvZ: the HRV term as recovery() builds it, (hrv - mu)/sigma. Higher is better; a + /// NEGATIVE value means HRV is below baseline (the penalty direction). + /// - rhrZ: the resting-HR term as recovery() builds it, (mu - rhr)/sigma. Higher is better; a + /// POSITIVE value means resting HR is below baseline (the recovery-good direction). Pass nil + /// when there is no resting-HR baseline (no RHR term) — with nothing to corroborate the low + /// HRV, the guard refuses to guess and never fires. + /// - Returns: the (possibly eased) effective HRV z plus whether the guard fired. + static func parasympatheticSaturation(hrvZ: Double, rhrZ: Double?) -> ParasympatheticSaturation { + // Without a resting-HR term there is nothing to corroborate the low HRV: benign saturation + // and real fatigue are indistinguishable from HRV alone, so score the night as-is. + guard let rhrZ = rhrZ else { + return ParasympatheticSaturation(effectiveHrvZ: hrvZ, active: false, dampFraction: 0) + } + // Saturation SIGNATURE, in personal-sigma units: + // hrvLow > 0 <=> HRV below baseline (the penalty we might ease) + // rhrLow > 0 <=> resting HR below baseline (the corroborating "also low" signal) + // Both must clear satEnterZ, so only a clear low-HRV + low-RHR decoupling is ever eased and a + // low-HRV + HIGH-RHR fatigue night (rhrLow < 0) can never qualify. + let hrvLow = -hrvZ + let rhrLow = rhrZ + guard hrvLow >= satEnterZ, rhrLow >= satEnterZ else { + return ParasympatheticSaturation(effectiveHrvZ: hrvZ, active: false, dampFraction: 0) + } + // Strength is driven by the WEAKER of the two arms (min), normalized from the entry gate to + // satFullZ and clamped to [0, 1]. Using the weaker arm is deliberately conservative: full + // easing requires HRV clearly low AND resting HR clearly low; a very low HRV paired with only + // a mildly low resting HR is limited by the resting-HR arm. + let couplingStrength = min(hrvLow, rhrLow) + let s = max(0.0, min(1.0, (couplingStrength - satEnterZ) / (satFullZ - satEnterZ))) + let dampFraction = satMaxDampFraction * s + // Ease by shrinking the (negative) HRV z toward 0. (1 - dampFraction) stays in + // [1 - satMaxDampFraction, 1], so the penalty is only ever REDUCED — never removed, never + // sign-flipped into a bonus. + let effectiveHrvZ = hrvZ * (1.0 - dampFraction) + return ParasympatheticSaturation(effectiveHrvZ: effectiveHrvZ, + active: dampFraction > 0, + dampFraction: dampFraction) + } + /// Rolling-mean HR window (seconds) for the resting-HR estimate. public static let restingHRWindowS: Int = 5 * 60 @@ -330,13 +418,21 @@ public enum RecoveryScorer { var terms: [(z: Double, w: Double)] = [] - // HRV term: higher is better. + // Resting-HR z, computed up front: the parasympathetic-saturation guard needs to inspect the + // HRV<->RHR coupling BEFORE the dominant HRV term is folded in. nil when there is no RHR + // baseline (no RHR term), in which case the guard cannot fire. + let rhrZ: Double? = rhrBaseline.map { zScore($0.mean, mean: rhr, spread: $0.spread) } + + // HRV term: higher is better. When resting HR corroborates a benign parasympathetic-saturation + // pattern (HRV low AND resting HR low, decoupled), the low-HRV penalty is eased — never on a + // low-HRV + HIGH-resting-HR fatigue night. See `parasympatheticSaturation`. if let b = hrvBaseline { - terms.append((zScore(hrv, mean: b.mean, spread: b.spread), wHRV)) + let hrvZ = zScore(hrv, mean: b.mean, spread: b.spread) + terms.append((parasympatheticSaturation(hrvZ: hrvZ, rhrZ: rhrZ).effectiveHrvZ, wHRV)) } - // RHR term: lower is better → (μ − x) / σ. - if let b = rhrBaseline { - terms.append((zScore(b.mean, mean: rhr, spread: b.spread), wRHR)) + // RHR term: lower is better → (μ − x) / σ. Unchanged by the guard (reuses the z above). + if let z = rhrZ { + terms.append((z, wRHR)) } // Resp term: lower is better, optional. if let r = resp, let b = respBaseline { diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/ScoreConfidence.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/ScoreConfidence.swift index 5188bc1d43..6c7b2f0217 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/ScoreConfidence.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/ScoreConfidence.swift @@ -34,6 +34,15 @@ public enum ScoreConfidence: String, Equatable, Sendable, Codable { return b.trusted ? .solid : .building } + /// Readiness confidence from the HRV/RHR baseline density backing the read (readiness is HRV-led). + /// - calibrating: no read (insufficient history — the readiness level is `.insufficient`). + /// - solid: a read exists AND the full baseline window is present. + /// - building: a read exists but the baseline is shorter than the full window (e.g. 7–29 of 30). + public static func readiness(hasRead: Bool, baselineNights: Int, fullWindow: Int) -> ScoreConfidence { + guard hasRead else { return .calibrating } + return baselineNights >= fullWindow ? .solid : .building + } + /// Effort (strain) confidence. /// - calibrating: no score (no usable HR window) → absent. /// - solid: a score exists AND the HR window is dense (≥ solidReadings samples). diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepDebt.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepDebt.swift index 6d711dca63..9c92174e1e 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepDebt.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepDebt.swift @@ -1,24 +1,56 @@ import Foundation -// SleepDebt.swift — a rolling sleep-debt ledger over the last N nights. +// SleepDebt.swift — a recency-weighted, capped, asymmetric sleep-debt ledger. // // Pure, deterministic, DB-free. Given a chronological series of per-night total -// sleep (minutes) and a personal sleep need (hours), it accumulates a running -// balance of (actual − need) per night across a capped trailing window (14 nights -// by default) and reports the net balance plus the per-night deltas that make it -// up. +// sleep (minutes) and a personal sleep need (hours), it walks the recent nights +// oldest → newest and folds each night's (slept − need) into a running balance +// with three principled properties — recency weighting, a cap, and accrual/repay +// asymmetry — then reports the net balance plus the raw per-night deltas. +// +// The recurrence (oldest → newest over the counted, windowed nights, seeded at 0): +// +// contribution = delta < 0 ? delta // a DEFICIT accrues in FULL +// : delta * surplusRepayFraction // a SURPLUS repays only partly +// balance = recencyRetention * balance + contribution // yesterday's balance DECAYS +// balance = clamp(balance, −cap, +cap) // cap = maxBalanceNights * need +// +// where delta = sleptMin − needMin (positive = surplus, negative = deficit). +// +// Why each term (vs the old flat Σ(slept − need)): +// - RECENCY WEIGHTING. `recencyRetention` (< 1) multiplies the carried balance each +// night, so a night's imprint decays geometrically: its weight `k` counted nights +// later is retentionᵏ. A fortnight-old deficit no longer sits at full weight the way +// a flat sum held it. At the 0.90 default the half-life is ln0.5/ln0.90 ≈ 6.6 nights, +// so across the 14-night window the oldest night carries ≈ 0.90¹³ ≈ 0.25 weight. +// - CAP. Even decayed, a sustained deficit converges toward need/(1−retention) ≈ 10 +// nights' worth — still an absurd headline. `maxBalanceNights` clamps |balance| to a +// few nights of need (a person-scaled ceiling), applied every step so the running +// value never transiently blows past it either. The cap only binds under severe +// chronic deprivation; realistic mild deficits sit well under it. +// - ASYMMETRY. Two mechanisms, both grounded in sleep homeostasis: (1) a deficit accrues +// in full but a surplus repays at `surplusRepayFraction` (< 1) — you cannot fully +// "catch up" by oversleeping one night; (2) the retention decay itself repays debt over +// time and prevents surplus from banking indefinitely. So an oversleep night moves the +// balance strictly less than an equal-magnitude short night does. // // HONEST by construction: -// - It is a plain debt accumulator — sum of nightly (slept − need) — NOT a -// physiological model. A surplus night (slept > need) genuinely offsets a -// deficit one, the same way a checking balance nets credits and debits. -// - The window is capped (default 14) so "debt" never compounds indefinitely -// across months of history — only the recent fortnight is in scope. -// - Nights with no usable sleep total are SKIPPED entirely (no zero-fill), so a -// gap in wear never reads as a full night of debt. -// - The need value is supplied by the caller (AnalyticsEngine.Rest.defaultNeedHours -// = 8.0 by default; the caller passes any personal override). Computation here -// stays a pure function of (series, need, window). +// - The per-night `deltaMin` stored on each `SleepDebtNight` stays the RAW signed +// (slept − need), so the UI's diverging per-night bars remain a faithful nightly read. +// Only the accumulated `balanceMin` applies the decay/asymmetry/cap — it is deliberately +// NOT the plain Σ of the deltas any more (the model is the point). +// - The trailing WINDOW (default 14) still caps how many counted nights are folded in; +// decay and window together bound history. +// - Nights with no usable sleep total are SKIPPED entirely (no zero-fill), so a gap in +// wear never reads as a full night of debt, and decay is applied per COUNTED night (a +// measurement gap does not silently "recover" the debt). +// - The need value is supplied by the caller (AnalyticsEngine.Rest.defaultNeedHours = 8.0 +// by default; the caller passes any personal override — see SleepView, which now feeds +// the population-anchored personalizedNeedHours rather than a self-referential mean). +// Computation here stays a pure function of (series, need, window). +// +// The three tuning constants (`recencyRetention`, `surplusRepayFraction`, `maxBalanceNights`) +// are first-cut, defensible defaults that WANT empirical tuning on real wearer histories. // // Constant-explicit + dependency-free so the Kotlin mirror (android … SleepDebt.kt) // is byte-identical. @@ -40,11 +72,15 @@ public struct SleepDebtNight: Equatable, Sendable { /// The rolling sleep-debt ledger over the capped trailing window. public struct SleepDebtLedger: Equatable, Sendable { - /// Net running balance (minutes) across the window: Σ(slept − need). Negative = - /// net DEBT (under-slept overall), positive = net SURPLUS, 0 = on target. + /// Net running balance (minutes): the recency-weighted, asymmetric, capped fold of the + /// per-night deltas (see `SleepDebt.ledger`). Negative = net DEBT (under-slept overall), + /// positive = net SURPLUS, 0 = on target. NOT the plain Σ of `nights[].deltaMin` — the + /// decay/asymmetry/cap are applied to the accumulation, the raw deltas are kept only for + /// the per-night bars. public let balanceMin: Double - /// Per-night contributions, oldest → newest, one per counted night (skipped - /// nights are absent). The `deltaMin` values are the per-night bar/spark. + /// Per-night contributions, oldest → newest, one per counted night (skipped nights are + /// absent). Each `deltaMin` is the RAW signed (slept − need) — the honest per-night + /// bar/spark — before the balance's decay/asymmetry/cap are applied. public let nights: [SleepDebtNight] /// Personal sleep need (minutes) the ledger was computed against (for labelling). public let needMin: Double @@ -64,13 +100,37 @@ public struct SleepDebtLedger: Equatable, Sendable { public enum SleepDebt { /// Cap the ledger at the trailing two weeks — recent enough to be actionable, - /// short enough that one rough patch doesn't read as months of compounding debt. + /// short enough that one rough patch doesn't read as months of compounding debt. With the + /// recency decay layered on top, the window is a hard outer bound; the decay is what + /// actually tapers a night's weight within it. public static let defaultWindowNights: Int = 14 /// "On target" deadband (minutes): a |balance| under this reads as balanced rather /// than as a debt/surplus, so a few stray minutes don't flip the headline. public static let onTargetBandMin: Double = 30.0 + /// RECENCY WEIGHTING. Per-night retention of the carried balance: each night the running + /// balance is multiplied by this before the night's contribution is added, so a night's + /// weight decays geometrically (retentionᵏ, k counted nights later). 0.90 ⇒ 10 %/night + /// decay, a ≈ 6.6-night half-life (ln0.5/ln0.90), and ≈ 0.25 weight on the oldest night of + /// a 14-night window. Lower = shorter memory (only the last few nights matter); higher = + /// longer memory (closer to the old flat sum). WANTS empirical tuning. Range (0, 1). + public static let recencyRetention: Double = 0.90 + + /// ASYMMETRY. A surplus (over-need) night repays debt at only this fraction of the weight a + /// deficit night accrues at — you can't fully "bank" catch-up sleep by oversleeping once. + /// 0.5 ⇒ an oversleep night moves the balance half as far as an equal short night. Combined + /// with `recencyRetention` (which also quietly repays debt over time), this is the ledger's + /// accrue-fast / repay-slow behaviour. WANTS empirical tuning. Range [0, 1]. + public static let surplusRepayFraction: Double = 0.5 + + /// CAP. |balance| is clamped to this many nights of the personal need (cap = value × needMin), + /// applied every step so the running total can't even transiently exceed it. Person-scaled so + /// a longer-sleeping teen's ceiling is proportionally higher. 3 ⇒ "at most ~three full nights + /// behind (or ahead)". Guards the absurd-number failure mode (a sustained deficit's decayed + /// series still converges toward ~10 nights' worth without it). WANTS empirical tuning. + public static let maxBalanceNights: Double = 3.0 + /// Sleep credited toward debt for one day. `mainSleepMin` remains the canonical /// main-night total used by Rest and the sleep headline; separately-recorded naps /// add repayment credit without changing that canonical figure. A day with no @@ -97,27 +157,37 @@ public enum SleepDebt { /// - window: how many of the most-recent COUNTED nights to include. Defaults to /// `defaultWindowNights` (14). Clamped to ≥ 1. /// - /// The balance is Σ over the window of (sleptMin − needMin): a surplus night - /// offsets a deficit one. Returns an empty ledger (balance 0, no nights) when no - /// night has usable data. + /// The balance folds the windowed nights oldest → newest: each night's carried balance + /// DECAYS by `recencyRetention`, a DEFICIT adds in full while a SURPLUS adds only + /// `surplusRepayFraction` of itself, and |balance| is clamped to `maxBalanceNights` × need + /// every step (see the file header for the full rationale). Each `SleepDebtNight.deltaMin` + /// keeps the RAW (slept − need) for the per-night bars. Returns an empty ledger (balance 0, + /// no nights) when no night has usable data. public static func ledger(series: [(day: String, totalSleepMin: Double?)], needHours: Double = AnalyticsEngine.Rest.defaultNeedHours, window: Int = defaultWindowNights) -> SleepDebtLedger { let needMin = max(needHours, 0.0) * 60.0 - let cap = max(window, 1) + let windowCap = max(window, 1) + // Person-scaled magnitude ceiling for the running balance (a few nights of need). + let balanceCap = maxBalanceNights * needMin // Keep only nights with usable sleep, preserving chronological order, then take - // the most-recent `cap` of them. + // the most-recent `windowCap` of them. let usable = series.filter { ($0.totalSleepMin ?? 0) > 0 } - let windowed = usable.suffix(cap) + let windowed = usable.suffix(windowCap) var nights: [SleepDebtNight] = [] nights.reserveCapacity(windowed.count) var balance = 0.0 for row in windowed { let slept = row.totalSleepMin ?? 0 - let delta = slept - needMin - balance += delta + let delta = slept - needMin // RAW signed per-night; kept for the bars + // Asymmetry: a deficit (delta < 0) accrues in full; a surplus repays only partly. + let contribution = delta < 0 ? delta : delta * surplusRepayFraction + // Recency weighting: yesterday's balance decays before today's contribution lands. + balance = recencyRetention * balance + contribution + // Cap: clamp every step so the running total never even transiently exceeds it. + balance = min(max(balance, -balanceCap), balanceCap) nights.append(SleepDebtNight(day: row.day, sleptMin: slept, deltaMin: delta)) } return SleepDebtLedger(balanceMin: round1(balance), nights: nights, needMin: needMin) diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift index 4778b8609e..4a26c27a6d 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift @@ -190,6 +190,82 @@ public enum SleepStager { /// while still dropping an all-day desk strap (≈100% gap) or a session genuinely spent off-wrist. public static let maxOffWristSleepFraction: Double = 0.5 + // MARK: - Wake-gap bridging (experimental, default OFF) + + // A brief mid-night get-up splits the night into separate stored sessions: the CENTERED 15-min + // rolling stillness window smears an ~8-min bathroom trip past mergeMin into an "active" run (a + // real MG night split at a 16-min seam, 06:17 end / 06:33 start), detection emits one stored + // session per still-run, and nothing at the detection layer re-merges them — the #561/#861 + // bridges only group sessions later, at selection/aggregation, so sub-minSleepMin sleep + // fragments between arousals are dropped by the ladder before those bridges ever see them. + // Industry convention (and the selector's own <60-min "same sleep period" threshold) treats a + // brief wake as belonging INSIDE the night. + // + // When enabled (the live call site threads PuffinExperiment.wakeBridgeEnabled, default OFF), + // `bridgeWakeGaps` assembles adjacent sleep runs separated by ≤ `wakeBridgeMaxMin` into ONE + // candidate run between the run spine and the gate ladder; the stager then scores the wake + // INSIDE the single session, and every downstream gate (minSleepMin, the H4 span cap, hrConfirm, + // off-wrist, the daytime/morning guards) still applies to the ASSEMBLED run. A merge that would + // assemble a span over `maxMainSleepSpanS` is REFUSED: without that guard a pathological chain + // would assemble and then be dropped WHOLE by the H4 cap, converting a partially-scored night + // into a zero-score night. + // + // HISTORY — an HR-led "rescue" of motion-vetoed runs was built beside this bridge and WITHDRAWN + // after a 5-night real-data replay against the reporting wearer's server mirror (2026-08-24). + // The rescue reclassified an "active" run as sleep when its median in-run HR sat inside the + // `confirmSleepWithHR` band, on the premise that a motion-vetoed night with HR 47–58 bpm was + // restless-but-asleep. On the real wearer the discriminator failed: a low-HR wearer's quiet + // awake HR (median ~58 bpm, awake-resting low-50s, RHR ~44) also sits inside the ×1.05 band off + // the ~64 window baseline, so whole awake evenings were rescued and chained — the assembled run + // then either exceeded the H4 cap and dropped the night to ZERO, or squeaked under it and + // scored a 15 h "night" starting at 18:00. A follow-up decode investigation (2026-08-24) then + // verified the MG gravity decode CLEAN (unit-norm throughout; the alarming server "postureVar" + // is a per-axis RANGE, not a variance) and showed the premise itself was partly wrong: on the + // flagship "dropped" night the strap's own band state read wake across the disputed hours WITH + // corroborating 70–75 bpm stretches and continuous posture change — likely genuine in-bed + // wakefulness the stillness spine was RIGHT to reject. Median HR LEVEL cannot separate that + // wearer's quiet wakefulness from sleep; only dip DEPTH (sleep-only minima 42–48 bpm) looked + // separable, and placing an onset BOUNDARY from dips needs a properly validated design + // (#194/#345) plus per-night ground truth that HR alone cannot supply. Do not re-land an + // HR-band rescue without multi-night evidence that it refuses quiet awake stretches on low-HR + // wearers. + + /// Adjacent sleep runs separated by at most this many minutes are assembled into ONE candidate + /// run (wake-bridge pass only). Below the selector-side `SleepStageTotals.gapBridgeMaxMin` (60) + /// so selection/aggregation behaviour is unchanged for anything this bridge leaves split; well + /// above the ~16-min smeared seam a brief real get-up produces. + public static let wakeBridgeMaxMin: Int = 45 + + /// Assemble adjacent sleep runs separated by at most `wakeBridgeMaxMin` minutes (whatever lies + /// between — an active run, or a plain data hole) into one candidate run; the intervening runs + /// are absorbed and the stager later scores the wake INSIDE the single session. A gap over the + /// threshold, a negative/overlapping gap (pinned `gap >= 0` semantics, matching + /// `bridgeSparseSleep`), or a merge whose assembled span would exceed `maxMainSleepSpanS` (the + /// H4 guard above) never bridges. Pure + deterministic; mirrors Kotlin `bridgeWakeGaps`. + static func bridgeWakeGaps(_ periods: [Period], + traceSink: ((String) -> Void)? = nil) -> [Period] { + let bridgeS = wakeBridgeMaxMin * 60 + var out: [Period] = [] + for p in periods { + if p.stage == "sleep", + let j = out.lastIndex(where: { $0.stage == "sleep" }), + p.start - out[j].end >= 0, p.start - out[j].end <= bridgeS, + max(out[j].end, p.end) - out[j].start <= maxMainSleepSpanS { + let gapMin = (p.start - out[j].end) / 60 + if gapMin > 0 { + traceSink?(GateTrace.runLine(index: -1, startTs: out[j].end, endTs: p.start, + verdict: .kept, gate: "wakeBridge", + detail: "gapMin=\(gapMin) max=\(wakeBridgeMaxMin)")) + } + out[j] = Period(stage: "sleep", start: out[j].start, end: max(out[j].end, p.end)) + out.removeSubrange((j + 1)...) + } else { + out.append(p) + } + } + return out + } + /// Minimum average HR-stream density for the off-wrist HR-gap proxy to be trusted (#507). The proxy /// reads a >`offWristHRGapMin`-minute hole in HR as "off the wrist" — valid only when HR is otherwise /// dense (live 5/MG, or a worn night with continuous HR), so a real gap is anomalous. A WHOOP 4.0's @@ -849,6 +925,14 @@ public enum SleepStager { wristOff: [(start: Int, end: Int)] = [], bandSleepState: [(ts: Int, state: Int)] = [], useSleepStagerV2: Bool = false, + // Wake-gap bridging (experimental, default OFF — see the section + // comment above `wakeBridgeMaxMin`). When true, adjacent sleep runs + // across ≤ wakeBridgeMaxMin wake gaps are assembled into one + // candidate run before the gate ladder, so a brief get-up stays + // INSIDE one session as staged wake. The default keeps every + // existing caller byte-identical; the live call site threads + // PuffinExperiment.wakeBridgeEnabled. + wakeBridge: Bool = false, sleepHRBaseline: Double? = nil, traceSink: ((String) -> Void)? = nil) -> [SleepSession] { // Sleep & Rest test mode only: when a trace is requested we MUST run the live ladder, not a @@ -860,6 +944,7 @@ public enum SleepStager { return detectSleepUncached(hr: hr, rr: rr, resp: resp, gravity: gravity, tzOffsetSeconds: tzOffsetSeconds, wristOff: wristOff, bandSleepState: bandSleepState, useSleepStagerV2: useSleepStagerV2, + wakeBridge: wakeBridge, sleepHRBaseline: sleepHRBaseline, traceSink: traceSink) } // v7.0.2 perf (#707): the single heaviest analytics call — it sorts the dense full-day gravity @@ -879,11 +964,13 @@ public enum SleepStager { wristOff: StreamFingerprint.of(wristOff, ts: { $0.start }, quant: { $0.end }), band: StreamFingerprint.of(bandSleepState, ts: { $0.ts }, quant: { $0.state }), v2: useSleepStagerV2, + bridge: wakeBridge, sleepHRBaseline: sleepHRBaseline) return detectSleepCache.value(key) { detectSleepUncached(hr: hr, rr: rr, resp: resp, gravity: gravity, tzOffsetSeconds: tzOffsetSeconds, wristOff: wristOff, bandSleepState: bandSleepState, useSleepStagerV2: useSleepStagerV2, + wakeBridge: wakeBridge, sleepHRBaseline: sleepHRBaseline, traceSink: nil) } } @@ -894,6 +981,7 @@ public enum SleepStager { let tz: Int let wristOff: StreamFingerprint; let band: StreamFingerprint let v2: Bool + let bridge: Bool let sleepHRBaseline: Double? } /// ≈ the number of distinct days in a scoring window; FIFO-evicted, holds only small session arrays. @@ -908,6 +996,7 @@ public enum SleepStager { wristOff: [(start: Int, end: Int)], bandSleepState: [(ts: Int, state: Int)], useSleepStagerV2: Bool, + wakeBridge: Bool = false, sleepHRBaseline: Double? = nil, traceSink: ((String) -> Void)? = nil) -> [SleepSession] { let grav = gravity.sorted { $0.ts < $1.ts } @@ -959,6 +1048,15 @@ public enum SleepStager { } } + // Wake-gap bridge (experimental, default OFF — see the section comment above + // `wakeBridgeMaxMin`): assemble one candidate run per sleep period across short + // (≤ wakeBridgeMaxMin) wake gaps, BEFORE the ladder — so sub-minSleepMin fragments between + // arousals survive as part of the night instead of being dropped, and a brief get-up stays + // INSIDE one session as staged wake. Every gate below still applies to the assembled runs. + if wakeBridge { + runs = bridgeWakeGaps(runs, traceSink: traceSink) + } + let minSleepS = minSleepMin * 60 var sessions: [SleepSession] = [] diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStagerV2.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStagerV2.swift index c3faf09258..7260ae37a5 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStagerV2.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStagerV2.swift @@ -193,11 +193,11 @@ public enum SleepStagerV2 { /// Measured on one wearer's 36 recorded nights, against the strap's own band `sleep_state` (an /// independent reference the recipe cannot contaminate — 21 nights, 15 554 epochs): sleep/wake kappa /// 0.105 → 0.118 and wake sensitivity 16.0 % → 17.6 %, with the healthy-stratum wake fraction essentially - /// unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out). Confirmed afterwards against - /// human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / 26 773 epochs, #991): 4-class kappa - /// 0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → 30.84 %, and the #437 stage-fraction - /// guard holds against truth as well. n = 1 wearer for the band figures; see `Tools/SleepBench` and the - /// PR for the full ablation and its limits. + /// unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out) and first-REM latency MAE 53.9 → 41.6 + /// min. Confirmed afterwards against human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / + /// 26 773 epochs, #991): 4-class kappa 0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → + /// 30.84 %, and the #437 stage-fraction guard holds against truth as well. n = 1 wearer for the band + /// figures; see `Tools/SleepBench` and the PR for the full ablation and its limits. static let transition: [String: [String: Double]] = [ "deep": ["deep": 0.86, "rem": 0.007, "light": 0.126, "awake": 0.007], "rem": ["deep": 0.005, "rem": 0.88, "light": 0.10, "awake": 0.015], diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BaselinesTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BaselinesTests.swift index 5847831eaf..418bd97420 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BaselinesTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BaselinesTests.swift @@ -328,4 +328,26 @@ final class BaselinesTests: XCTestCase { let hardDay = Baselines.deviation(85.0, state: s) XCTAssertGreaterThan(hardDay.z, 0, "a much-harder-than-normal day must read ABOVE the effort baseline") } + + // MARK: - sigma() + daytime configs (DaytimeStress baseline-relative mode) + + /// `sigma(_:)` is a pure extraction of `deviation()`'s internal conversion — must match it + /// exactly and stay internally consistent (deviation at baseline + sigma is z == 1.0). + func testSigmaMatchesDeviationsInternalConversion() { + let s = Baselines.foldHistory(Array(repeating: 50.0, count: 14), cfg: Baselines.hrvCfg) + let sigma = Baselines.sigma(s) + XCTAssertEqual(sigma, max(1.253 * s.spread, 1e-9), accuracy: 1e-9) + let dev = Baselines.deviation(s.baseline + sigma, state: s) + XCTAssertEqual(dev.z, 1.0, accuracy: 1e-6) + } + + /// The new daytime_hr / daytime_rmssd configs exist, are reachable via both the dictionary + /// and the convenience accessor, and are distinct from the nightly resting_hr/hrv configs + /// they sit alongside (different bounds/floor — daytime HR runs warmer than nocturnal RHR). + func testDaytimeConfigsExistAndAreDistinctFromNightlyConfigs() { + XCTAssertEqual(Baselines.metricCfg["daytime_hr"], Baselines.daytimeHRCfg) + XCTAssertEqual(Baselines.metricCfg["daytime_rmssd"], Baselines.daytimeRMSSDCfg) + XCTAssertNotEqual(Baselines.daytimeHRCfg, Baselines.restingHRCfg) + XCTAssertNotEqual(Baselines.daytimeRMSSDCfg, Baselines.hrvCfg) + } } diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/CrossDeviceHRVTrendTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/CrossDeviceHRVTrendTests.swift new file mode 100644 index 0000000000..6c8076ce0d --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/CrossDeviceHRVTrendTests.swift @@ -0,0 +1,57 @@ +import XCTest +@testable import StrandAnalytics + +/// CrossDeviceHRVTrend — stitch a full HRV history across a device switch (Oura → WHOOP) into one +/// continuous trend: per-era-relative shape (continuous through the switch) + raw values (scale-honest +/// per device) + boundary markers. Scoring stays per-device; this is the display/trend lens. +final class CrossDeviceHRVTrendTests: XCTestCase { + + private func sd(_ day: String, _ src: String, _ hrv: Double) -> (day: String, sourceId: String, hrv: Double) { + (day: day, sourceId: src, hrv: hrv) + } + + func testSingleBrandHistoryIsOneEraNoBoundary() { + let rows = (1...20).map { sd(String(format: "2026-06-%02d", $0), "my-whoop", 90 + Double($0 % 5)) } + let r = CrossDeviceHRVTrend.build(rows) + XCTAssertEqual(r.eras.count, 1) + XCTAssertEqual(r.eras[0].brand, "whoop") + XCTAssertFalse(r.points.contains { $0.isEraStart }, "a single-brand history has no device-switch boundary") + } + + func testOuraToWhoopSwitchMarksBoundaryAndNormalisesPerEra() { + // Oura era ~131 ms, then a WHOOP era ~91 ms — incompatible scales, no overlap. + var rows: [(day: String, sourceId: String, hrv: Double)] = [] + for i in 1...15 { rows.append(sd(String(format: "2026-05-%02d", i), "oura-api", 130 + Double(i % 4))) } + for i in 1...15 { rows.append(sd(String(format: "2026-06-%02d", i), "my-whoop-noop", 90 + Double(i % 4))) } + let r = CrossDeviceHRVTrend.build(rows) + XCTAssertEqual(r.eras.map { $0.brand }, ["oura", "whoop"]) + XCTAssertEqual(r.eras[0].nights, 15) + XCTAssertEqual(r.eras[1].nights, 15) + // Exactly one boundary, on the first WHOOP night — never on the very first night of history. + let starts = r.points.filter { $0.isEraStart } + XCTAssertEqual(starts.count, 1) + XCTAssertEqual(starts.first?.brand, "whoop") + XCTAssertEqual(starts.first?.day, "2026-06-01") + // Per-era normalisation: a typical WHOOP night reads ~0% vs its OWN era, NOT "~31% below" the Oura scale. + let whoopMid = r.points.first { $0.day == "2026-06-08" }! + XCTAssertLessThan(abs(whoopMid.eraRelativePct), 5) + XCTAssertEqual(r.eras[0].centerMs, 131, accuracy: 3) + XCTAssertEqual(r.eras[1].centerMs, 91, accuracy: 3) + } + + func testEraRelativeShapeIsContinuousAcrossTheSwitch() { + // A declining Oura era continues declining on WHOOP: the era-relative series keeps falling across + // the boundary (continuous shape) though the raw scale steps down. + var rows: [(day: String, sourceId: String, hrv: Double)] = [] + for i in 1...20 { rows.append(sd(String(format: "2026-05-%02d", i), "oura-api", 150 - Double(i))) } + for i in 1...20 { rows.append(sd(String(format: "2026-06-%02d", i), "my-whoop", 100 - Double(i))) } + let r = CrossDeviceHRVTrend.build(rows) + XCTAssertLessThan(r.points.last { $0.brand == "oura" }!.eraRelativePct, 0) + XCTAssertLessThan(r.points.last!.eraRelativePct, 0, "the decline continues into the WHOOP era") + XCTAssertGreaterThan(r.eras[0].centerMs, r.eras[1].centerMs, "raw scale steps down at the switch") + } + + func testEmptyInput() { + XCTAssertTrue(CrossDeviceHRVTrend.build([]).points.isEmpty) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DailyStressEngineTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DailyStressEngineTests.swift new file mode 100644 index 0000000000..9b544ced4a --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DailyStressEngineTests.swift @@ -0,0 +1,135 @@ +import XCTest +@testable import StrandAnalytics +import WhoopStore + +/// DailyStressEngine — the redesigned 0–3 daily stress score: log-domain HRV, dual baselines (short +/// adaptive for the number + long reference for a sustained-shift readout), coupled RHR/HRV escalation +/// (Buchheit quadrant), and a smallest-worthwhile-change dead-zone. +final class DailyStressEngineTests: XCTestCase { + + private func dm(_ day: Int, hrv: Double?, rhr: Int?) -> DailyMetric { + DailyMetric(day: String(format: "2025-05-%02d", day), totalSleepMin: nil, efficiency: nil, + deepMin: nil, remMin: nil, lightMin: nil, disturbances: nil, restingHr: rhr, + avgHrv: hrv, recovery: nil, strain: nil, exerciseCount: nil, + spo2Pct: nil, skinTempDevC: nil, respRateBpm: nil) + } + + // MARK: - The coupling (the central design decision) + + func testHRVDownWithRHRUpReadsSympatheticStress() { + // Stable ~92ms / 50bpm baseline, today a clear HRV drop (66) AND RHR rise (60): the true + // autonomic-stress quadrant → high score. + var days: [DailyMetric] = [] + for i in 1...20 { days.append(dm(i, hrv: i % 2 == 0 ? 94 : 90, rhr: i % 2 == 0 ? 51 : 49)) } + days.append(dm(21, hrv: 66, rhr: 60)) + let s = DailyStressEngine.evaluate(days: days)! + XCTAssertEqual(s.quadrant, .sympatheticStress) + XCTAssertGreaterThan(s.score, DailyStressEngine.highBandFloor, "coupled HRV-down+RHR-up is high stress") + } + + func testHRVDownWithRHRLowIsSaturationAndScoresLowerThanCoupledStress() { + // THE coupling test: identical HRV drop (66), but RHR is LOW (44) not high. That's benign + // parasympathetic saturation, not stress — it must classify as saturation AND score clearly + // lower than the same HRV drop with RHR elevated. + func build(rhrToday: Int) -> [DailyMetric] { + var days: [DailyMetric] = [] + for i in 1...20 { days.append(dm(i, hrv: i % 2 == 0 ? 94 : 90, rhr: 50)) } + days.append(dm(21, hrv: 66, rhr: rhrToday)) + return days + } + let saturated = DailyStressEngine.evaluate(days: build(rhrToday: 44))! + let stressed = DailyStressEngine.evaluate(days: build(rhrToday: 60))! + XCTAssertEqual(saturated.quadrant, .parasympatheticSaturation) + XCTAssertLessThan(saturated.score, stressed.score, + "low HRV with LOW RHR (saturation) must read less stressed than low HRV with HIGH RHR") + } + + func testHRVAboveBaselineReadsRecovered() { + var days: [DailyMetric] = [] + for i in 1...20 { days.append(dm(i, hrv: 80, rhr: 52)) } + days.append(dm(21, hrv: 105, rhr: 48)) // HRV well above baseline, RHR below + let s = DailyStressEngine.evaluate(days: days)! + XCTAssertEqual(s.quadrant, .recovered) + XCTAssertLessThan(s.score, 1.5, "a recovered night reads below the 1.5 baseline") + } + + func testSWCDeadzoneKeepsAnOrdinaryNightNeutral() { + // Today sits right at the baseline → within the smallest-worthwhile-change band → ~1.5, not + // nudged off baseline by sub-threshold noise. + var days: [DailyMetric] = [] + for i in 1...20 { days.append(dm(i, hrv: i % 2 == 0 ? 91 : 89, rhr: 50)) } + days.append(dm(21, hrv: 90, rhr: 50)) + let s = DailyStressEngine.evaluate(days: days)! + XCTAssertEqual(s.quadrant, .balanced) + XCTAssertEqual(s.score, 1.5, accuracy: 0.2, "an ordinary night stays at ~baseline") + } + + // MARK: - The dual-baseline reframe (the point of the whole redesign) + + func testSustainedShiftNormalizesTheDailyScoreButFlagsChronicLoad() { + // A sustained autonomic shift: 60 nights at the long-term normal (120 ms / 46 bpm), then 14 + // recent nights settled at a suppressed level (72 ms / 55 bpm), today at that new level. + // The DAILY score must normalize (today ≈ its own recent baseline), while the CHRONIC readout + // must flag the sustained load (recent HRV well below the long-term normal, RHR above it). + var days: [DailyMetric] = [] + var d = 1 + for _ in 0..<60 { days.append(dm(d, hrv: 120, rhr: 46)); d += 1 } + for _ in 0..<14 { days.append(dm(d, hrv: 72, rhr: 55)); d += 1 } + days.append(dm(d, hrv: 72, rhr: 55)) // today: the new normal + let s = DailyStressEngine.evaluate(days: days)! + XCTAssertLessThan(s.score, DailyStressEngine.highBandFloor, + "today at its own recent (shifted) normal must NOT read high stress — the short baseline adapted") + let chronic = s.chronicShift! + XCTAssertGreaterThan(chronic.hrvPctBelowLongTerm, 5, + "the chronic readout must surface HRV suppressed vs the long-term normal") + XCTAssertGreaterThan(chronic.rhrBpmAboveLongTerm, 0, "RHR elevated vs long-term too") + XCTAssertTrue(chronic.isSustainedLoad, "both channels shifted in the load direction → sustained load") + XCTAssertGreaterThan(chronic.daysBelowBand, 5, "the shift has persisted for many nights") + } + + // MARK: - Robustness / degradation + + func testHRVOnlyWhenNoRHR() { + // No RHR anywhere → HRV-only scoring, no coupling, usedRHR false, still produces a score. + var days: [DailyMetric] = [] + for i in 1...20 { days.append(dm(i, hrv: 90, rhr: nil)) } + days.append(dm(21, hrv: 66, rhr: nil)) + let s = DailyStressEngine.evaluate(days: days)! + XCTAssertFalse(s.usedRHR) + XCTAssertGreaterThan(s.score, 1.5, "an HRV drop still reads as elevated when RHR is unavailable") + } + + func testInsufficientBaselineReturnsNil() { + let days = (1...4).map { dm($0, hrv: 90, rhr: 50) } + XCTAssertNil(DailyStressEngine.evaluate(days: days), "fewer than the minimum baseline nights → nil") + } + + func testGradualDeclineIsSurfacedByTheMedianLongReference() { + // A slow multi-week HRV decline (120 → 74 over 60 nights), today at the low end. A recency- + // WEIGHTED "long" baseline collapses onto the recent low and reads ~0% (the bug a real device's + // gradually-declining data exposed); the robust MEDIAN long reference captures the decline. + var days: [DailyMetric] = [] + for i in 0..<60 { + days.append(dm(i + 1, hrv: 120.0 - Double(i) * (46.0 / 59.0), rhr: 46)) // 120 → 74 linear + } + days.append(dm(61, hrv: 74, rhr: 46)) + let cs = DailyStressEngine.evaluate(days: days)!.chronicShift! + XCTAssertGreaterThan(cs.hrvPctBelowLongTerm, 10, + "the median long reference must surface a months-long decline that a recency-weighted long baseline would hide") + } + + func testMultipleDeviceRowsPerDayAreDedupedToOnePerDay() { + // Two source rows for the same day (Apple with only RHR, WHOOP with HRV+RHR): the richer row + // wins, and the baseline windows count DAYS not device-rows (else a "60-day" window shrinks). + var days: [DailyMetric] = [] + for i in 1...20 { + days.append(dm(i, hrv: nil, rhr: 55)) // apple-like: RHR only + days.append(dm(i, hrv: 90, rhr: 50)) // whoop-like: richer (HRV present) + } + days.append(dm(21, hrv: 66, rhr: 60)) + let s = DailyStressEngine.evaluate(days: days)! + // If dedup failed, the HRV baseline would still work but be built off far fewer than 20 days; + // the score should still classify a clear HRV-down + RHR-up night as stress. + XCTAssertEqual(s.quadrant, .sympatheticStress) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeBaselinesTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeBaselinesTests.swift new file mode 100644 index 0000000000..f89bcb8834 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeBaselinesTests.swift @@ -0,0 +1,234 @@ +import XCTest +@testable import StrandAnalytics +import WhoopProtocol + +/// Coverage for the CALLER-side daytime-baseline fold (DaytimeBaselines.swift) that feeds +/// `DaytimeStress.analyze(mode: .baselineRelative(hr:rmssd:))`: the per-day aggregate definition, the +/// trailing-history fold, the graceful-degradation `scoringMode` gate, and the fold→score round trip. +final class DaytimeBaselinesTests: XCTestCase { + + // MARK: - Fixtures (multi-day: day `d` is offset by d·86 400 s; local hour-of-day is preserved) + + /// One local day of HR: every hour in `hours` filled with `n` 1 Hz samples at a per-hour bpm. + /// `bpms` maps to `hours` in order. tz offset 0, so local hour-of-day == wall-clock hour. + private func dayHR(_ dayIndex: Int, hours: [Int], bpms: [Int], + n: Int = DaytimeStress.minHourHRSamples) -> [HRSample] { + let dayBase = dayIndex * 86_400 + var out: [HRSample] = [] + for (h, bpm) in zip(hours, bpms) { + let base = dayBase + h * 3_600 + out += (0.. [HRSample] { + dayHR(dayIndex, hours: Array(8...17), bpms: Array(repeating: bpm, count: 10)) + } + + /// R-R for one local hour with a controllable beat-to-beat jitter (drives RMSSD). + private func hourRR(_ dayIndex: Int, hour: Int, rrMs: Int, jitter: Int, n: Int = 120) -> [RRInterval] { + let base = dayIndex * 86_400 + hour * 3_600 + return (0.. [RRInterval] { + (8...17).flatMap { hourRR(dayIndex, hour: $0, rrMs: rrMs, jitter: jitter) } + } + + // MARK: - Per-day aggregate + + func testDayHRAggregateIsTheTenthPercentileOfWakingHourMeanHRs() { + // Ten waking hours (08…17) with a spread of constant per-hour HRs; each hour's MEAN HR is just + // its bpm. P10 of [60,62,64,66,68,70,72,74,76,78] via the shared linear-interp quantile is + // 60 + 0.9·(62−60) = 61.8 — the calm floor, well below the day's median. + let bpms = [60, 62, 64, 66, 68, 70, 72, 74, 76, 78] + let hr = dayHR(0, hours: Array(8...17), bpms: bpms) + let agg = DaytimeStress.dayDaytimeAggregate(hr: hr, rr: [], tzOffsetSeconds: 0) + XCTAssertNotNil(agg.hr) + XCTAssertEqual(agg.hr!, 61.8, accuracy: 1e-6) + XCTAssertNil(agg.rmssd, "no R-R → no daytime RMSSD aggregate") + } + + func testDayHRAggregateAppliesTheSameMinSamplesGateAsTheScorer() { + // An under-gate sparse hour at a very low bpm must NOT drag the P10 floor down — the scorer would + // never score it, so the aggregate must not reference it either. + let dense = dayHR(0, hours: Array(8...17), bpms: [60, 62, 64, 66, 68, 70, 72, 74, 76, 78]) + let sparse = dayHR(0, hours: [7], bpms: [40], n: DaytimeStress.minHourHRSamples - 1) + let withSparse = DaytimeStress.dayDaytimeAggregate(hr: dense + sparse, rr: [], tzOffsetSeconds: 0) + let denseOnly = DaytimeStress.dayDaytimeAggregate(hr: dense, rr: [], tzOffsetSeconds: 0) + XCTAssertEqual(withSparse.hr!, denseOnly.hr!, accuracy: 1e-9, + "a below-gate hour leaked into the daytime-HR floor") + } + + func testDayHRAggregateExcludesNonWakingHours() { + // A very low 03:00 hour (outside 06:00–22:00) must not become the calm floor. + let waking = dayHR(0, hours: Array(8...17), bpms: [60, 62, 64, 66, 68, 70, 72, 74, 76, 78]) + let night = dayHR(0, hours: [3], bpms: [45]) + let withNight = DaytimeStress.dayDaytimeAggregate(hr: waking + night, rr: [], tzOffsetSeconds: 0) + XCTAssertEqual(withNight.hr!, 61.8, accuracy: 1e-6, + "a non-waking hour leaked into the daytime-HR floor") + } + + func testDayRMSSDAggregateIsPresentWithRRAndTracksVariability() { + // A relaxed day (high jitter → high RMSSD) vs a suppressed day (low jitter → low RMSSD): both + // produce a non-nil median RMSSD aggregate, and the relaxed day's is clearly higher. + let hr = flatDayHR(0, bpm: 65) + let relaxed = DaytimeStress.dayDaytimeAggregate(hr: hr, rr: flatDayRR(0, rrMs: 900, jitter: 45), + tzOffsetSeconds: 0) + let suppressed = DaytimeStress.dayDaytimeAggregate(hr: hr, rr: flatDayRR(0, rrMs: 900, jitter: 5), + tzOffsetSeconds: 0) + XCTAssertNotNil(relaxed.rmssd) + XCTAssertNotNil(suppressed.rmssd) + XCTAssertGreaterThan(relaxed.rmssd!, suppressed.rmssd!, + "the higher-variability day should carry a higher median daytime RMSSD") + } + + // MARK: - Fold trailing history + + func testFoldConvergesHRBaselineToThePersonalDaytimeFloor() { + // Twelve identical days whose P10 daytime HR is exactly 65 → the EWMA center converges to 65 and + // the baseline is usable (12 ≥ minNightsSeed). No R-R anywhere → RMSSD baseline is nil. + let days = (0..<12).map { DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), rr: [], + tzOffsetSeconds: 0) } + let b = DaytimeStress.foldDaytimeBaselines(days: days) + XCTAssertEqual(b.hr.baseline, 65.0, accuracy: 1e-6) + XCTAssertTrue(b.hr.usable) + XCTAssertNil(b.rmssd, "no daytime R-R history → HR-only baseline") + } + + func testFoldReturnsRMSSDBaselineOnlyWhenEnoughDaytimeRRDays() { + // Two days of R-R is below minNightsSeed → RMSSD baseline withheld (nil → HR-only). Eight days + // clears the seed → a usable RMSSD baseline is returned. + let twoDays = (0..<2).map { + DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), + rr: flatDayRR($0, rrMs: 900, jitter: 40), tzOffsetSeconds: 0) + } + XCTAssertNil(DaytimeStress.foldDaytimeBaselines(days: twoDays).rmssd, + "an untrustworthy 2-day RMSSD baseline must be withheld") + + let eightDays = (0..<8).map { + DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), + rr: flatDayRR($0, rrMs: 900, jitter: 40), tzOffsetSeconds: 0) + } + let rmssd = DaytimeStress.foldDaytimeBaselines(days: eightDays).rmssd + XCTAssertNotNil(rmssd, "eight days of daytime R-R should yield a usable RMSSD baseline") + XCTAssertTrue(rmssd!.usable) + } + + // MARK: - scoringMode graceful degradation + + func testScoringModeFallsBackToDayRelativeOnColdStartAndSparseHistory() { + // No history → day-relative (cold start unchanged). + guard case .dayRelative = DaytimeStress.scoringMode(history: []) else { + return XCTFail("empty history must fall back to .dayRelative") + } + // Three days is below minNightsSeed → still day-relative. + let threeDays = (0..<3).map { DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), rr: [], + tzOffsetSeconds: 0) } + guard case .dayRelative = DaytimeStress.scoringMode(history: threeDays) else { + return XCTFail("sub-seed history must fall back to .dayRelative") + } + } + + func testScoringModeUsesBaselineRelativeOnceHistoryIsUsable() { + // Six days of real daytime HR clears the seed → the Oura-style baseline-relative mode, carrying + // the folded HR baseline (~65) and a nil RMSSD baseline (no R-R history here). + let days = (0..<6).map { DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), rr: [], + tzOffsetSeconds: 0) } + guard case let .baselineRelative(hr, rmssd) = DaytimeStress.scoringMode(history: days) else { + return XCTFail("usable history must select .baselineRelative") + } + XCTAssertEqual(hr.baseline, 65.0, accuracy: 1e-6) + XCTAssertNil(rmssd) + } + + // MARK: - Fold → score round trip (the whole point) + + func testFoldedBaselineScoresElevationAgainstThePersonalFloorNotTheDaysOwnHours() { + // Personal floor folded from history = 65 bpm. TODAY is elevated ALL day (every waking hour 80 = + // 65 + the validated 15 bpm margin). A DAY-relative read would anchor on the day's OWN hours and + // miss an all-day-high day; the baseline-relative read scores it against the personal 65 floor, + // so every hour lands at/above the HIGH band and the day logs high-stress minutes. + let history = (0..<20).map { DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), rr: [], + tzOffsetSeconds: 0) } + let mode = DaytimeStress.scoringMode(history: history) + guard case .baselineRelative = mode else { return XCTFail("expected baseline-relative") } + + let today = flatDayHR(0, bpm: 80) + let r = DaytimeStress.analyze(hr: today, rr: [], tzOffsetSeconds: 0, mode: mode) + XCTAssertFalse(r.scored.isEmpty) + XCTAssertTrue(r.hrOnlyFallback, "no RMSSD baseline → HR-only, honestly flagged") + XCTAssertGreaterThan(r.highStressMinutes, 0, "an all-day-elevated day must log high-stress time") + for p in r.scored { + XCTAssertGreaterThanOrEqual(p.level!, DaytimeStress.highBandFloor - 0.05, + "80 bpm is the personal floor + the validated 15 bpm margin → the HIGH band") + } + } + + func testFoldedBaselineCalmDayAtThePersonalFloorReadsNeutralNotHigh() { + // TODAY sits exactly at the personal 65 floor all day → each hour reads ~1.5 (neutral), never + // high. Guards the calm end of the scale end-to-end through the fold. + let history = (0..<20).map { DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), rr: [], + tzOffsetSeconds: 0) } + let mode = DaytimeStress.scoringMode(history: history) + let r = DaytimeStress.analyze(hr: flatDayHR(0, bpm: 65), rr: [], tzOffsetSeconds: 0, mode: mode) + XCTAssertEqual(r.highStressMinutes, 0) + for p in r.scored { XCTAssertEqual(p.level!, 1.5, accuracy: 0.05) } + } + + func testMedianRMSSDBaselineKeepsANormalHourNeutralNoStacking() { + // The anti-stacking guarantee behind anchoring the RMSSD aggregate at the MEDIAN (not a calm + // ceiling): with BOTH a HR baseline (65) and an RMSSD baseline folded from history, a TODAY that + // sits at the HR floor AND at its typical daytime HRV must stay out of the HIGH band. A ceiling + // anchor would make this ordinary hour read ~1–2σ "below baseline" and stack a false stress term. + let history = (0..<20).map { + DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), + rr: flatDayRR($0, rrMs: 900, jitter: 40), tzOffsetSeconds: 0) + } + // Build the RMSSD-active mode DIRECTLY from the fold, bypassing the `scoringMode` gate that + // holds RMSSD out of LIVE scoring until validated (`DaytimeStress.daytimeRMSSDScoringEnabled` + // is false). This test pins the anti-stacking guarantee of the RMSSD-active scoring path + // itself, so it must exercise that path regardless of the production gate. `scoringMode`'s + // own gating is covered by `testScoringModeHoldsRMSSDOutOfLiveScoreWhileGated` below. + let baselines = DaytimeStress.foldDaytimeBaselines(days: history) + XCTAssertNotNil(baselines.rmssd, "20 days of daytime R-R should yield a usable RMSSD baseline") + let mode = DaytimeStress.ScoringMode.baselineRelative(hr: baselines.hr, rmssd: baselines.rmssd) + + // Today: HR at the floor, HRV at the SAME typical variability as history. + let today = flatDayHR(0, bpm: 65) + let todayRR = flatDayRR(0, rrMs: 900, jitter: 40) + let r = DaytimeStress.analyze(hr: today, rr: todayRR, tzOffsetSeconds: 0, mode: mode) + XCTAssertFalse(r.hrOnlyFallback, "an RMSSD baseline was supplied") + XCTAssertEqual(r.highStressMinutes, 0, "a typical HR+HRV day must not read as high stress") + for p in r.scored { + XCTAssertLessThan(p.level!, DaytimeStress.highBandFloor, + "a normal hour with an RMSSD baseline must not stack into the HIGH band") + } + } + + func testScoringModeHoldsRMSSDOutOfLiveScoreWhileGated() { + // The live-scoring gate: even with a full daytime-RR history that folds a USABLE RMSSD + // baseline, `scoringMode` (the production entry point) keeps RMSSD OUT of the mode while + // `DaytimeStress.daytimeRMSSDScoringEnabled` is false — so production scores HR-only, the + // validated r≈0.6 channel. The fold still produces the usable baseline (the machinery is + // live + tested); only its arrival at the score is gated. Flip the flag → this expectation + // inverts, which is the intended tripwire for when RMSSD is validated on. + let history = (0..<20).map { + DaytimeStress.DaytimeDayStreams(hr: flatDayHR($0, bpm: 65), + rr: flatDayRR($0, rrMs: 900, jitter: 40), tzOffsetSeconds: 0) + } + XCTAssertNotNil(DaytimeStress.foldDaytimeBaselines(days: history).rmssd, + "precondition: the history must fold a usable RMSSD baseline") + guard case let .baselineRelative(hr, rmssd) = DaytimeStress.scoringMode(history: history) else { + return XCTFail("usable HR history must still select .baselineRelative") + } + XCTAssertTrue(hr.usable, "HR baseline still drives the mode") + if DaytimeStress.daytimeRMSSDScoringEnabled { + XCTAssertNotNil(rmssd, "flag on: the usable RMSSD baseline should reach live scoring") + } else { + XCTAssertNil(rmssd, "flag off: RMSSD is held out of the live score despite a usable baseline") + } + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeStressTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeStressTests.swift index dd72a66aa2..2e5d2395a1 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeStressTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/DaytimeStressTests.swift @@ -132,4 +132,254 @@ final class DaytimeStressTests: XCTestCase { let base = hour * 3_600 return (0.. NOT sustained + let r = DaytimeStress.analyze(hr: hr, rr: []) + + XCTAssertFalse(r.sustainedHigh, "the trailing hour is calm, so sustained-high must not fire") + let expectedHighHours = r.scored.filter { $0.level! >= DaytimeStress.highBandFloor }.count + XCTAssertGreaterThan(expectedHighHours, 0, "the isolated morning spike should read as high band") + XCTAssertEqual(r.highStressMinutes, expectedHighHours * (DaytimeStress.bucketSeconds / 60)) + XCTAssertFalse(r.hrOnlyFallback, "day-relative mode never sets the baseline-relative fallback flag") + } + + // MARK: - Baseline-relative mode (Oura-style, vs a PERSONAL rolling baseline) + // + // Fixtures below use a 65 bpm personal HR baseline (matching the ~65 bpm pooled + // 10th-percentile figure from the validated 26-day Oura-reference correlation — see + // `DaytimeStress.baselineRelativeHighMarginBPM`) and elevations measured from it in terms of + // that validated ~15 bpm margin, so the expected band crossings are exact, not approximate. + + func testMarginToSigmaLandsExactlyOnBand() { + // The validated 15 bpm margin over baseline must land EXACTLY on highBandFloor (2.0) on + // the shared squash curve — the core identity `.baselineRelative` scoring relies on. + let sd = DaytimeStress.marginToSigma(marginBPM: DaytimeStress.baselineRelativeHighMarginBPM, + atBand: DaytimeStress.highBandFloor) + XCTAssertEqual(DaytimeStress.squash(DaytimeStress.baselineRelativeHighMarginBPM / sd), + DaytimeStress.highBandFloor, accuracy: 1e-9) + } + + func testBaselineRelativeModeRecoversMultipleInjectedElevations() { + // Personal daytime-HR baseline: 20 constant "days" at 65 bpm converges the EWMA center + // to exactly 65 (spread is folded but NOT used for the HR high-band threshold — see + // baselineRelativeHighMarginBPM). + let hrBaseline = Baselines.foldHistory(Array(repeating: 65.0, count: 20), cfg: Baselines.daytimeHRCfg) + XCTAssertEqual(hrBaseline.baseline, 65.0, accuracy: 1e-6) + + // FOUR distinct injected HR elevations across the SAME day's waking hours — the repo's + // derived-signal rule (CLAUDE.md "validate against the artifact, not one match") requires + // recovering MULTIPLE injected values, not a single high-vs-low pair. 65 (at baseline), + // 72 (+7, mild), 80 (+15, exactly the validated margin), 95 (+30, well past it). + let levels: [(hour: Int, bpm: Int)] = [(8, 65), (10, 72), (13, 80), (16, 95)] + var hr: [HRSample] = [] + for (h, bpm) in levels { hr += hourHR(h, bpm: bpm) } + + let r = DaytimeStress.analyze(hr: hr, rr: [], mode: .baselineRelative(hr: hrBaseline, rmssd: nil)) + let scores = levels.map { pair in r.scored.first { $0.hour == pair.hour }!.level! } + + // Strictly increasing with the injected elevation — all four levels recovered, in order. + for i in 1..= DaytimeStress.highBandFloor }.count * (DaytimeStress.bucketSeconds / 60)) + for p in r.scored { XCTAssertGreaterThanOrEqual(p.level!, DaytimeStress.highBandFloor) } + } + + func testBaselineRelativeNilRMSSDFallsBackToHROnlyAndFlagsDegraded() { + // An imported, Oura-era day: no personal RMSSD baseline exists yet (rmssd: nil) and no + // R-R stream is available either. The read must still complete honestly, never crash. + let hrBaseline = Baselines.foldHistory(Array(repeating: 65.0, count: 20), cfg: Baselines.daytimeHRCfg) + var hr: [HRSample] = [] + for h in [9, 14] { hr += hourHR(h, bpm: 80) } // right at the validated +15 bpm margin + let r = DaytimeStress.analyze(hr: hr, rr: [], mode: .baselineRelative(hr: hrBaseline, rmssd: nil)) + + XCTAssertTrue(r.hrOnlyFallback) + XCTAssertFalse(r.scored.isEmpty, "HR-only scoring must still produce a timeline") + for p in r.scored { XCTAssertNotNil(p.level) } + } + + func testBaselineRelativeUsesRMSSDBaselineWhenAvailable() { + // Personal baselines: HR steady at 65 bpm, RMSSD steady at 40 ms (both spread-floored). + let hrBaseline = Baselines.foldHistory(Array(repeating: 65.0, count: 20), cfg: Baselines.daytimeHRCfg) + let rmssdBaseline = Baselines.foldHistory(Array(repeating: 40.0, count: 20), cfg: Baselines.daytimeRMSSDCfg) + + var hr: [HRSample] = [] + var rr: [RRInterval] = [] + for h in [9, 14] { hr += hourHR(h, bpm: 65) } // HR AT baseline in both hours — isolates RMSSD + rr += hourRRVariable(9, rrMs: 900, jitter: 40) // normal variability + rr += hourRRVariable(14, rrMs: 900, jitter: 2) // suppressed HRV -> more stressed + + let r = DaytimeStress.analyze(hr: hr, rr: rr, + mode: .baselineRelative(hr: hrBaseline, rmssd: rmssdBaseline)) + XCTAssertFalse(r.hrOnlyFallback, "an RMSSD baseline was supplied — no fallback") + let normal = r.scored.first { $0.hour == 9 }!.level! + let suppressed = r.scored.first { $0.hour == 14 }!.level! + XCTAssertGreaterThan(suppressed, normal, + "suppressed RMSSD vs. the personal baseline should read MORE stressed than normal variability") + } + + // MARK: - Motion gate (MOVE E): an ambulatory hour is EXERTION — masked, not scored as stress + + /// Fill one local hour with gravity whose consecutive L2 delta ≈ `delta` on every step (x + /// alternates 0↔delta), so the derived activity intensity ≈ `delta` across the hour. delta > + /// `WorkoutDetector.motionThreshold` (0.20) ⇒ AMBULATORY; a small delta (0.02, desk) ⇒ still. + private func hourGravity(_ hour: Int, delta: Double, n: Int = 180) -> [GravitySample] { + let base = hour * 3_600 + let step = max(1, 3_600 / n) + return (0.. 0.20 floor) + + let r = DaytimeStress.analyze(hr: hr, rr: [], gravity: grav) + let h11 = r.hours.first { $0.hour == 11 }! + XCTAssertNil(h11.level, "an ambulatory hour must not be scored") + XCTAssertTrue(h11.maskedForActivity, "the walking hour must be flagged masked-for-activity") + XCTAssertFalse(r.scored.contains { $0.hour == 11 }, "masked hour is excluded from the scored set") + XCTAssertEqual(r.activityMaskedHours, 1) + } + + func testSameElevatedHRIsScoredWhenStill() { + // CONTROL for the test above: identical HR, but the elevated hour is STILL (low gravity). The + // elevation is no longer explained by motion, so it IS scored as stress — proving the mask is + // driven by MOTION, not by the HR value. + var hr: [HRSample] = [] + for h in [8, 9, 10] { hr += hourHR(h, bpm: 60) } + hr += hourHR(11, bpm: 110) + var grav: [GravitySample] = [] + for h in [8, 9, 10, 11] { grav += hourGravity(h, delta: 0.02) } // ALL still + + let r = DaytimeStress.analyze(hr: hr, rr: [], gravity: grav) + let h11 = r.hours.first { $0.hour == 11 }! + XCTAssertNotNil(h11.level, "a still hour with elevated HR must be scored, not masked") + XCTAssertFalse(h11.maskedForActivity) + XCTAssertEqual(r.activityMaskedHours, 0) + XCTAssertEqual(r.peak?.hour, 11, "the still elevated hour is the day's stress peak") + } + + func testNoGravityIsByteIdenticalToOmittingIt() { + // The motion gate is purely additive: no gravity (or an empty array) must produce a + // byte-identical Result to the pre-motion call — nothing is masked without a motion signal. + var hr: [HRSample] = [] + for h in [8, 9, 10] { hr += hourHR(h, bpm: 60) } + hr += hourHR(11, bpm: 110) + let withoutArg = DaytimeStress.analyze(hr: hr, rr: []) + let emptyGrav = DaytimeStress.analyze(hr: hr, rr: [], gravity: []) + XCTAssertEqual(withoutArg, emptyGrav) + XCTAssertEqual(emptyGrav.activityMaskedHours, 0) + XCTAssertTrue(emptyGrav.scored.contains { $0.hour == 11 }, + "with no gravity the elevated hour scores as before (the pre-motion behaviour)") + } + + func testPostExerciseShadowMasksTheNextStillButElevatedHour() { + // A walking hour (11), then a STILL hour (12) whose HR is still elevated (post-exercise + // recovery) → 12 is shadow-masked even though it wasn't itself ambulatory. + var hr: [HRSample] = [] + for h in [8, 9, 10] { hr += hourHR(h, bpm: 58) } + hr += hourHR(11, bpm: 120) // walking + hr += hourHR(12, bpm: 95) // still, HR still elevated + var grav: [GravitySample] = [] + for h in [8, 9, 10] { grav += hourGravity(h, delta: 0.02) } + grav += hourGravity(11, delta: 0.30) // ambulatory + grav += hourGravity(12, delta: 0.02) // still + + let r = DaytimeStress.analyze(hr: hr, rr: [], gravity: grav) + XCTAssertTrue(r.hours.first { $0.hour == 11 }!.maskedForActivity, "the walking hour is masked") + let h12 = r.hours.first { $0.hour == 12 }! + XCTAssertTrue(h12.maskedForActivity, "the still-but-elevated hour after exercise is shadow-masked") + XCTAssertNil(h12.level) + } + + func testShadowReleasesWhenHRHasRecovered() { + // Same walking hour (11), but the following still hour (12) has HR already back at the calm + // reference → the shadow self-limits and 12 is scored normally, not masked. + var hr: [HRSample] = [] + for h in [8, 9, 10] { hr += hourHR(h, bpm: 58) } + hr += hourHR(11, bpm: 120) // walking + hr += hourHR(12, bpm: 58) // still, HR fully recovered + var grav: [GravitySample] = [] + for h in [8, 9, 10] { grav += hourGravity(h, delta: 0.02) } + grav += hourGravity(11, delta: 0.30) // ambulatory + grav += hourGravity(12, delta: 0.02) // still + + let r = DaytimeStress.analyze(hr: hr, rr: [], gravity: grav) + XCTAssertTrue(r.hours.first { $0.hour == 11 }!.maskedForActivity, "the walking hour is masked") + let h12 = r.hours.first { $0.hour == 12 }! + XCTAssertFalse(h12.maskedForActivity, "a recovered following hour is not shadow-masked") + XCTAssertNotNil(h12.level, "the recovered hour is scored normally") + } } diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/HRVAnalyzerTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/HRVAnalyzerTests.swift index e1e843e17a..b60477e847 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/HRVAnalyzerTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/HRVAnalyzerTests.swift @@ -214,6 +214,50 @@ final class HRVAnalyzerTests: XCTestCase { XCTAssertEqual(HRVAnalyzer.duplicateBeatCount(tsSec: [100, 100], rrMs: [1000, 1010]), 0) // diff rr = distinct } + // MARK: - SDNN index (5-min segmented SDNN, the Apple-comparable window) + + func testSdnnIndexExcludesInterSegmentDrift() { + // Three 100 s segments, each internally near-steady (±5 ms) but at very different levels + // (800 / 900 / 1000 ms). Whole-night SDNN sees the big 800→1000 drift and reads large; the SDNN + // index averages each segment's OWN (small) SDNN, so it stays small — the exact property that makes + // it comparable to a watch's short-window reading instead of the drift-inflated whole-night value. + var rr: [RRInterval] = [] + for t in 0..<50 { rr.append(RRInterval(ts: 0 + t, rrMs: t.isMultiple(of: 2) ? 795 : 805)) } + for t in 0..<50 { rr.append(RRInterval(ts: 100 + t, rrMs: t.isMultiple(of: 2) ? 895 : 905)) } + for t in 0..<50 { rr.append(RRInterval(ts: 200 + t, rrMs: t.isMultiple(of: 2) ? 995 : 1005)) } + + let index = HRVAnalyzer.sdnnIndex(rr, segmentSec: 100) + let wholeNight = HRVAnalyzer.analyze(rawRR: rr.map { Double($0.rrMs) }).sdnn + let idx = try! XCTUnwrap(index) + let whole = try! XCTUnwrap(wholeNight) + XCTAssertEqual(idx, 5.05, accuracy: 1.5, "each segment's own SDNN is ~5 ms") + XCTAssertGreaterThan(whole, 50, "whole-night SDNN is inflated by the 800→1000 drift") + XCTAssertLessThan(idx, whole / 5, "the index must strip out the inter-segment drift") + } + + func testSdnnIndexSteadySeriesIsSmallPositive() { + // A single steady segment (±5 ms) → a small, sane, non-nil index. + let rr = (0..<60).map { RRInterval(ts: $0, rrMs: $0.isMultiple(of: 2) ? 795 : 805) } + let idx = try! XCTUnwrap(HRVAnalyzer.sdnnIndex(rr, segmentSec: 100)) + XCTAssertEqual(idx, 5.05, accuracy: 1.5) + } + + func testSdnnIndexSingleSegmentEqualsWholeSdnn() { + // When all beats fall in ONE segment, the index is just that segment's SDNN = the whole SDNN. + let rr = (0..<60).map { RRInterval(ts: $0, rrMs: [800, 820, 780, 810, 790][$0 % 5]) } + let index = try! XCTUnwrap(HRVAnalyzer.sdnnIndex(rr, segmentSec: 1000)) + let whole = try! XCTUnwrap(HRVAnalyzer.analyze(rr, windowStart: 0, windowEnd: 999).sdnn) + XCTAssertEqual(index, whole, accuracy: 1e-9) + } + + func testSdnnIndexSparseSeriesIsNil() { + // Fewer than minBeats in the only segment → no qualifying segment → nil (honest absence). + let rr = (0..<10).map { RRInterval(ts: $0, rrMs: 800) } + XCTAssertNil(HRVAnalyzer.sdnnIndex(rr, segmentSec: 100)) + XCTAssertNil(HRVAnalyzer.sdnnIndex([], segmentSec: 100)) + XCTAssertNil(HRVAnalyzer.sdnnIndex(rr, segmentSec: 0), "non-positive segment length is rejected") + } + // #550 — collapsedCoverage: previews a SAME-SECOND R-R de-dup so the always-on diag reveals whether // the #257 over-count is same-second (collapsible) or cross-second (needs an ingest-path fix). diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD1LogDomainTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD1LogDomainTests.swift new file mode 100644 index 0000000000..e98339facd --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD1LogDomainTests.swift @@ -0,0 +1,35 @@ +import XCTest +@testable import StrandAnalytics +import WhoopStore + +/// RD1 — the readiness HRV signal is z-scored in the LOG domain (lnRMSSD), not raw ms. RMSSD is +/// right-skewed, so a symmetric z on raw ms over-weights the long upper tail; lnRMSSD is closer to +/// normal (Plews/Altini; the app's own HRVReadiness works this way). Observable consequence: the +/// baseline the signal reports is the GEOMETRIC mean (a typical night), not an arithmetic mean that +/// a few big-recovery nights inflate. RHR stays linear (it is ~normal). +final class RD1LogDomainTests: XCTestCase { + private func d(_ i: Int, hrv: Double?, rhr: Int?) -> DailyMetric { + DailyMetric(day: String(format: "2024-03-%02d", i), totalSleepMin: nil, efficiency: nil, + deepMin: nil, remMin: nil, lightMin: nil, disturbances: nil, restingHr: rhr, + avgHrv: hrv, recovery: nil, strain: 10, exerciseCount: nil, + spo2Pct: nil, skinTempDevC: nil, respRateBpm: nil) + } + + func testHrvBaselineIsGeometricNotArithmetic() { + // 20 nights at 42 ms + 8 big-recovery nights at 85 ms: arithmetic mean ≈ 54, geometric ≈ 51. + var days: [DailyMetric] = [] + for i in 1...20 { days.append(d(i, hrv: 42, rhr: i % 2 == 0 ? 53 : 51)) } + for i in 21...28 { days.append(d(i, hrv: 85, rhr: i % 2 == 0 ? 53 : 51)) } + days.append(d(29, hrv: 50, rhr: 52)) // today: a typical night + + let hrv = ReadinessEngine.evaluate(days: days).signals.first { $0.key == "hrv" } + // Log domain baselines against a GEOMETRIC center (49 ms), well below the arithmetic mean (54) + // a raw-ms z would report. Under RD2 the center is the recency-weighted, Winsor-clamped EWMA + // (Baselines spine, reject off): the 8 recent 85 ms nights nudge it up from the 42 ms cluster + // but Winsorization caps their pull, so it lands at 49 — still geometric, not the tail-inflated + // arithmetic 54. + XCTAssertEqual(hrv?.evidence, "50 vs 49 ms") + // 50 sits at the typical night → neutral, read against a representative (not tail-inflated) baseline. + XCTAssertEqual(hrv?.flag, .neutral) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD2SpineTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD2SpineTests.swift new file mode 100644 index 0000000000..2da22551db --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RD2SpineTests.swift @@ -0,0 +1,73 @@ +import XCTest +@testable import StrandAnalytics +import WhoopStore + +/// RD2 — the readiness HRV/RHR baselines are folded through the shared Winsorized-EWMA spine +/// (`Baselines.foldHistory`) instead of a naive mean + sample SD, in the WINDOW-FOLD mode +/// (`rejectHardOutliers: false`). Chosen over the full incremental spine after replaying real HRV +/// history: hard-outlier rejection is right for RecoveryScorer's incremental nightly fold but WRONG +/// for a re-folded trailing window, where a recent sustained shift lands past the young grace period +/// and would be rejected as a run of "outliers". Winsorization (kept) still damps single freak nights. +final class RD2SpineTests: XCTestCase { + + // MARK: - The spine capability itself: reject on (incremental) vs off (window-fold) + + func testWindowFoldAdaptsToSustainedShiftThatIncrementalRejects() { + // 20 settled nights at 140 ms, then 10 recent nights at 85 ms (a real sustained shift). The + // incremental fold (reject on) discards the recent block as >5σ outliers and stays anchored + // high; the window fold (reject off) follows the shift down. This is the exact readiness + // failure mode (device swap / supplement onset) the mode switch fixes. + let vals: [Double?] = Array(repeating: 140.0, count: 20) + Array(repeating: 85.0, count: 10) + let rejectOn = Baselines.foldHistory(vals, cfg: Baselines.hrvCfg, rejectHardOutliers: true) + let rejectOff = Baselines.foldHistory(vals, cfg: Baselines.hrvCfg, rejectHardOutliers: false) + XCTAssertGreaterThan(rejectOn.baseline, 130, "hard-reject keeps the stale high baseline") + XCTAssertLessThan(rejectOff.baseline, rejectOn.baseline - 10, + "window-fold adapts meaningfully toward the recent sustained level") + } + + func testSingleFreakNightIsDampedNotRejectedInWindowFold() { + // One freak sensor night in an otherwise stable baseline. With reject OFF the freak is not + // discarded — but Winsorization still CLAMPS it, so it barely moves the center. (Contrast the + // sustained block above, which the same mode follows: single spike damped, sustained shift + // adapted — the whole point of Winsor-without-reject.) + let stable: [Double?] = Array(repeating: 90.0, count: 29) + let withFreak: [Double?] = stable + [180.0] + let base0 = Baselines.foldHistory(stable, cfg: Baselines.hrvCfg, rejectHardOutliers: false) + let base1 = Baselines.foldHistory(withFreak, cfg: Baselines.hrvCfg, rejectHardOutliers: false) + XCTAssertLessThan(abs(base1.baseline - base0.baseline), 8, + "a single 2× freak night must be Winsor-damped, not swing the baseline") + } + + // MARK: - Readiness end-to-end (ln-space HRV, reject off) + + private func d(_ i: Int, hrv: Double?, rhr: Int?) -> DailyMetric { + DailyMetric(day: String(format: "2024-03-%02d", i), totalSleepMin: nil, efficiency: nil, + deepMin: nil, remMin: nil, lightMin: nil, disturbances: nil, restingHr: rhr, + avgHrv: hrv, recovery: nil, strain: 10, exerciseCount: nil, + spo2Pct: nil, skinTempDevC: nil, respRateBpm: nil) + } + + func testReadinessAdaptsToRecentSustainedHRVShift() { + // 12 old nights at 130 ms, then 18 recent nights settled at 85 ms (a sustained shift), today at + // the new normal 85. It must NOT read BAD — the window fold adapts. Under the incremental + // (reject-on) mode this exact case reads BAD (the recent block is rejected, baseline stuck at + // 130), which is the readiness regression RD2 exists to prevent. + var days: [DailyMetric] = [] + for i in 1...12 { days.append(d(i, hrv: 130, rhr: 52)) } + for i in 13...30 { days.append(d(i, hrv: 85, rhr: 52)) } + days.append(d(31, hrv: 85, rhr: 52)) // today: the new normal + let hrv = ReadinessEngine.evaluate(days: days).signals.first { $0.key == "hrv" } + XCTAssertNotNil(hrv) + XCTAssertNotEqual(hrv?.flag, .bad, + "today at the recent sustained normal must not read BAD (the reject-mode failure)") + } + + func testGenuineLowStillFlagsAfterSpineAdoption() { + // Regression: the spine must not blunt a real drop. Stable ~95 ms baseline, today a clear 68 ms. + var days: [DailyMetric] = [] + for i in 1...30 { days.append(d(i, hrv: i % 2 == 0 ? 97 : 93, rhr: 52)) } + days.append(d(31, hrv: 68, rhr: 52)) + let hrv = ReadinessEngine.evaluate(days: days).signals.first { $0.key == "hrv" } + XCTAssertEqual(hrv?.flag, .bad, "a genuine ~28% HRV drop must still read BAD") + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RDConfidenceTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RDConfidenceTests.swift new file mode 100644 index 0000000000..1fd5f7d27f --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RDConfidenceTests.swift @@ -0,0 +1,43 @@ +import XCTest +@testable import StrandAnalytics +import WhoopStore + +/// RD-confidence — the readiness read now carries a `ScoreConfidence` from its baseline density, so +/// the card can say calibrating / building / solid instead of a confident number off a 7-night +/// baseline (Charge already does this; readiness didn't). +final class RDConfidenceTests: XCTestCase { + + func testReadinessConfidenceTiers() { + XCTAssertEqual(ScoreConfidence.readiness(hasRead: false, baselineNights: 40, fullWindow: 30), .calibrating) + XCTAssertEqual(ScoreConfidence.readiness(hasRead: true, baselineNights: 12, fullWindow: 30), .building) + XCTAssertEqual(ScoreConfidence.readiness(hasRead: true, baselineNights: 30, fullWindow: 30), .solid) + } + + private func d(_ i: Int, hrv: Double?, rhr: Int?) -> DailyMetric { + DailyMetric(day: String(format: "2024-03-%02d", i), totalSleepMin: nil, efficiency: nil, + deepMin: nil, remMin: nil, lightMin: nil, disturbances: nil, restingHr: rhr, + avgHrv: hrv, recovery: nil, strain: 10, exerciseCount: nil, + spo2Pct: nil, skinTempDevC: nil, respRateBpm: nil) + } + + func testEmptyHistoryReadsCalibrating() { + XCTAssertEqual(ReadinessEngine.evaluate(days: []).confidence, .calibrating) + } + + func testThinBaselineReadsBuilding() { + // 10 baseline nights + today: a read exists but baseline < the 30-night window → building. + var days: [DailyMetric] = [] + for i in 1...10 { days.append(d(i, hrv: i % 2 == 0 ? 62 : 58, rhr: 52)) } + days.append(d(11, hrv: 60, rhr: 52)) + let r = ReadinessEngine.evaluate(days: days) + XCTAssertNotEqual(r.level, .insufficient) + XCTAssertEqual(r.confidence, .building) + } + + func testFullBaselineReadsSolid() { + var days: [DailyMetric] = [] + for i in 1...30 { days.append(d(i, hrv: i % 2 == 0 ? 62 : 58, rhr: 52)) } + days.append(d(31, hrv: 60, rhr: 52)) + XCTAssertEqual(ReadinessEngine.evaluate(days: days).confidence, .solid) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RecoverySaturationGuardTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RecoverySaturationGuardTests.swift new file mode 100644 index 0000000000..f4e0bd3ce3 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RecoverySaturationGuardTests.swift @@ -0,0 +1,183 @@ +import XCTest +import Foundation +@testable import StrandAnalytics + +/// Parasympathetic-saturation guard on Charge (recovery). A low nightly HRV normally drives Charge +/// DOWN via the dominant HRV term, but in a very fit / very relaxed state HRV can read LOW while +/// resting HR reads LOW too and the two decouple — a benign saturated vagal state, not fatigue. +/// The guard eases (never removes) the low-HRV penalty ONLY in that pattern. These tests pin BOTH +/// directions: a saturation night is NOT tanked, a real-fatigue night (low HRV + HIGH resting HR) +/// still IS. No em-dashes. +final class RecoverySaturationGuardTests: XCTestCase { + + /// A usable (trusted) baseline with a given mean and Gaussian sigma (spread is internal + /// abs-dev units; zScore multiplies by 1.253 to recover sigma). + private func baseline(mean: Double, sigma: Double, nValid: Int = 14) -> BaselineState { + BaselineState(baseline: mean, spread: sigma / 1.253, nValid: nValid, + nightsSinceUpdate: 0, status: nValid >= 14 ? .trusted : .provisional) + } + + // MARK: - The guard helper in isolation + + func testGuardFiresOnSaturationSignatureAndEasesButNeverRemovesPenalty() { + // HRV clearly below baseline (penalty direction) AND resting HR clearly below baseline + // (recovery-good direction): the saturation signature. The penalty is eased (effective z + // closer to 0) but stays negative — never inverted, never fully removed. + let s = RecoveryScorer.parasympatheticSaturation(hrvZ: -1.5, rhrZ: 1.5) + XCTAssertTrue(s.active) + XCTAssertGreaterThan(s.effectiveHrvZ, -1.5) // eased toward 0 + XCTAssertLessThan(s.effectiveHrvZ, 0.0) // still a penalty (never inverted) + XCTAssertGreaterThan(s.dampFraction, 0.0) + XCTAssertLessThanOrEqual(s.dampFraction, RecoveryScorer.satMaxDampFraction) // capped + } + + func testGuardIsSilentOnRealFatigue() { + // Low HRV + HIGH resting HR (rhrZ negative): normal sympathetic coupling, genuine fatigue. + // The guard must NOT fire — the low-HRV penalty passes through untouched. + let s = RecoveryScorer.parasympatheticSaturation(hrvZ: -1.5, rhrZ: -1.5) + XCTAssertFalse(s.active) + XCTAssertEqual(s.effectiveHrvZ, -1.5, accuracy: 1e-12) + XCTAssertEqual(s.dampFraction, 0.0, accuracy: 1e-12) + } + + func testGuardIsSilentWhenHRVIsNotLow() { + // HRV at or above baseline is not a penalty to ease, even with a very low resting HR. + XCTAssertFalse(RecoveryScorer.parasympatheticSaturation(hrvZ: 1.5, rhrZ: 1.5).active) + XCTAssertFalse(RecoveryScorer.parasympatheticSaturation(hrvZ: 0.0, rhrZ: 1.5).active) + } + + func testGuardIsSilentWithoutARestingHRSignal() { + // No RHR term (nil): nothing corroborates the low HRV, so real fatigue and benign + // saturation are indistinguishable from HRV alone. Refuse to guess -> no easing. + let s = RecoveryScorer.parasympatheticSaturation(hrvZ: -1.5, rhrZ: nil) + XCTAssertFalse(s.active) + XCTAssertEqual(s.effectiveHrvZ, -1.5, accuracy: 1e-12) + } + + func testGuardIsSilentOnMarginalDivergenceBelowTheGate() { + // Both arms present but shallow (below satEnterZ): marginal noise, not a clear pattern. + let below = RecoveryScorer.satEnterZ - 0.05 + let s = RecoveryScorer.parasympatheticSaturation(hrvZ: -below, rhrZ: below) + XCTAssertFalse(s.active) + XCTAssertEqual(s.effectiveHrvZ, -below, accuracy: 1e-12) + } + + func testDampingIsMonotonicInCorroborationAndCapped() { + // Stronger corroboration (both arms deeper below baseline) eases more, up to the cap. + let weak = RecoveryScorer.parasympatheticSaturation(hrvZ: -0.8, rhrZ: 0.8).dampFraction + let strong = RecoveryScorer.parasympatheticSaturation(hrvZ: -1.4, rhrZ: 1.4).dampFraction + let saturated = RecoveryScorer.parasympatheticSaturation(hrvZ: -3.0, rhrZ: 3.0).dampFraction + XCTAssertGreaterThan(strong, weak) + XCTAssertEqual(saturated, RecoveryScorer.satMaxDampFraction, accuracy: 1e-12) // never exceeds cap + // The weaker arm caps the strength (conservative): a very low HRV with only a mildly low + // resting HR is limited by the resting-HR arm. + let limited = RecoveryScorer.parasympatheticSaturation(hrvZ: -3.0, rhrZ: 0.8) + let bothStrong = RecoveryScorer.parasympatheticSaturation(hrvZ: -3.0, rhrZ: 3.0) + XCTAssertLessThan(limited.dampFraction, bothStrong.dampFraction) + } + + // MARK: - End to end through recovery(): both directions + + func testSaturationNightIsNotTankedButRealFatigueStillIs() { + let hrvB = baseline(mean: 50, sigma: 6.265) // sigma == 6.265 (spread * 1.253) + let rhrB = baseline(mean: 55, sigma: 5.0) + + // Same LOW HRV (41 ms, ~1.44 sigma below) for both nights; only resting HR differs. + let saturation = RecoveryScorer.recovery( + hrv: 41, rhr: 48, resp: nil, // resting HR ~1.4 sigma BELOW baseline (good) + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil)! + let fatigue = RecoveryScorer.recovery( + hrv: 41, rhr: 62, resp: nil, // resting HR ~1.4 sigma ABOVE baseline (bad) + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil)! + + // Real fatigue: still tanked into the red band. + XCTAssertLessThan(fatigue, RecoveryScorer.bandRedMax, "low HRV + high resting HR must stay red") + // Saturation: lifted OUT of red, and clearly above the fatigue night. + XCTAssertGreaterThanOrEqual(saturation, RecoveryScorer.bandRedMax, "saturation must not read red") + XCTAssertGreaterThan(saturation - fatigue, 20.0, "the guard must meaningfully separate the two") + } + + func testGuardLiftsSaturationAboveTheUndampedComposite() { + // Isolate the guard's effect (not just the low-RHR term's own contribution): reconstruct + // the score the SAME night would get with the RAW, undamped HRV z, and show recovery() is + // strictly higher because the guard eased the penalty. + let hrvB = baseline(mean: 50, sigma: 6.265) + let rhrB = baseline(mean: 55, sigma: 5.0) + + let hrvZ = RecoveryScorer.zScore(41, mean: hrvB.baseline, spread: hrvB.spread) + let rhrZ = RecoveryScorer.zScore(rhrB.baseline, mean: 48, spread: rhrB.spread) // (mu - x)/sigma + let wsum = RecoveryScorer.wHRV + RecoveryScorer.wRHR + let undampedZ = (RecoveryScorer.wHRV * hrvZ + RecoveryScorer.wRHR * rhrZ) / wsum + let undamped = 100.0 / (1.0 + exp(-RecoveryScorer.logisticK * (undampedZ - RecoveryScorer.logisticZ0))) + + let guarded = RecoveryScorer.recovery( + hrv: 41, rhr: 48, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil)! + + XCTAssertGreaterThan(guarded, undamped + 5.0, "the guard must add real points on a saturation night") + } + + func testHighRHRHighHRVGoodNightIsUnchangedByGuard() { + // A genuinely great night (HRV up, resting HR down) must be untouched: the guard only + // engages when HRV is BELOW baseline. Recompute the plain composite and require equality. + let hrvB = baseline(mean: 50, sigma: 6.265) + let rhrB = baseline(mean: 55, sigma: 5.0) + let hrvZ = RecoveryScorer.zScore(62, mean: hrvB.baseline, spread: hrvB.spread) // HRV above + let rhrZ = RecoveryScorer.zScore(rhrB.baseline, mean: 49, spread: rhrB.spread) + let wsum = RecoveryScorer.wHRV + RecoveryScorer.wRHR + let z = (RecoveryScorer.wHRV * hrvZ + RecoveryScorer.wRHR * rhrZ) / wsum + let expected = 100.0 / (1.0 + exp(-RecoveryScorer.logisticK * (z - RecoveryScorer.logisticZ0))) + let actual = RecoveryScorer.recovery( + hrv: 62, rhr: 49, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil)! + XCTAssertEqual(actual, expected, accuracy: 1e-9, "HRV-above-baseline night must bypass the guard") + } + + // MARK: - Explainability: trace + driver breakdown + + func testTraceEmitsSaturationLineAndDampedHRVTermOnlyWhenActive() { + let hrvB = baseline(mean: 50, sigma: 6.265) + let rhrB = baseline(mean: 55, sigma: 5.0) + + // Saturation night: the trace names the eased penalty and its score still equals recovery(). + let (satScore, satLines) = RecoveryScorer.recoveryTrace( + hrv: 41, rhr: 48, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil) + let plain = RecoveryScorer.recovery( + hrv: 41, rhr: 48, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil) + XCTAssertEqual(satScore, plain) + XCTAssertTrue(satLines.contains { $0.contains("charge saturation active") }) + + // Real-fatigue night: NO saturation line (guard silent). + let (_, fatLines) = RecoveryScorer.recoveryTrace( + hrv: 41, rhr: 62, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil) + XCTAssertFalse(fatLines.contains { $0.contains("charge saturation") }) + for l in satLines + fatLines { XCTAssertFalse(l.contains("\u{2014}"), "em-dash in: \(l)") } + } + + func testChargeDriversHRVPenaltyEasedAndVerdictExplainsSaturation() { + let hrvB = baseline(mean: 50, sigma: 6.265) + let rhrB = baseline(mean: 55, sigma: 5.0) + + // Saturation night: the HRV row is still a penalty (below baseline) but its verdict + // explains the parasympathetic-saturation easing rather than the plain "limiting recovery". + let sat = RecoveryScorer.chargeDrivers( + hrv: 41, rhr: 48, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil) + let satHRV = sat.first { $0.label == "Heart rate variability" }! + XCTAssertTrue(satHRV.verdict.contains("saturation"), "verdict must explain the eased penalty") + XCTAssertFalse(satHRV.verdict.contains("\u{2014}")) + + // Real-fatigue night: plain limiting verdict, penalty NOT eased. + let fat = RecoveryScorer.chargeDrivers( + hrv: 41, rhr: 62, resp: nil, + hrvBaseline: hrvB, rhrBaseline: rhrB, respBaseline: nil, sleepPerf: nil) + let fatHRV = fat.first { $0.label == "Heart rate variability" }! + XCTAssertTrue(fatHRV.verdict.contains("limiting recovery")) + XCTAssertLessThan(fatHRV.deltaPoints, 0) + // The eased saturation penalty is less negative than the raw fatigue penalty for the same HRV. + XCTAssertGreaterThan(satHRV.deltaPoints, fatHRV.deltaPoints) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestNeedTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestNeedTests.swift new file mode 100644 index 0000000000..f5a54c8ad7 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestNeedTests.swift @@ -0,0 +1,53 @@ +import XCTest +@testable import StrandAnalytics + +/// `AnalyticsEngine.Rest.personalizedNeedHours` (Wave 0 · T1) — population-anchored, age-floored +/// sleep need. The load-bearing property is the anti-self-referential guard: a chronic +/// under-sleeper's need must NOT drift down toward their own deficit (which a plain trailing mean +/// would do, inverting the honesty philosophy for exactly the users with the most sleep debt); it +/// stays anchored at the age-appropriate population floor. +final class RestNeedTests: XCTestCase { + private typealias Rest = AnalyticsEngine.Rest + + func testChronicUnderSleeperStaysAtFloor() { + // 14 nights of ~5.5 h. A self-referential mean returns ~5.5; the anchor must hold >= 7.0. + let nights = Array(repeating: 5.5, count: 14) + XCTAssertGreaterThanOrEqual(Rest.personalizedNeedHours(nightlyHours: nights, age: 30), 7.0) + } + + func testNormalSleeperReflectsUnrestrictedNights() { + let nights = [7.2, 7.8, 8.1, 7.5, 8.4, 7.9, 8.0, 7.6, 8.2, 7.7] + let need = Rest.personalizedNeedHours(nightlyHours: nights, age: 35) + XCTAssertGreaterThanOrEqual(need, 7.0) + XCTAssertLessThanOrEqual(need, 9.5) + // Upper-quartile, not the mean: reflects what they sleep on their longer nights. + XCTAssertGreaterThanOrEqual(need, 7.8) + } + + func testColdStartReturnsPopulationDefault() { + // Fewer than minNeedNights → the population default (8.0), floored. + XCTAssertEqual(Rest.personalizedNeedHours(nightlyHours: [7.0, 8.0, 6.5], age: 30), + Rest.defaultNeedHours, accuracy: 0.001) + } + + func testCapsLongSleeper() { + let nights = Array(repeating: 11.0, count: 10) + XCTAssertLessThanOrEqual(Rest.personalizedNeedHours(nightlyHours: nights, age: 40), 9.5) + } + + func testMinorHasHigherFloor() { + let nights = Array(repeating: 6.0, count: 10) // an under-sleeping teen + XCTAssertGreaterThanOrEqual(Rest.personalizedNeedHours(nightlyHours: nights, age: 15), 8.0) + } + + func testNilAgeUsesAdultFloor() { + let nights = Array(repeating: 5.0, count: 10) + XCTAssertGreaterThanOrEqual(Rest.personalizedNeedHours(nightlyHours: nights, age: nil), 7.0) + } + + func testZeroAndNegativeNightsIgnored() { + // no-data days (0) must not drag the estimate down or count toward the minimum. + let nights = [0.0, -1.0, 7.5, 8.0, 7.8, 8.2, 7.6, 8.1, 7.9, 8.3, 0.0] + XCTAssertGreaterThanOrEqual(Rest.personalizedNeedHours(nightlyHours: nights, age: 30), 7.5) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestWiringImpactTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestWiringImpactTests.swift new file mode 100644 index 0000000000..dcb0d2a0e4 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/RestWiringImpactTests.swift @@ -0,0 +1,54 @@ +import XCTest +@testable import StrandAnalytics + +/// Behavioral validation of SL1/T1: score the SAME night for different sleeper HISTORIES and show +/// how Rest moves once real regularity + personalized need replace neutral-0.5 / fixed-8h. Prints an +/// old→new table for eyeballing magnitude/direction, and pins the load-bearing directions. +final class RestWiringImpactTests: XCTestCase { + private typealias Rest = AnalyticsEngine.Rest + + // One fixed scored night so only (need, consistency) vary: 7h TST, 90% eff, 1h deep, 1.5h REM. + private let tst = 7.0 * 3600, inbed = 7.0 * 3600 / 0.9, eff = 0.90 + private let deep = 1.0 * 3600, rem = 1.5 * 3600 + + private func restOld() -> Double { + Rest.composite(tstSeconds: tst, inBedSeconds: inbed, efficiency: eff, + restorativeSeconds: deep + rem, needHours: Rest.defaultNeedHours, + consistency: nil, deepSeconds: deep) // nil → neutral 0.5 + } + private func restNew(history: [Double], age: Int?) -> (rest: Double, need: Double, cons: Double) { + let need = Rest.personalizedNeedHours(nightlyHours: history, age: age) + let cons = VitalityEngine.sleepConsistency(nightlyHours: Array(history.suffix(28))) + let rest = Rest.composite(tstSeconds: tst, inBedSeconds: inbed, efficiency: eff, + restorativeSeconds: deep + rem, needHours: need, + consistency: cons, deepSeconds: deep) + return (rest, need, cons ?? -1) + } + + func testArchetypeImpactTable() { + let regular = Array(repeating: 8.0, count: 14) + let irregular = (0..<14).map { $0 % 2 == 0 ? 5.0 : 10.0 } // mean 7.5, high variance + let chronicShort = Array(repeating: 5.5, count: 14) + let longSleeper = Array(repeating: 9.2, count: 14) + + let old = restOld() + let reg = restNew(history: regular, age: 30) + let irr = restNew(history: irregular, age: 30) + let shortS = restNew(history: chronicShort, age: 30) + let longS = restNew(history: longSleeper, age: 30) + + print(String(format: "OLD (neutral 0.5, need 8h): Rest %5.1f", old)) + print(String(format: "regular good sleeper: need %.1f cons %.2f Rest %5.1f", reg.need, reg.cons, reg.rest)) + print(String(format: "irregular sleeper: need %.1f cons %.2f Rest %5.1f", irr.need, irr.cons, irr.rest)) + print(String(format: "chronic short (5.5h): need %.1f cons %.2f Rest %5.1f", shortS.need, shortS.cons, shortS.rest)) + print(String(format: "long sleeper (9.2h): need %.1f cons %.2f Rest %5.1f", longS.need, longS.cons, longS.rest)) + + // Load-bearing directions (SL1): a REGULAR history lifts Rest above an IRREGULAR one for the + // identical night — regularity now actually matters. + XCTAssertGreaterThan(reg.rest, irr.rest) + // T1: chronic-short need is floored (never collapses toward the 5.5h deficit). + XCTAssertGreaterThanOrEqual(shortS.need, 7.0) + // T1: a genuine long sleeper's need exceeds the old fixed 8h (more demanding duration term). + XCTAssertGreaterThan(longS.need, 8.0) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepDebtTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepDebtTests.swift index 1a2eebd690..7e7e37a38a 100644 --- a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepDebtTests.swift +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepDebtTests.swift @@ -3,7 +3,10 @@ import XCTest final class SleepDebtTests: XCTestCase { - /// Three nights at need (8 h = 480 min) → zero balance, three counted nights. + // MARK: - Baseline shape (unchanged by the recency/cap/asymmetry model) + + /// Three nights exactly at need (8 h = 480 min) → zero balance, three counted nights. + /// Every delta is 0, so decay/asymmetry/cap never engage: the model still reads 0. func testOnTargetNetsToZero() { let series: [(day: String, totalSleepMin: Double?)] = [ ("2026-06-01", 480), ("2026-06-02", 480), ("2026-06-03", 480), @@ -15,22 +18,9 @@ final class SleepDebtTests: XCTestCase { XCTAssertEqual(l.needMin, 480.0, accuracy: 1e-9) } - /// A surplus night offsets a deficit one (ledger nets credits and debits). - func testSurplusOffsetsDeficit() { - let series: [(day: String, totalSleepMin: Double?)] = [ - ("2026-06-01", 360), // −120 - ("2026-06-02", 540), // +60 - ("2026-06-03", 420), // −60 - ] - // need 8 h = 480. Σ = −120 + 60 − 60 = −120. - let l = SleepDebt.ledger(series: series, needHours: 8.0) - XCTAssertEqual(l.balanceMin, -120.0, accuracy: 1e-9) - XCTAssertTrue(l.isDebt) - XCTAssertEqual(l.magnitudeMin, 120.0, accuracy: 1e-9) - XCTAssertEqual(l.nights.map { $0.deltaMin }, [-120, 60, -60]) - } - - /// Nights with no usable sleep are skipped entirely (never zero-filled as debt). + /// Nights with no usable sleep are skipped entirely (never zero-filled as debt). Two + /// counted nights [480 (δ0), 420 (δ−60)]: the at-need night contributes 0, the deficit + /// accrues fully with nothing before it to decay → balance −60. func testSkipsNoDataNights() { let series: [(day: String, totalSleepMin: Double?)] = [ ("2026-06-01", 480), @@ -42,19 +32,8 @@ final class SleepDebtTests: XCTestCase { XCTAssertEqual(l.nightCount, 2) XCTAssertEqual(l.balanceMin, -60.0, accuracy: 1e-9) XCTAssertEqual(l.nights.map { $0.day }, ["2026-06-01", "2026-06-04"]) - } - - /// Only the most-recent `window` COUNTED nights are in scope. - func testWindowCapKeepsMostRecent() { - // 16 nights, each 60 min UNDER need → each delta −60. - let series: [(day: String, totalSleepMin: Double?)] = (1...16).map { - (String(format: "2026-06-%02d", $0), Double(420)) - } - let l = SleepDebt.ledger(series: series, needHours: 8.0, window: 14) - XCTAssertEqual(l.nightCount, 14) // capped - XCTAssertEqual(l.balanceMin, -840.0, accuracy: 1e-9) // 14 × −60 - XCTAssertEqual(l.nights.first?.day, "2026-06-03") // oldest kept - XCTAssertEqual(l.nights.last?.day, "2026-06-16") // newest kept + // The per-night deltas stay RAW (slept − need), so the diverging bars remain honest. + XCTAssertEqual(l.nights.map { $0.deltaMin }, [0, -60]) } /// Empty / all-skipped input → empty ledger, zero balance. @@ -68,15 +47,135 @@ final class SleepDebtTests: XCTestCase { XCTAssertEqual(SleepDebt.ledger(series: allNil).nightCount, 0) } - /// The default need is AnalyticsEngine.Rest.defaultNeedHours (8 h). + /// The default need is AnalyticsEngine.Rest.defaultNeedHours (8 h). One deficit night + /// (420, δ−60) accrues fully → −60. func testDefaultNeedIsEightHours() { let l = SleepDebt.ledger(series: [("2026-06-01", 420)]) XCTAssertEqual(l.needMin, AnalyticsEngine.Rest.defaultNeedHours * 60.0, accuracy: 1e-9) XCTAssertEqual(l.balanceMin, -60.0, accuracy: 1e-9) } + /// Only the most-recent `window` COUNTED nights are in scope, and the per-night deltas + /// stay raw. 16 uniform-deficit nights, window 14: the oldest two drop, 14 remain. + func testWindowCapKeepsMostRecentNights() { + let series: [(day: String, totalSleepMin: Double?)] = (1...16).map { + (String(format: "2026-06-%02d", $0), Double(420)) // each δ −60 + } + let l = SleepDebt.ledger(series: series, needHours: 8.0, window: 14) + XCTAssertEqual(l.nightCount, 14) // window still caps the NIGHT count + XCTAssertEqual(l.nights.first?.day, "2026-06-03") // oldest kept + XCTAssertEqual(l.nights.last?.day, "2026-06-16") // newest kept + // Balance is the decayed accumulation Σ −60·0.9^k, k=0…13 = −600·(1−0.9^14) ≈ −462.7, + // NOT the flat 14×−60 = −840 the old symmetric sum reported (well under the cap). + XCTAssertEqual(l.balanceMin, -462.7, accuracy: 0.05) + XCTAssertGreaterThan(l.balanceMin, -840.0) // strictly gentler than a flat sum + } + + // MARK: - Recency weighting (recent deficits weigh more than old ones) + + /// The SAME single deficit night weighs more when it is RECENT than when it is OLD: a + /// deficit on the newest night sits at full weight, one four at-need nights ago has + /// decayed. Pins the EWMA recency-weighting. + func testRecencyWeightsRecentDeficitMore() { + let atNeed = 480.0, deficit = 360.0 // δ 0 and δ −120 + let recent: [(day: String, totalSleepMin: Double?)] = [ + ("2026-06-01", atNeed), ("2026-06-02", atNeed), ("2026-06-03", atNeed), + ("2026-06-04", atNeed), ("2026-06-05", deficit), // deficit NEWEST + ] + let old: [(day: String, totalSleepMin: Double?)] = [ + ("2026-06-01", deficit), ("2026-06-02", atNeed), ("2026-06-03", atNeed), + ("2026-06-04", atNeed), ("2026-06-05", atNeed), // deficit OLDEST + ] + let lRecent = SleepDebt.ledger(series: recent, needHours: 8.0) + let lOld = SleepDebt.ledger(series: old, needHours: 8.0) + XCTAssertEqual(lRecent.balanceMin, -120.0, accuracy: 1e-6) // full weight + XCTAssertEqual(lOld.balanceMin, -78.7, accuracy: 0.05) // −120·0.9^4, decayed + XCTAssertGreaterThan(lRecent.magnitudeMin, lOld.magnitudeMin) + XCTAssertTrue(lRecent.isDebt && lOld.isDebt) + } + + /// A single deficit fades as at-need nights pass — the debt decays over time even without + /// any surplus to repay it (the "debt is not a permanent scar" asymmetry). + func testDeficitDecaysAsAtNeedNightsPass() { + let fresh = SleepDebt.ledger(series: [("2026-06-01", 360)], needHours: 8.0) + let aged = SleepDebt.ledger(series: [ + ("2026-06-01", 360), ("2026-06-02", 480), ("2026-06-03", 480), + ("2026-06-04", 480), ("2026-06-05", 480), + ], needHours: 8.0) + XCTAssertEqual(fresh.balanceMin, -120.0, accuracy: 1e-6) + XCTAssertLessThan(aged.magnitudeMin, fresh.magnitudeMin) // decayed toward zero + XCTAssertTrue(aged.isDebt) // but not yet cleared + } + + // MARK: - Cap (debt can't grow to an absurd number) + + /// Sustained heavy deficits are bounded at `maxBalanceNights` × need — without the cap the + /// decayed geometric series would still pile to ~10 nights' worth. 14 nights of a −420 + /// deficit clamp to exactly −1440 (3 × 480), not the ≈ −4000 the unclamped series implies. + func testDebtIsCappedAtMaxNights() { + let series: [(day: String, totalSleepMin: Double?)] = (1...14).map { + (String(format: "2026-06-%02d", $0), Double(60)) // slept 1 h, δ −420 + } + let l = SleepDebt.ledger(series: series, needHours: 8.0) + let cap = SleepDebt.maxBalanceNights * l.needMin // 3 × 480 = 1440 + XCTAssertEqual(l.balanceMin, -cap, accuracy: 1e-6) // clamped to the floor + XCTAssertLessThanOrEqual(l.magnitudeMin, cap + 1e-6) // never exceeds the cap + XCTAssertEqual(cap, 1440.0, accuracy: 1e-9) + } + + /// A surplus is capped symmetrically at +cap so a long banked-sleep run can't read as an + /// absurd credit either. + func testSurplusIsCappedSymmetrically() { + let series: [(day: String, totalSleepMin: Double?)] = (1...14).map { + (String(format: "2026-06-%02d", $0), Double(900)) // slept 15 h, δ +420 + } + let l = SleepDebt.ledger(series: series, needHours: 8.0) + let cap = SleepDebt.maxBalanceNights * l.needMin + XCTAssertLessThanOrEqual(l.magnitudeMin, cap + 1e-6) + XCTAssertFalse(l.isDebt) + } + + // MARK: - Asymmetry (a repay night moves the balance less than an equal deficit night) + + /// Accrual and repayment are NOT symmetric: an X-minute deficit night accrues the full X, + /// but an X-minute surplus night repays only `surplusRepayFraction`·X. Pins the core + /// asymmetry the ticket asks for. + func testSurplusRepaysLessThanDeficitAccrues() { + let deficit = SleepDebt.ledger(series: [("2026-06-01", 360)], needHours: 8.0) // δ −120 + let surplus = SleepDebt.ledger(series: [("2026-06-01", 600)], needHours: 8.0) // δ +120 + XCTAssertEqual(deficit.balanceMin, -120.0, accuracy: 1e-6) // full accrual + XCTAssertEqual(surplus.balanceMin, 60.0, accuracy: 1e-6) // half repay (0.5·120) + XCTAssertGreaterThan(deficit.magnitudeMin, surplus.magnitudeMin) // asymmetric + XCTAssertEqual(surplus.magnitudeMin, + SleepDebt.surplusRepayFraction * deficit.magnitudeMin, accuracy: 1e-6) + XCTAssertTrue(deficit.isDebt) + XCTAssertFalse(surplus.isDebt) + // The RAW per-night deltas are still symmetric (±120) — only the accumulation is asymmetric. + XCTAssertEqual(deficit.nights.first?.deltaMin, -120.0) + XCTAssertEqual(surplus.nights.first?.deltaMin, 120.0) + } + + /// A surplus night dropped onto an existing debt repays less than a fresh deficit of equal + /// size would deepen it — the asymmetry holds mid-ledger, not just from zero. + func testSurplusOffsetsDeficitButNotFully() { + // [360, 540, 420] need 8 h → deltas −120, +60, −60. + // n1: 0·0.9 + (−120) = −120 + // n2: −120·0.9 + (0.5·60) = −108 + 30 = −78 + // n3: −78·0.9 + (−60) = −70.2 − 60 = −130.2 + let l = SleepDebt.ledger(series: [ + ("2026-06-01", 360), ("2026-06-02", 540), ("2026-06-03", 420), + ], needHours: 8.0) + XCTAssertEqual(l.balanceMin, -130.2, accuracy: 1e-6) + XCTAssertTrue(l.isDebt) + XCTAssertEqual(l.nights.map { $0.deltaMin }, [-120, 60, -60]) // raw deltas intact + } + + // MARK: - Nap credit (upstream #1042: naps count toward debt repayment) + /// Main sleep remains the canonical nightly figure, while a separately-recorded nap /// contributes its asleep minutes when the caller prepares the debt series. + /// (Balance expectation goes through the fork's recency-weighted ledger: single + /// night, full weight, so δ −40 lands as −40.) func testNapMinutesAddDebtRepaymentCredit() throws { let credited = try XCTUnwrap(SleepDebt.creditedSleepMin(mainSleepMin: 392, napSleepMin: 48)) XCTAssertEqual(credited, 440, accuracy: 1e-9) @@ -90,24 +189,18 @@ final class SleepDebtTests: XCTestCase { XCTAssertEqual(SleepDebt.creditedSleepMin(mainSleepMin: 392, napSleepMin: -10), 392.0) } - /// A NEGATIVE EXACT half-tie balance rounds AWAY from zero (−0.05 → −0.1), the documented - /// `round1` contract the Kotlin mirror must match (audit #6). needMin = 0.1, slept 0.05 → - /// delta −0.05 exactly → balance −0.1. Kotlin's old `roundToInt()` (half toward +∞) gave - /// 0.0 on this exact tie — the real divergence this pins shut. - func testNegativeHalfTieRoundsAwayFromZero() { - let l = SleepDebt.ledger(series: [("2026-06-01", 0.05)], needHours: 0.1 / 60.0) - XCTAssertEqual(l.balanceMin, -0.1, accuracy: 1e-9) // away-from-zero, not 0.0 - } + // MARK: - Rounding parity (Kotlin mirror must match byte-for-byte) - /// The symmetric POSITIVE exact half-tie (+0.05 → +0.1), pinned so the sign-aware Kotlin - /// rounding can't silently regress one direction. needMin = 0, slept 0.05 → delta +0.05. - func testPositiveHalfTieRoundsAwayFromZero() { - let l = SleepDebt.ledger(series: [("2026-06-01", 0.05)], needHours: 0.0) - XCTAssertEqual(l.balanceMin, 0.1, accuracy: 1e-9) + /// A deficit lands its raw delta on the ledger (no repay-halving), so a −0.05 tie rounds + /// AWAY from zero to −0.1 through the full model path (need 8 h keeps the cap slack). + func testNegativeHalfTieRoundsAwayFromZero() { + let l = SleepDebt.ledger(series: [("2026-06-01", 479.95)], needHours: 8.0) // δ ≈ −0.05 + XCTAssertEqual(l.balanceMin, -0.1, accuracy: 1e-9) } - /// round1 is exercised on the directly-constructed value too, so the rounding mode is - /// pinned independent of the (slept − need) arithmetic path, both signs + a non-tie. + /// `round1` itself rounds half AWAY from zero for BOTH signs — the load-bearing contract the + /// Kotlin mirror's sign-aware `round1` must match (Swift `.rounded()` is half-away-from-zero; + /// Kotlin `roundToInt()` is half-toward-+∞, which diverged on negative ties, audit #6). func testRound1HalfTiesAwayFromZero() { XCTAssertEqual(SleepDebt.round1(-0.05), -0.1, accuracy: 1e-9) XCTAssertEqual(SleepDebt.round1(0.05), 0.1, accuracy: 1e-9) diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerWakeBridgeTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerWakeBridgeTests.swift new file mode 100644 index 0000000000..cd4798d6c0 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerWakeBridgeTests.swift @@ -0,0 +1,194 @@ +import XCTest +@testable import StrandAnalytics +import WhoopProtocol + +/// Wake-gap bridging (experimental, default OFF): assemble adjacent sleep runs across ≤ 45-min wake +/// gaps into ONE candidate run, so a brief get-up stays inside one session as staged wake. +/// +/// Includes a REAL-DATA pin: the exact seam layout of a reporting wearer's stored fragmented nights +/// (Aug 2026 mirror), asserting precisely which seams the bridge merges and which it leaves split. +/// An HR-led rescue of restless-but-asleep nights was prototyped alongside this bridge and WITHDRAWN +/// after a real-data replay — see the history note above `SleepStager.wakeBridgeMaxMin`; the +/// restless-night test below pins that the bridge deliberately does NOT fix that failure. +/// The Kotlin twin (`SleepStagerWakeBridgeTest`) runs the same scenarios with the same numbers. +final class SleepStagerWakeBridgeTests: XCTestCase { + + // MARK: - Stream builders (1 Hz, mirroring SleepStagerTests) + + private func stillGravity(start: Int, durationS: Int) -> [GravitySample] { + (0.. [GravitySample] { + (0.. GravitySample in + let phase = Double(i % 2) * 0.5 + return GravitySample(ts: start + i, x: phase, y: 0, z: 1.0) + } + } + + private func hrStream(start: Int, durationS: Int, bpm: Int) -> [HRSample] { + (0.. Int { + 1_749_513_600 + hourUTC * 3_600 + min * 60 + } + + // MARK: - Wake-gap bridging (the SPLIT-night failure) + + /// A 16-min get-up (real movement + HR 80) mid-night. The centered rolling window smears it to a + /// >mergeMin active run, so OFF stores TWO sessions. ON bridges them into ONE session. + func testBriefGetUpBridgedIntoOneSession() { + let grav = stillGravity(start: at(23), durationS: 4 * 3_600) // 23:00–03:00 + + restlessGravity(start: at(27), durationS: 16 * 60) // 03:00–03:16 get-up + + stillGravity(start: at(27, min: 16), durationS: 3 * 3_600 + 44 * 60) // 03:16–07:00 + let hr = hrStream(start: at(21), durationS: 2 * 3_600, bpm: 75) // 21:00–23:00 awake + + hrStream(start: at(23), durationS: 4 * 3_600, bpm: 52) // 23:00–03:00 asleep + + hrStream(start: at(27), durationS: 16 * 60, bpm: 80) // get-up spike + + hrStream(start: at(27, min: 16), durationS: 3 * 3_600 + 44 * 60, bpm: 52) // back asleep + + hrStream(start: at(31), durationS: 2 * 3_600, bpm: 75) // 07:00–09:00 awake + + let off = SleepStager.detectSleep(hr: hr, gravity: grav) + XCTAssertEqual(off.count, 2, "OFF: the smeared get-up splits the night") + + let on = SleepStager.detectSleep(hr: hr, gravity: grav, wakeBridge: true) + XCTAssertEqual(on.count, 1, "ON: one session bridging the get-up") + XCTAssertLessThanOrEqual(abs(on[0].start - at(23)), 120) + XCTAssertLessThanOrEqual(abs(on[0].end - at(31)), 120) + } + + /// An 80-min genuinely-awake block (elevated HR, real movement) exceeds the 45-min bridge: + /// the night stays two sessions even with the bridge ON. + func testLongWakeGapStaysSplit() { + let grav = stillGravity(start: at(22), durationS: 3 * 3_600) // 22:00–01:00 + + restlessGravity(start: at(25), durationS: 80 * 60) // 01:00–02:20 awake + + stillGravity(start: at(26, min: 20), durationS: 3 * 3_600 + 40 * 60) // 02:20–06:00 + let hr = hrStream(start: at(20), durationS: 2 * 3_600, bpm: 75) + + hrStream(start: at(22), durationS: 3 * 3_600, bpm: 52) + + hrStream(start: at(25), durationS: 80 * 60, bpm: 78) + + hrStream(start: at(26, min: 20), durationS: 3 * 3_600 + 40 * 60, bpm: 52) + + hrStream(start: at(30), durationS: 2 * 3_600, bpm: 75) + + let on = SleepStager.detectSleep(hr: hr, gravity: grav, wakeBridge: true) + XCTAssertEqual(on.count, 2, "an over-threshold wake gap must not be bridged") + } + + /// The withdrawn-rescue pin: a restless night (motion "active", HR at clear sleep levels) is NOT + /// rescued by the bridge — ON must change nothing versus OFF. The bridge only assembles runs the + /// unchanged stillness spine already accepted; the restless-night DROPPING failure is documented + /// open (see the history note above `wakeBridgeMaxMin`). + func testRestlessNightIsNotRescuedByBridge() { + let grav = restlessGravity(start: at(22), durationS: 6 * 3_600 + 1_800) // 22:00–04:30 + + stillGravity(start: at(28, min: 30), durationS: 90 * 60) // 04:30–06:00 + let hr = hrStream(start: at(20), durationS: 2 * 3_600, bpm: 75) + + hrStream(start: at(22), durationS: 8 * 3_600, bpm: 52) + + hrStream(start: at(30), durationS: 2 * 3_600, bpm: 75) + + let off = SleepStager.detectSleep(hr: hr, gravity: grav) + let on = SleepStager.detectSleep(hr: hr, gravity: grav, wakeBridge: true) + XCTAssertEqual(on, off, "the bridge must not reclassify restless stretches") + XCTAssertEqual(on.count, 1, "only the still morning fragment scores either way") + XCTAssertGreaterThanOrEqual(on[0].start, at(28, min: 25)) + } + + // MARK: - Guards still hold on assembled runs + + /// Two still DAYTIME stretches bridged across a 30-min gap (flat in-band HR, no cardiac dip) + /// must still be dropped by the daytime false-sleep guard on the ASSEMBLED window. + func testDaytimeGuardStillAppliesToBridgedRuns() { + let grav = stillGravity(start: at(12), durationS: 2 * 3_600) // 12:00–14:00 + + stillGravity(start: at(14, min: 30), durationS: 2 * 3_600) // 14:30–16:30 + let hr = hrStream(start: at(10), durationS: 8 * 3_600, bpm: 60) // flat, no dip + + let on = SleepStager.detectSleep(hr: hr, gravity: grav, wakeBridge: true) + XCTAssertEqual(on.count, 0, "no resting-HR dip: the daytime guard must reject the bridged window") + } + + /// Default-off byte-identity: omitting the flag and passing `false` produce the identical result. + func testDefaultOffIsByteIdentical() { + let grav = stillGravity(start: at(23), durationS: 4 * 3_600) + + restlessGravity(start: at(27), durationS: 16 * 60) + + stillGravity(start: at(27, min: 16), durationS: 3 * 3_600 + 44 * 60) + let hr = hrStream(start: at(23), durationS: 8 * 3_600, bpm: 52) + + hrStream(start: at(31), durationS: 4 * 3_600, bpm: 75) + + let implicitDefault = SleepStager.detectSleep(hr: hr, gravity: grav) + let explicitFalse = SleepStager.detectSleep(hr: hr, gravity: grav, wakeBridge: false) + XCTAssertEqual(implicitDefault, explicitFalse) + } + + // MARK: - Pure helper + + func testBridgeWakeGapsMergesOnlyUnderThreshold() { + let a = SleepStager.Period(stage: "sleep", start: 0, end: 3_600) + let gap = SleepStager.Period(stage: "active", start: 3_600, end: 3_600 + 40 * 60) + let b = SleepStager.Period(stage: "sleep", start: 3_600 + 40 * 60, end: 3 * 3_600) + let merged = SleepStager.bridgeWakeGaps([a, gap, b]) + XCTAssertEqual(merged.count, 1) + XCTAssertEqual(merged[0].start, 0) + XCTAssertEqual(merged[0].end, 3 * 3_600) + + let farGap = SleepStager.Period(stage: "active", start: 3_600, end: 3_600 + 46 * 60) + let c = SleepStager.Period(stage: "sleep", start: 3_600 + 46 * 60, end: 4 * 3_600) + let unmerged = SleepStager.bridgeWakeGaps([a, farGap, c]) + XCTAssertEqual(unmerged.count, 3, "a 46-min gap is over wakeBridgeMaxMin (45) and must not merge") + } + + /// H4 span guard: a merge whose ASSEMBLED span would exceed maxMainSleepSpanS (16 h) is refused, + /// so a pathological chain can never assemble and then be dropped whole by the ladder's cap — + /// the exact failure the withdrawn HR rescue exhibited on real data. + func testBridgeRefusesOverSpanCapMerge() { + let a = SleepStager.Period(stage: "sleep", start: 0, end: 8 * 3_600) + let b = SleepStager.Period(stage: "sleep", start: 8 * 3_600 + 30 * 60, end: 17 * 3_600) + let out = SleepStager.bridgeWakeGaps([a, b]) + XCTAssertEqual(out.count, 2, "a merge assembling a >16 h span must be refused") + } + + // MARK: - Real-data seam pin (Aug 2026 mirror, reporting wearer's stored fragments) + + /// The stored fragment layouts of five REAL split nights, as (start, end) unix pairs from the + /// wearer's server mirror. Asserts exactly which seams the 45-min bridge merges (16/12/28/25+19/ + /// 17-min arousal seams) and which it leaves split (2.2–3.9 h genuine wake gaps, and a 90-min + /// morning re-sleep gap — over the threshold, left to the selector's #861 night-tail bridge). + func testRealNightSeamsBridgeExactly() { + func sleeps(_ spans: [(Int, Int)]) -> [SleepStager.Period] { + spans.map { SleepStager.Period(stage: "sleep", start: $0.0, end: $0.1) } + } + // Aug 11→12 ET: 4 fragments; only the 16-min bathroom seam (06:17→06:33 ET) merges → 3 blocks. + let aug12 = sleeps([(1786491049, 1786496628), (1786510824, 1786514588), + (1786522512, 1786529853), (1786530794, 1786541952)]) + let aug12Out = SleepStager.bridgeWakeGaps(aug12) + XCTAssertEqual(aug12Out.map { [$0.start, $0.end] }, + [[1786491049, 1786496628], [1786510824, 1786514588], [1786522512, 1786541952]]) + + // Aug 17→18 ET: 3 fragments; the 12-min seam merges, the 3.8 h gap stays → 2 blocks. + let aug18 = sleeps([(1787032890, 1787037056), (1787050829, 1787054649), + (1787055394, 1787063445)]) + XCTAssertEqual(SleepStager.bridgeWakeGaps(aug18).map { [$0.start, $0.end] }, + [[1787032890, 1787037056], [1787050829, 1787063445]]) + + // Aug 18→19 ET: 2 fragments at a 28-min seam → 1 block. + let aug19 = sleeps([(1787126248, 1787132602), (1787134284, 1787141425)]) + XCTAssertEqual(SleepStager.bridgeWakeGaps(aug19).map { [$0.start, $0.end] }, + [[1787126248, 1787141425]]) + + // Aug 20→21 ET: 3 fragments at 25-min and 19-min seams → 1 block (11.4 h < the 16 h cap). + // NOTE: the mirror's correction pass (an HR-evidence-led edit session, not a human label) + // kept the third block (a 10:44–12:03 post-wake morning doze) as a SEPARATE doze; the + // 45-min rule cannot distinguish "brief wake, more sleep" from "woke, dozed again", so the + // bridge deliberately folds it in (wake staged inside) and a wearer who disagrees hand-edits + // that one seam. Pinned as-is, eyes open. + let aug21 = sleeps([(1787287220, 1787297404), (1787298936, 1787322337), + (1787323440, 1787328166)]) + XCTAssertEqual(SleepStager.bridgeWakeGaps(aug21).map { [$0.start, $0.end] }, + [[1787287220, 1787328166]]) + + // Aug 23→24 ET: 2 fragments at a 90-min gap → stays split (over the 45-min bridge). The + // mirror's correction pass filled this gap as SLEEP on HR evidence — it is the + // withdrawn-rescue (restless dropping) class, not a seam; the bridge must not guess it. + let aug24 = sleeps([(1787546768, 1787551261), (1787556701, 1787575851)]) + XCTAssertEqual(SleepStager.bridgeWakeGaps(aug24).count, 2) + } +} diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/Spo2PctNightlyTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/Spo2PctNightlyTests.swift new file mode 100644 index 0000000000..d7bb371988 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/Spo2PctNightlyTests.swift @@ -0,0 +1,174 @@ +import XCTest +import WhoopProtocol +import WhoopStore +@testable import StrandAnalytics + +/// v34 — the nightly SpO2 percentage banked on `DailyMetric.spo2Pct` for a 5/MG night. +/// +/// The strap samples SpO2 in RUNS: ~30 one-second samples roughly once per ~1,200 s while asleep. The +/// first samples of each run are the optical front end settling and they read LOW. That makes a naive +/// mean over the night's in-band samples biased downward — and biased *more* on nights made of many short +/// runs, which is the opposite of when you would think to check. These tests pin the two corrections +/// (per-run ramp trim, then median) and, more usefully, pin the size of the bias they remove. +final class Spo2PctNightlyTests: XCTestCase { + + private func session(_ start: Int, _ durSec: Int) -> SleepSession { + SleepSession(start: start, end: start + durSec, efficiency: 0.9, + stages: [], restingHR: 50, avgHRV: 60) + } + + /// A measurement run: `count` consecutive one-second samples starting at `ts`, the first + /// `rampCount` of them at the depressed `rampValue`. + private func run(_ ts: Int, count: Int, settled: Int, + rampCount: Int = 0, rampValue: Int = 88) -> [Spo2PctSample] { + (0..= oldest - SESSION_RANGE_MARGIN && ts <= newest + SESSION_RANGE_MARGIN } -/// The HISTORICAL_DATA record frames in `rawFrames` that FAIL decode — a genuine CRC failure, or an -/// unmapped firmware layout whose envelope parsed but yielded no usable biometrics. These are the -/// records the strap is about to free once we ack the trim, so without an archive they are lost -/// forever while the UI reports a clean sync (#77 / #91). +/// The HISTORICAL_DATA record frames in `rawFrames` that NOOP cannot turn into rows — a genuine CRC +/// failure, an unmapped firmware layout (5/MG: any `hist_version` outside +/// `mappedWhoop5HistoricalVersions`), or a mapped layout whose envelope parsed but yielded no usable +/// biometrics. These are the records the strap is about to free once we ack the trim, so without an +/// archive they are lost forever while the UI reports a clean sync (#77 / #91). /// /// Console (type-50, `frame[typeIndex] == 0x32`) frames are strap-side debug-log text that decode to /// zero rows BY DESIGN and are never returned. 5/MG v26 (raw PPG block, hist_version 26) is also @@ -91,6 +92,20 @@ public func rejectedHistoricalRecords(_ rawFrames: [[UInt8]], family: DeviceFami // different type byte, so they never pass this gate — they are excluded by construction. guard f.count > typeIndex, Int(f[typeIndex]) == 47 else { return false } if family == .whoop5, f.count > versionIndex, Int(f[versionIndex]) == 26 { return false } // v26 PPG: has its own durable stream (ppgWaveform), not this reject archive + // UNMAPPED LAYOUT (5/MG) — archive UNCONDITIONALLY, whatever it decoded. + // + // The decode-outcome test below is the wrong question for a layout NOOP has no field map for. + // `decodeWhoop5Historical`'s unmapped branch reads no offsets, so anything that DOES appear in + // `parsed` for such a record came from the envelope, not from a mapped biometric — and a record + // that happened to yield a plausible `unix` plus a `gravity_x`/`heart_rate` used to pass the + // screen and be kept NOWHERE, its bytes freed by the very next trim ack. That is exactly the + // shape a novel record (an on-demand ECG capture, a new firmware's rollup) can take. + // + // This cannot flood the archive with records we already understand: v18/v20/v21/v26 are in + // `mappedWhoop5HistoricalVersions` and never reach here. Retention for the records that DO + // (`RawHistoryArchive.evictLines`) evicts entirely-zero-payload frames first, so a firmware that + // banks empty placeholder records at 1 Hz cannot push out the one informative frame either. + if family == .whoop5, isUnmappedWhoop5HistoricalRecord(f) { return true } let p = parseFrame(f, family: family) // Envelope/CRC reject: parse failed outright or the CRC32 trailer mismatched. if !p.ok || p.crcOK == false { return true } diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift index b3dac26860..1c6294ff2f 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift @@ -331,6 +331,26 @@ private func parseFrameWhoop5(_ frame: [UInt8], collectFields: Bool) -> ParsedFr fields: fb.fields, parsed: fb.parsed) } +/// The WHOOP 5/MG type-47 `hist_version` values `decodeWhoop5Historical` has a REAL field map for. +/// Every other version falls through to the "unmapped layout" branch, which decodes nothing and only +/// describes the payload as an opaque region. +/// +/// This is the single source of truth for "does NOOP understand this layout": the dispatch `switch` in +/// `decodeWhoop5Historical` reads nothing else, and `rejectedHistoricalRecords` uses it to decide which +/// records must be archived raw. Keeping one set means the two can never disagree — the failure mode +/// this replaced was a record from an unmapped layout that happened to decode a plausible `unix` and +/// `gravity_x`/`heart_rate` slipping past the archive filter, so its bytes were kept nowhere at all. +/// `Whoop5HistoricalLayoutMapTests` pins the set against the decoder's actual behaviour. +public let mappedWhoop5HistoricalVersions: Set = [18, 20, 21, 26] + +/// True when `frame` is a WHOOP 5/MG type-47 record whose layout version has NO field map — the +/// records that reach the unmapped branch of `decodeWhoop5Historical` and are therefore never +/// re-derivable from anything NOOP stores. Non-type-47 and too-short frames are false. +public func isUnmappedWhoop5HistoricalRecord(_ frame: [UInt8]) -> Bool { + guard frame.count > 9, Int(frame[8]) == 47 else { return false } + return !mappedWhoop5HistoricalVersions.contains(Int(frame[9])) +} + /// Decode a WHOOP 5.0 HISTORICAL_DATA (type 47) DSP biometric record. /// /// The layout version is carried in the byte at frame[9] — the inner record's seq slot, which the @@ -351,16 +371,21 @@ private func decodeWhoop5Historical(_ frame: [UInt8], fb: FieldBuilder, payloadE let version = frame.count > 9 ? Int(frame[9]) : -1 fb.parsed["hist_version"] = .int(version) fb.add(9, 1, "hist_version", "meta", value: .int(version)) - if version == 26 { + // One dispatch, one list: every `case` here must appear in `mappedWhoop5HistoricalVersions` and + // vice versa, because that set is what decides whether a record gets archived raw. A `switch` + // rather than the old if-chain so the mapped versions are readable in one place. + switch version { + case 26: decodeWhoop5HistoricalV26(frame, fb: fb) return - } - if version == 20 || version == 21 { + case 20, 21: decodeWhoop5HistoricalV2021(frame, fb: fb, version: version, payloadEnd: payloadEnd) return - } - guard version == 18 else { - // Unknown historical layout — describe it faithfully without inventing offsets. + case 18: + break // falls through to the v18 field decode below + default: + // Unknown historical layout — describe it faithfully without inventing offsets. The bytes are + // NOT lost: `rejectedHistoricalRecords` archives every record that lands here (#77 / #91). if let payloadEnd = payloadEnd, 11 < payloadEnd, payloadEnd <= frame.count { fb.region(11, payloadEnd, "HISTORICAL_DATA v\(version) (unmapped layout)", "unknown") } diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift index b9a2996281..b9a2958f9a 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift @@ -483,6 +483,45 @@ public struct V18AuxSample: Equatable, Codable, Sendable { public var isEmpty: Bool { slotValues.allSatisfy { $0 == nil } } } +/// The in-band window for the `@82` SpO2 candidate byte: a value in `70...100` is a MEASUREMENT, and +/// everything else on that byte is something else entirely. +/// +/// `@82` is MULTIPLEXED and separable BY VALUE — this is the separation, established over 626,725 +/// production records: +/// - `70...100` a real percentage. Distribution modes at 97; per-night medians 94-97; sleep-gated; +/// emitted as a run of ~30 one-second samples once per ~1,200 s. +/// - bit 7 set status sentinel (`0x80`, `0x88`, `0x90`, `0xA0`, `0xA8` are the observed values). +/// - `1...69` diagnostic codes, not a saturation of anything. +/// - `0` the strap emitted nothing this second (awake, or between runs). +/// - `101...127` never observed — the gap is what makes the bit-7 sentinels and the measurements +/// unambiguously separable without a side channel. +/// +/// This is the SAME window `Interpreter` applies when it emits `spo2_candidate_82` (see the +/// `(70...100).contains(v)` gate there) and the same one `AnalyticsEngine.nightlySpo2CandidateMean` +/// applies. Named here so the three agree structurally rather than by three matching literals. +public let spo2CandidateInBand: ClosedRange = 70...100 + +/// One in-band `@82` SpO2 percentage, durably banked (v34). +/// +/// WHY THIS EXISTS AS ITS OWN STREAM. The byte is carried raw in `V18AuxSample.auxByte82`, but +/// `v18AuxSample` is a rolling capped table (`WhoopStore.v18AuxRetentionRows` = 604,800 rows/device ≈ 7 +/// days at 1 Hz) because the aux blob costs ~30 MB/day. The SpO2 signal inside it is ~2,000 samples per +/// WEEK — roughly 100k rows/year — so pulling it into its own narrow table makes it retainable forever at +/// trivial cost, while the wide aux capture stays capped. Without this, every SpO2 reading older than +/// seven days is destroyed, and the strap has already trimmed its own copy. +/// +/// LOSSLESS BY CONSTRUCTION. `pct` is the raw in-band byte, verbatim. No run grouping, no ramp trimming, +/// no smoothing, no dedupe beyond the (deviceId, ts) primary key. Those are all READ-TIME policies — the +/// cloud reader has already changed its run/ramp rules once, and a device store that had baked the old +/// policy in would have destroyed the evidence needed to change them. The only thing applied on the way +/// in is the `spo2CandidateInBand` demultiplex, which is not a policy but the definition of which values +/// on this byte are percentages at all. +public struct Spo2PctSample: Equatable, Codable, Sendable { + public let ts: Int // wall-clock unix seconds (one per strap-second that carried a reading) + public let pct: Int // the raw in-band `@82` byte, 70...100 + public init(ts: Int, pct: Int) { self.ts = ts; self.pct = pct } +} + public struct Streams: Equatable, Codable { public var hr: [HRSample] public var rr: [RRInterval] diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5Ecg.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5Ecg.swift index 18f7038c37..4c671c11b2 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5Ecg.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5Ecg.swift @@ -23,7 +23,13 @@ import Foundation // What is NOT established, and is therefore never asserted: // • The packet TYPE byte these records arrive under. No capture exists, and the repo's `PacketType` // table has no Labrador entry. So this file decodes a PAYLOAD, and the app layer discovers the type -// empirically by running `plausibleFilteredPayload` over unclassified frames and logging the hits. +// empirically: it runs `filteredBytesPerSampleCandidates` over unclassified frames, and — because a +// heuristic that only logs its own hits destroys the evidence for its own misses — CENSUSES every +// unclassified frame by type byte whether the triage passed or not (`Whoop5EcgProbe.FrameCensus`). +// • The filtered stream's BYTES PER SAMPLE. `decodeFiltered` implements 2, which is what the filtered +// layout is documented with; the triage admits 2, 3 and 4, because a populated RAW record read off +// hardware turned out to be 3 bytes/sample and a hardcoded 2 in the triage silently discarded every +// frame that disagreed. See `filteredWidthCandidates`. // • The `WristSelection` raw values. `right` is listed first in the client's enum, so right=0/left=1 is // the natural reading — but it is an INFERENCE, not an attested fact, and it is labelled as such // everywhere it surfaces (including in the UI, because picking the wrong one writes the wrong @@ -203,12 +209,32 @@ public struct FilteredLabradorPacket: Equatable, Sendable { /// diagnostic arrays. The blob's bytes-per-sample is `rawECGDataRaw.count / numberOfECGSamples` — which /// means the blob's LENGTH is not itself on the wire, so a decode needs that width supplied or resolved /// (see `rawBytesPerSampleCandidates`). +/// +/// BOTH variable-length-looking regions are really FIXED-SIZE containers with a count that says how many +/// leading slots are valid: +/// • the sample region — see `unusedSampleBytes` and `Whoop5Ecg.decodeRaw(payload:bytesPerSample:)`; +/// • the leads-off block — `Whoop5Ecg.leadsOffSlotCount` slots for I and the same again for Q, with +/// `numberOfLeadsOffSamples` selecting how many leading slots are valid. +/// In both, the unused slots are dropped rather than carried, so a packet's arrays only ever hold values +/// the record says are real. public struct RawLabradorPacket: Equatable, Sendable { public let header: EcgStatusHeader /// Opaque. The container width and encoding are unattested, so the bytes are carried verbatim. + /// + /// Exactly `numberOfECGSamples * bytesPerSample` bytes — the VALID part of the sample region, which on + /// a partly-filled record is shorter than the region itself. public let rawECGDataRaw: [UInt8] + /// Bytes of the fixed sample region that `numberOfECGSamples` did not fill. Zero for a full record. + /// + /// The strap zero-fills them, and the decoder requires that (a non-zero byte past the declared samples + /// is how a mis-placed record end is caught). They are counted, not carried, so + /// `headerLength + rawECGDataRaw.count + unusedSampleBytes` is where the leads-off block begins. + public let unusedSampleBytes: Int public let numberOfLeadsOffSamples: UInt8 + /// The VALID leads-off I values — `numberOfLeadsOffSamples` of them, taken from the front of the + /// record's fixed `Whoop5Ecg.leadsOffSlotCount`-slot block. Unused slots are dropped, not carried. public let leadsOffIRaw: [UInt16] + /// The VALID leads-off Q values, on the same terms as `leadsOffIRaw`. public let leadsOffQRaw: [UInt16] public let padding: [UInt8] @@ -219,10 +245,15 @@ public struct RawLabradorPacket: Equatable, Sendable { return rawECGDataRaw.count / n } - public init(header: EcgStatusHeader, rawECGDataRaw: [UInt8], numberOfLeadsOffSamples: UInt8, + /// The full sample region the record reserved, valid part plus unused capacity. + public var sampleRegionBytes: Int { rawECGDataRaw.count + unusedSampleBytes } + + public init(header: EcgStatusHeader, rawECGDataRaw: [UInt8], unusedSampleBytes: Int = 0, + numberOfLeadsOffSamples: UInt8, leadsOffIRaw: [UInt16], leadsOffQRaw: [UInt16], padding: [UInt8]) { self.header = header self.rawECGDataRaw = rawECGDataRaw + self.unusedSampleBytes = unusedSampleBytes self.numberOfLeadsOffSamples = numberOfLeadsOffSamples self.leadsOffIRaw = leadsOffIRaw self.leadsOffQRaw = leadsOffQRaw @@ -244,8 +275,27 @@ public enum Whoop5Ecg { /// Trailing bytes tolerated after the last decoded field. The puffin inner record is padded to a /// 4-byte boundary (`puffinCommandFrame`'s pad4), so a well-formed record leaves at most 3 spare /// bytes. Callers scanning an unfamiliar layout can widen it. + /// + /// On the RAW side this is not just a tolerance but the search bound: `decodeRaw` locates the + /// leads-off block by trying each of these end positions, closest to the end of the payload first. public static let defaultMaxPadding = 3 + /// Slots in the raw record's leads-off diagnostic block, per array. + /// + /// MEASURED FROM HARDWARE, not attested: two complete type-47 layout-16 flash records captured + /// 2026-08-06 (embedded verbatim in `Whoop5EcgRawHardwareTests`) show the block is FIXED-SIZE — the + /// count byte is followed by eleven i16 I slots and then eleven i16 Q slots, present in full whether + /// or not `numberOfLeadsOffSamples` fills them, with the unused tail slots zeroed. Both counts seen + /// in the capture (10 and 11) place the Q array at the same offset and leave the same single trailing + /// byte, which is what identifies the block as fixed rather than packed. + /// + /// Reading it as PACKED — `count` elements each — was the original assumption, and it is wrong in two + /// compounding ways whenever `count < 11`: Q is read two bytes early per missing slot (so it gains a + /// spurious leading value and loses its last real one), and the misplaced end-of-record leaves a + /// remainder over `defaultMaxPadding`, which discarded 227 of the capture's 351 populated records + /// before any field was read. + public static let leadsOffSlotCount = 11 + // MARK: Commands // // All four share the shape `{revision: UInt8, arg, padding}`. `revision` is the leading inner byte the @@ -410,14 +460,45 @@ public enum Whoop5Ecg { // MARK: Raw decode + /// Bytes the raw record's leads-off block occupies: the count byte, then a full I array, then a full + /// Q array. Fixed — see `leadsOffSlotCount`. + public static let leadsOffBlockLength = 1 + leadsOffSlotCount * 4 + /// Decode a `RawLabradorPacket` from the inner record's PAYLOAD with an explicit sample width. /// /// `bytesPerSample` has to be supplied because the raw blob's length is NOT on the wire: the client /// derives the width by dividing the blob it already holds by `numberOfECGSamples`, which a /// byte-stream decoder cannot do until it knows where the blob ends. `rawBytesPerSampleCandidates` /// enumerates the widths a given buffer admits. - public static func decodeRaw(payload: [UInt8], bytesPerSample: Int) -> RawLabradorPacket? { - guard bytesPerSample > 0, let header = EcgStatusHeader(payload: payload) else { return nil } + /// + /// **The sample region is a fixed-size container, not `n * bytesPerSample` bytes.** MEASURED FROM + /// HARDWARE, on the same capture and by the same argument as `leadsOffSlotCount`: the capture's one + /// partly-filled record (`numberOfECGSamples == 245`, embedded in `Whoop5EcgRawHardwareTests`) is the + /// same 1584-byte frame as every full 500-sample record, its sample data stops after `245 * 3` bytes, + /// the rest of the region is zeroed, and its leads-off block sits at the SAME frame offset as every + /// other record's. So `numberOfECGSamples` says how many of the region's slots are valid in exactly + /// the way `numberOfLeadsOffSamples` says how many leads-off slots are — one field over, one layer up. + /// + /// That is why the leads-off block is located from the END of the payload rather than from + /// `headerLength + n * bytesPerSample`: the region's LENGTH is no more on the wire than the width is, + /// and a decoder that assumes the region is exactly full lands the count byte inside the zero fill + /// (which is what discarded this record — it read a zero as the count, an empty leads-off block, and + /// left 766 bytes of "padding"). No region length is hardcoded here; the block's own fixed size + /// anchors it. + /// + /// The candidate ends are tried closest-to-the-end first, so the record claims the least padding it + /// can — the puffin pad4 filler is minimal by construction, and a longer claim would eat real bytes. + /// That ordering is a tie-break, not a proof: an all-zero tail can also validate a block placed a + /// byte or two earlier (reading a zero as `numberOfLeadsOffSamples` and empty arrays), and the + /// tightest placement is the one that explains the most bytes rather than the fewest. + /// Two things then have to hold, and either failing moves the search on: the region must be long + /// enough for the declared samples, and every byte between them and the block must be ZERO. That + /// zero-fill check is what keeps the width enumeration honest — a width that under-reads the samples + /// leaves real sample bytes in the unused span and is rejected there. + public static func decodeRaw(payload: [UInt8], bytesPerSample: Int, + maxPadding: Int = defaultMaxPadding) -> RawLabradorPacket? { + guard bytesPerSample > 0, maxPadding >= 0, + let header = EcgStatusHeader(payload: payload) else { return nil } let n = Int(header.numberOfECGSamples) // `bytesPerSample` is caller-supplied on a public API, and `numberOfECGSamples` comes off the // wire — so BOTH the product and the following add are checked rather than assumed. Either would @@ -426,32 +507,46 @@ public enum Whoop5Ecg { // the multiply alone is not enough.) let (blobLength, mulOverflow) = n.multipliedReportingOverflow(by: bytesPerSample) guard !mulOverflow else { return nil } - let (rawEnd, addOverflow) = headerLength.addingReportingOverflow(blobLength) - guard !addOverflow else { return nil } - // The leads-off count byte must itself be inside the buffer. - guard rawEnd >= headerLength, rawEnd < payload.count else { return nil } - let leadsOffCount = Int(payload[rawEnd]) - let iStart = rawEnd + 1 - let qStart = iStart + leadsOffCount * 2 - let qEnd = qStart + leadsOffCount * 2 - guard qEnd <= payload.count else { return nil } - - var leadsOffI = [UInt16]() - var leadsOffQ = [UInt16]() - leadsOffI.reserveCapacity(leadsOffCount) - leadsOffQ.reserveCapacity(leadsOffCount) - for i in 0..= headerLength, sampleEnd <= payload.count else { return nil } + + for pad in 0...maxPadding { + let blockEnd = payload.count - pad + let countIndex = blockEnd - leadsOffBlockLength + // The block must sit entirely after the declared samples. This also bounds `countIndex` from + // below, since `sampleEnd >= headerLength >= 0`. + guard countIndex >= sampleEnd else { continue } + // Unused sample capacity is zero-filled on the wire. A non-zero byte here means the block is + // not at this offset — including the case where it holds the samples a wider width would read. + guard payload[sampleEnd.. [Int] { - widths.filter { width in - guard let packet = decodeRaw(payload: payload, bytesPerSample: width) else { return false } - return packet.padding.count <= maxPadding - } + widths.filter { decodeRaw(payload: payload, bytesPerSample: $0, maxPadding: maxPadding) != nil } } /// Decode a raw record only when the buffer admits exactly ONE sample width. Ambiguous or @@ -476,36 +574,87 @@ public enum Whoop5Ecg { maxPadding: Int = defaultMaxPadding) -> RawLabradorPacket? { let candidates = rawBytesPerSampleCandidates(payload: payload, widths: widths, maxPadding: maxPadding) guard candidates.count == 1 else { return nil } - return decodeRaw(payload: payload, bytesPerSample: candidates[0]) + return decodeRaw(payload: payload, bytesPerSample: candidates[0], maxPadding: maxPadding) } /// CRC-gated raw decode straight off a complete 5/MG frame, with an explicit sample width. public static func decodeRawFrame(_ frame: [UInt8], bytesPerSample: Int, - payloadStart: Int = puffinPayloadStart) -> RawLabradorPacket? { + payloadStart: Int = puffinPayloadStart, + maxPadding: Int = defaultMaxPadding) -> RawLabradorPacket? { guard let payload = innerPayload(frame, payloadStart: payloadStart) else { return nil } - return decodeRaw(payload: payload, bytesPerSample: bytesPerSample) + return decodeRaw(payload: payload, bytesPerSample: bytesPerSample, maxPadding: maxPadding) } // MARK: Discovery + /// The bytes-per-sample widths the FILTERED triage admits, in the order it reports them. + /// + /// 2 leads because it is the width `decodeFiltered` implements and the width the filtered stream is + /// documented with, so the triage's existing behaviour is a strict subset of the widened one. + /// + /// 3 and 4 are here because the 2-byte assumption was, on this repo's own evidence, unsafe. A + /// populated RAW flash record read off real hardware carried `numberOfECGSamples = 500` against a + /// 1500-byte sample blob — **3 bytes per sample**. Nothing attests that the FILTERED stream uses the + /// same width, and nothing rules it out either; what a hardcoded 2 does is make the difference + /// unobservable, because a 3-byte frame fails the length agreement and is discarded before anyone can + /// look at it. `rawBytesPerSampleCandidates` has enumerated widths on the raw side from the start; + /// this is the filtered side getting the same treatment. + /// + /// Widening the WIDTH is the only loosening: the four Bool-typed bytes, the two classifier enums, the + /// signal-quality range and `n > 0` are all unchanged, so a buffer that failed those still fails. + public static let filteredWidthCandidates = [2, 3, 4] + + /// Every width in `widths` under which `payload` could be a filtered Labrador payload — i.e. the + /// widths whose length agreement holds, once the buffer has passed the field guards. + /// + /// Empty means "not a filtered payload under any admitted width", and is the triage's rejection. More + /// than one width means the buffer genuinely does not determine the answer; like + /// `rawBytesPerSampleCandidates` this REPORTS the ambiguity rather than picking a favourite. + public static func filteredBytesPerSampleCandidates(_ payload: [UInt8], + widths: [Int] = filteredWidthCandidates, + maxPadding: Int = defaultMaxPadding) -> [Int] { + guard let header = EcgStatusHeader(payload: payload) else { return [] } + guard header.signalQualityRaw <= 3, + header.heartKeyArrhythmiaCheckResult != nil, + header.heartKeyArrhythmiaCheckStatus != nil else { return [] } + // The four booleans are Bool-typed on the wire, so anything but 0/1 rules the buffer out. + guard payload[2] <= 1, payload[3] <= 1, payload[4] <= 1, payload[5] <= 1 else { return [] } + let n = Int(header.numberOfECGSamples) + guard n > 0 else { return [] } + return widths.filter { width in + // `widths` is caller-supplied and `n` comes off the wire, so both the multiply and the add are + // checked — the same discipline `decodeRaw` uses, for the same reason: either would trap in + // Swift and wrap NEGATIVE in the Kotlin twin. + guard width > 0 else { return false } + let (blobLength, mulOverflow) = n.multipliedReportingOverflow(by: width) + guard !mulOverflow else { return false } + let (end, addOverflow) = headerLength.addingReportingOverflow(blobLength) + guard !addOverflow else { return false } + return end <= payload.count && payload.count - end <= maxPadding + } + } + + /// The frame-level form of `filteredBytesPerSampleCandidates`, CRC-gated. + public static func filteredBytesPerSampleCandidates(frame: [UInt8], + payloadStart: Int = puffinPayloadStart, + widths: [Int] = filteredWidthCandidates, + maxPadding: Int = defaultMaxPadding) -> [Int] { + guard let payload = innerPayload(frame, payloadStart: payloadStart) else { return [] } + return filteredBytesPerSampleCandidates(payload, widths: widths, maxPadding: maxPadding) + } + /// A cheap structural triage for "could these bytes be a filtered Labrador payload?". /// /// Used by the app layer to hunt for the packet TYPE byte, which is not attested: while an ECG probe /// is armed, every unclassified 5/MG frame is run through this and the hits are logged with their /// type. It is a HEURISTIC — four booleans, three enum ranges and a length agreement — not a /// classifier, and nothing downstream may treat a hit as proof. + /// + /// A pass under ANY width in `filteredWidthCandidates` is a pass; callers that need to know WHICH + /// width agreed call `filteredBytesPerSampleCandidates` instead of re-deriving it. public static func plausibleFilteredPayload(_ payload: [UInt8], maxPadding: Int = defaultMaxPadding) -> Bool { - guard let header = EcgStatusHeader(payload: payload) else { return false } - guard header.signalQualityRaw <= 3, - header.heartKeyArrhythmiaCheckResult != nil, - header.heartKeyArrhythmiaCheckStatus != nil else { return false } - // The four booleans are Bool-typed on the wire, so anything but 0/1 rules the buffer out. - guard payload[2] <= 1, payload[3] <= 1, payload[4] <= 1, payload[5] <= 1 else { return false } - let n = Int(header.numberOfECGSamples) - guard n > 0 else { return false } - let end = headerLength + n * 2 - return end <= payload.count && payload.count - end <= maxPadding + !filteredBytesPerSampleCandidates(payload, maxPadding: maxPadding).isEmpty } /// The frame-level form of `plausibleFilteredPayload`, CRC-gated. This is what the app layer runs diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5EcgProbe.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5EcgProbe.swift index d2c1bdc226..3be83ee269 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5EcgProbe.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Whoop5EcgProbe.swift @@ -110,16 +110,35 @@ public enum Whoop5EcgProbe { /// read silence as evidence turn on this flag and a silently-omitted `true` is exactly the bug /// this field exists to prevent. public let requestsRealtimeData: Bool + /// The ARGUMENT byte this command was sent with, when it is known. + /// + /// `label` carries only the opcode, because it is also the key the inbound handler matches a + /// COMMAND_RESPONSE against and the reply echoes no argument — so the argument cannot live in + /// the label without breaking that match. It is recorded separately instead, because two runs + /// of the same opcode can differ only in this byte: `mainControlECGDataGeneration` takes + /// `stop`/`start`/`restart` (0/1/2), so a start run and a restart run render identically + /// without it and a future reader of a copied report cannot tell which one produced the + /// verdict. + /// + /// Reported as the RAW NUMBER, never a token name, for the same reason the classifier byte is: + /// the number is lossless and cannot drift from the bytes actually put on the wire. + /// + /// `nil` is "not known", which is the honest value for an UNSOLICITED reply — nothing here + /// sent it. It is display-only and feeds no verdict, so unlike `requestsRealtimeData` a + /// default is safe: an omission loses an annotation, never changes a claim. + public let sentArgument: UInt8? /// The reply frame as hex, when one arrived. public let replyHex: String? public init(label: String, outcome: CommandOutcome, requestsRealtimeData: Bool, + sentArgument: UInt8? = nil, replyHex: String? = nil) { self.label = label self.outcome = outcome self.requestsRealtimeData = requestsRealtimeData + self.sentArgument = sentArgument self.replyHex = replyHex } @@ -128,6 +147,179 @@ public enum Whoop5EcgProbe { requestsRealtimeData ? "asks for realtime ECG data" : "cannot produce ECG data (configuration or OFF)" } + + /// The `label`, plus the argument when it is known — what every report line leads with. + public var labelWithArgument: String { + guard let sentArgument else { return label } + return "\(label) arg=\(sentArgument)" + } + } + + // MARK: - Unclassified-frame census + + /// A bounded census of EVERY unclassified 5/MG frame seen while a probe run is armed — the ones the + /// structural triage rejected as well as the ones it accepted. + /// + /// ## Why the rejects are the point + /// + /// The packet TYPE byte the Labrador records arrive under is NOT attested (see `Whoop5Ecg`'s file + /// header), so the probe hunts for it with a shape heuristic. Logging only the heuristic's HITS makes + /// that hunt unfalsifiable: when the heuristic is wrong, the frame it was wrong about is discarded, + /// the report truthfully says "no frame passed the structural triage", and the bytes that would have + /// shown the mistake are gone. That is exactly what a hardcoded 2-bytes-per-sample length agreement + /// did to every 3-byte frame before `Whoop5Ecg.filteredWidthCandidates` widened it — and the next + /// wrong assumption will be invisible the same way unless the raw census exists beside the verdict. + /// + /// So this records the type byte, the frame and payload lengths, `numberOfECGSamples` when the header + /// parses, which sample widths the length agreed with, and the frame's opening bytes as hex. It makes + /// no claim about any of them: it is a census, and every line in it is a frame the probe could not + /// classify. + /// + /// ## Bounds + /// + /// A 5/MG link carries ordinary live traffic at ~1 Hz, and a report is a copyable string, so the + /// census is capped in both directions: at most `maxSamplesPerType` (3) recorded frames per distinct + /// type byte, and at most `maxTypes` (16) distinct type bytes. Everything past a cap is COUNTED, never + /// silently dropped — `count` per type keeps rising after its samples are full, and frames whose type + /// byte arrives after the 16th distinct one land in `framesBeyondTypeCap`. Worst case is 48 recorded + /// frames × 64 hex-rendered bytes, which is bounded regardless of what the strap does. + public struct FrameCensus: Equatable, Sendable { + + /// Recorded frames per distinct type byte. Small on purpose: three examples of a type byte answer + /// "what does this look like", and the running count answers "how often". + public static let maxSamplesPerType = 3 + /// Distinct type bytes tracked. A type byte is one octet, so 256 buckets are possible in theory; + /// 16 is well past what a live link actually shows and keeps the report readable. + public static let maxTypes = 16 + /// Leading bytes of each recorded frame rendered as hex. Enough to carry the whole envelope plus + /// the 17-byte status header and the first samples. + public static let headBytes = 64 + + /// One recorded frame. Nothing here is interpreted — these are measurements of the buffer. + public struct Sample: Equatable, Sendable { + public let frameLength: Int + /// Length of the inner record's payload, or nil when the frame did not yield one (it failed a + /// CRC or was too short for the envelope). + public let payloadLength: Int? + /// `numberOfECGSamples` as the status header would read it, or nil when the payload is too + /// short to hold a header. Reading it does NOT claim this frame is an ECG record. + public let numberOfECGSamples: Int? + /// The bytes-per-sample widths the payload's length agreed with, per + /// `Whoop5Ecg.filteredBytesPerSampleCandidates`. Empty means the triage rejected the frame, + /// which is the case this census exists to preserve. + public let widths: [Int] + /// The frame's first `headBytes` bytes, lowercase hex. + public let headHex: String + + public init(frameLength: Int, payloadLength: Int?, numberOfECGSamples: Int?, + widths: [Int], headHex: String) { + self.frameLength = frameLength + self.payloadLength = payloadLength + self.numberOfECGSamples = numberOfECGSamples + self.widths = widths + self.headHex = headHex + } + + public var line: String { + let payload = payloadLength.map(String.init) ?? "?" + let samples = numberOfECGSamples.map(String.init) ?? "?" + let widthText = widths.isEmpty ? "none" : widths.map(String.init).joined(separator: ",") + return "len=\(frameLength) payload=\(payload) samples=\(samples) widths=\(widthText) head=\(headHex)" + } + } + + /// Everything seen under one type byte. + public struct Bucket: Equatable, Sendable { + public let typeByte: UInt8 + /// Every frame seen with this type byte, including the ones past the per-type sample cap. + public var count: Int + public var samples: [Sample] + + public init(typeByte: UInt8, count: Int, samples: [Sample]) { + self.typeByte = typeByte + self.count = count + self.samples = samples + } + } + + /// Buckets in FIRST-SEEN order, which is the order the report prints them in. Insertion order is + /// kept rather than sorted by count so the census reads as a timeline of what the link did. + public private(set) var buckets: [Bucket] = [] + /// Every frame offered to the census, including those past a cap. + public private(set) var framesSeen = 0 + /// Frames dropped because their type byte was the 17th distinct one. Counted, not hidden. + public private(set) var framesBeyondTypeCap = 0 + + public init() {} + + /// Fold one frame into the census. + /// + /// The caller CRC-gates the frame first (the BLE safety contract's inbound rule); this reads the + /// type byte directly and gets the payload through `Whoop5Ecg.innerPayload`, which re-checks both + /// CRCs itself, so a frame that somehow arrives unverified yields `payloadLength == nil` rather + /// than a decoded field. Frames with no type byte at all (fewer than 9 bytes — shorter than the + /// puffin envelope's minimum) are not records and are ignored. + public mutating func record(frame: [UInt8], + payloadStart: Int = Whoop5Ecg.puffinPayloadStart, + maxPadding: Int = Whoop5Ecg.defaultMaxPadding) { + guard frame.count > 8 else { return } + framesSeen += 1 + let typeByte = frame[8] + guard let index = buckets.firstIndex(where: { $0.typeByte == typeByte }) else { + guard buckets.count < FrameCensus.maxTypes else { + framesBeyondTypeCap += 1 + return + } + buckets.append(Bucket(typeByte: typeByte, + count: 1, + samples: [FrameCensus.sample(frame: frame, + payloadStart: payloadStart, + maxPadding: maxPadding)])) + return + } + buckets[index].count += 1 + guard buckets[index].samples.count < FrameCensus.maxSamplesPerType else { return } + buckets[index].samples.append(FrameCensus.sample(frame: frame, + payloadStart: payloadStart, + maxPadding: maxPadding)) + } + + /// Measure one frame. Public so a test can build a `Sample` without a census. + public static func sample(frame: [UInt8], + payloadStart: Int = Whoop5Ecg.puffinPayloadStart, + maxPadding: Int = Whoop5Ecg.defaultMaxPadding) -> Sample { + let payload = Whoop5Ecg.innerPayload(frame, payloadStart: payloadStart) + let widths = payload.map { + Whoop5Ecg.filteredBytesPerSampleCandidates($0, maxPadding: maxPadding) + } ?? [] + let samples = payload.flatMap { EcgStatusHeader(payload: $0) } + .map { Int($0.numberOfECGSamples) } + let head = frame.prefix(headBytes).map { String(format: "%02x", $0) }.joined() + return Sample(frameLength: frame.count, + payloadLength: payload?.count, + numberOfECGSamples: samples, + widths: widths, + headHex: head) + } + + public var isEmpty: Bool { framesSeen == 0 } + + /// The census as report lines, already indented for `report`. + public var lines: [String] { + var out: [String] = [] + for bucket in buckets { + out.append(String(format: " type=0x%02x frames=%d", Int(bucket.typeByte), bucket.count)) + for sample in bucket.samples { out.append(" \(sample.line)") } + if bucket.count > bucket.samples.count { + out.append(" (+\(bucket.count - bucket.samples.count) more of this type, not recorded)") + } + } + if framesBeyondTypeCap > 0 { + out.append(" (\(framesBeyondTypeCap) frame(s) of further type bytes past the " + + "\(FrameCensus.maxTypes)-type cap, counted only)") + } + return out + } } /// The gate verdict, kept separate from the report text so it is assertable in a test. @@ -220,11 +412,11 @@ public enum Whoop5EcgProbe { let failures = steps.filter { $0.outcome == .failure } if !failures.isEmpty { // A refusal is only evidence about the BLOCK question when what was refused asked for data. - let dataFailures = failures.filter(\.requestsRealtimeData).map(\.label) + let dataFailures = failures.filter(\.requestsRealtimeData).map(\.labelWithArgument) if !dataFailures.isEmpty { return .dataRequestRefused(commands: dataFailures) } - return .commandRefused(commands: failures.map(\.label)) + return .commandRefused(commands: failures.map(\.labelWithArgument)) } - let unsupported = steps.filter { $0.outcome == .unsupported }.map(\.label) + let unsupported = steps.filter { $0.outcome == .unsupported }.map(\.labelWithArgument) if !unsupported.isEmpty { return .opcodeUnsupported(commands: unsupported) } if ecgPacketsSeen > 0 { return .ecgCandidatesArrived(packets: ecgPacketsSeen) } guard !steps.isEmpty else { return .noReplies } @@ -232,9 +424,9 @@ public enum Whoop5EcgProbe { // Past this point the verdict INTERPRETS SILENCE, which only carries information about the ECG // data path when the run asked that path for something and the strap said yes. let requests = steps.filter(\.requestsRealtimeData) - guard !requests.isEmpty else { return .noDataRequested(commands: steps.map(\.label)) } + guard !requests.isEmpty else { return .noDataRequested(commands: steps.map(\.labelWithArgument)) } guard requests.contains(where: { $0.outcome == .success }) else { - return .dataRequestNotAccepted(commands: requests.map(\.label)) + return .dataRequestNotAccepted(commands: requests.map(\.labelWithArgument)) } if steps.allSatisfy({ $0.outcome == .success }) { return .acceptedButSilent(windowSeconds: windowSeconds) @@ -247,10 +439,16 @@ public enum Whoop5EcgProbe { /// `candidateFrames` are the type/length lines for frames that passed the structural triage in /// `Whoop5Ecg.plausibleFilteredPayload` — the empirical answer to "which packet type do these arrive /// under", which no table in this repo yet holds. + /// + /// `census` is the same question asked WITHOUT the heuristic: every unclassified frame the run saw, + /// bucketed by type byte, hits and misses alike. It defaults to empty so a caller that has none still + /// renders, but a live run always passes one — see `FrameCensus` for why the misses matter more than + /// the hits. public static func report(steps: [Step], ecgPacketsSeen: Int, candidateFrames: [String], - windowSeconds: Int) -> String { + windowSeconds: Int, + census: FrameCensus = FrameCensus()) -> String { var sb = "" sb += "WHOOP MG ECG (Labrador) TURN-ON PROBE\n" sb += "Verdict: \(verdict(steps: steps, ecgPacketsSeen: ecgPacketsSeen, windowSeconds: windowSeconds).headline)\n" @@ -261,7 +459,7 @@ public enum Whoop5EcgProbe { // Each step carries WHY it does or does not bear on the block question, so the verdict above // can be checked against its own inputs without reading the source. for step in steps { - sb += " \(step.label): \(step.outcome.token) — \(step.roleNote)\n" + sb += " \(step.labelWithArgument): \(step.outcome.token) — \(step.roleNote)\n" } } sb += "\nECG-shaped packets seen in \(windowSeconds)s: \(ecgPacketsSeen)\n" @@ -287,7 +485,22 @@ public enum Whoop5EcgProbe { sb += "Candidate packet types (structural triage only, NOT a confirmed mapping):\n" for line in candidateFrames { sb += " \(line)\n" } } - let replies = steps.compactMap { step in step.replyHex.map { " \(step.label): \($0)" } } + // The census goes IMMEDIATELY under the triage result, because it is the check on it: "no frame + // passed the structural triage" is only informative next to what the frames actually were. + if census.isEmpty { + sb += "Unclassified-frame census: no unclassified frame arrived at all — the triage had " + + "nothing to reject.\n" + } else { + sb += "Unclassified-frame census — EVERY frame this run could not classify, triage hits and " + + "misses alike, bucketed by the inner record's type byte. The ECG type byte is not " + + "attested, so this, not the triage, is the record of what arrived. widths= are the " + + "bytes-per-sample the payload length agreed with; none = the triage rejected it. " + + "Recorded: \(census.framesSeen) frame(s), first \(FrameCensus.maxSamplesPerType) per " + + "type byte, first \(FrameCensus.maxTypes) type bytes, first \(FrameCensus.headBytes) " + + "bytes each:\n" + for line in census.lines { sb += "\(line)\n" } + } + let replies = steps.compactMap { step in step.replyHex.map { " \(step.labelWithArgument): \($0)" } } if !replies.isEmpty { sb += "\nRaw replies:\n" sb += replies.joined(separator: "\n") + "\n" diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/UnmappedHistoricalLayoutTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/UnmappedHistoricalLayoutTests.swift new file mode 100644 index 0000000000..9078e864d2 --- /dev/null +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/UnmappedHistoricalLayoutTests.swift @@ -0,0 +1,125 @@ +import XCTest +@testable import WhoopProtocol + +/// The archive decision for a WHOOP 5/MG type-47 record must be made on its LAYOUT VERSION, not on +/// what the record happened to decode to. +/// +/// `rejectedHistoricalRecords` used to ask only "did this decode to a `unix` plus a +/// `heart_rate`/`gravity_x`?". That screen is the wrong question for a layout NOOP has no field map +/// for: a record from an unmapped version that answered yes was kept NOWHERE — not as rows (nothing +/// mapped it) and not as bytes (it passed the archive filter) — and the strap freed it on the very next +/// trim ack. An on-demand capture (the MG's ~30 s ECG record, banked to flash and pulled back in a +/// later offload) is exactly the shape of record that can look well-formed enough to pass. +/// +/// The screen survived in practice only because of an accident: the unmapped branch of +/// `decodeWhoop5Historical` reads no offsets, so today nothing but `hist_version` ever lands in +/// `parsed` for such a record (`testUnmappedLayoutsDecodeNothingButTheVersionByte` pins that this is +/// true right now). One added static schema field for type 47, or one partially-mapped new version, +/// would silently reopen the hole. These tests pin the version-based decision instead, so the archive +/// no longer depends on that accident holding. +final class UnmappedHistoricalLayoutTests: XCTestCase { + + private func bytes(_ s: String) -> [UInt8] { + var out = [UInt8](); out.reserveCapacity(s.count / 2); var i = s.startIndex + while i < s.endIndex { let j = s.index(i, offsetBy: 2) + out.append(UInt8(s[i.. [UInt8] { + var f = bytes(whoop5V18Hex) + f[9] = version + let payloadEnd = f.count - 4 + let c = crc32(f, 8, payloadEnd) + f[payloadEnd] = UInt8(c & 0xFF) + f[payloadEnd + 1] = UInt8((c >> 8) & 0xFF) + f[payloadEnd + 2] = UInt8((c >> 16) & 0xFF) + f[payloadEnd + 3] = UInt8((c >> 24) & 0xFF) + return f + } + + // MARK: - defect: an unmapped layout that decodes real biometrics must still be archived + + /// The headline case. The frame's BYTES are a real v18 record — at the v18 offsets they carry a + /// valid unix, a plausible heart rate and a ~1 g gravity vector, the exact combination that made + /// the old screen say "decodable, nothing lost". Only the layout-version byte differs. Whatever + /// such a record decodes to, its bytes must reach the archive. + func testUnmappedLayoutIsArchivedEvenThoughItsBytesDecodeCleanlyAsV18() { + // Precondition: read as its true version, this record decodes everything the old screen wanted. + let asV18 = parseFrame(bytes(whoop5V18Hex), family: .whoop5) + XCTAssertEqual(asV18.crcOK, true) + XCTAssertNotNil(asV18.parsed["unix"], "precondition: these bytes DO decode a unix") + XCTAssertNotNil(asV18.parsed["heart_rate"], "precondition: these bytes DO decode a heart rate") + XCTAssertNotNil(asV18.parsed["gravity_x"], "precondition: these bytes DO decode a gravity vector") + + let unmapped = whoop5FrameWithVersion(22) + XCTAssertEqual(parseFrame(unmapped, family: .whoop5).crcOK, true, + "precondition: the record is CRC-VALID, so only the layout decision can archive it") + XCTAssertTrue(isUnmappedWhoop5HistoricalRecord(unmapped)) + XCTAssertEqual(rejectedHistoricalRecords([unmapped], family: .whoop5), [unmapped], + "a record from a layout NOOP cannot map must be archived whatever it decoded") + } + + /// Every version outside `mappedWhoop5HistoricalVersions` is archived — no gaps, no lucky values. + func testEveryUnmappedVersionIsArchived() { + for v in 0...255 where !mappedWhoop5HistoricalVersions.contains(v) { + let f = whoop5FrameWithVersion(UInt8(v)) + XCTAssertEqual(rejectedHistoricalRecords([f], family: .whoop5), [f], + "hist_version \(v) has no field map, so its bytes must be archived") + } + } + + // MARK: - lockstep: the version set and the decoder's dispatch cannot drift apart + + /// `mappedWhoop5HistoricalVersions` is the single source of truth for the archive decision, so it + /// must not claim more (or fewer) versions than `decodeWhoop5Historical` actually maps. Proved from + /// the decoder's own behaviour: for EVERY version outside the set, the decode yields nothing but + /// `hist_version` — i.e. the dispatch `switch` has no case the set omits. + func testUnmappedLayoutsDecodeNothingButTheVersionByte() { + for v in 0...255 where !mappedWhoop5HistoricalVersions.contains(v) { + let p = parseFrame(whoop5FrameWithVersion(UInt8(v)), family: .whoop5) + XCTAssertEqual(p.parsed.keys.sorted(), ["hist_version"], + "v\(v) is outside the mapped set but the decoder populated fields for it — " + + "the set and the dispatch switch have drifted") + } + } + + /// The four mapped versions ARE recognised as mapped (the other direction of the lockstep). + func testMappedVersionsAreNotTreatedAsUnmapped() { + XCTAssertEqual(mappedWhoop5HistoricalVersions, [18, 20, 21, 26]) + for v in mappedWhoop5HistoricalVersions { + XCTAssertFalse(isUnmappedWhoop5HistoricalRecord(whoop5FrameWithVersion(UInt8(v))), + "v\(v) has a field map and must not be archived on layout grounds") + } + } + + /// A cleanly-decoding v18 record is still NOT archived — widening the screen must not start + /// archiving the layouts NOOP already understands wholesale. + func testMappedV18RecordStillNotArchived() { + XCTAssertTrue(rejectedHistoricalRecords([bytes(whoop5V18Hex)], family: .whoop5).isEmpty) + } + + /// The layout rule is WHOOP 5-only: it keys off `frame[9]`, which on a WHOOP 4 frame is a payload + /// byte, not a version. WHOOP 4's unmapped versions go through the schema's validated v24 fallback + /// (`PostHooks`), which keeps a record only when it decodes to a ~1 g gravity vector and a plausible + /// HR and otherwise drops the biometrics — so those records reach the archive by the decode-outcome + /// route and must not be dragged in by this one. + func testWhoop4FramesAreUnaffectedByTheWhoop5LayoutRule() { + // A synthetic WHOOP 4 V24 type-47 record (HR=63) that decodes cleanly (from HistoricalV24Tests). + let v24Hex = + "aa5a008e2f18000000000000f153650000000000003f0152030000000000000000dc053075" + + "000000cdcc4c3dcdcccc3d5a657e3f00000040cdcc4c3dcdcccc3d5a657e3f504668428403" + + "200364006400b80bb80b000000000000c25c1a88" + XCTAssertTrue(rejectedHistoricalRecords([bytes(v24Hex)], family: .whoop4).isEmpty) + XCTAssertFalse(isUnmappedWhoop5HistoricalRecord(bytes(v24Hex)), + "a WHOOP 4 frame has no type-47 byte at index 8 — the rule must not fire on it") + } +} diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgRawHardwareTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgRawHardwareTests.swift new file mode 100644 index 0000000000..4782cfe5ab --- /dev/null +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgRawHardwareTests.swift @@ -0,0 +1,565 @@ +import XCTest +@testable import WhoopProtocol + +/// Characterisation of REAL WHOOP MG ECG flash records against this package's own decoder. +/// +/// Unlike `Whoop5EcgTests`, whose fixtures are all synthetic, every byte here came off hardware: three +/// complete 1584-byte type-47 records (layout version 16, `frame[9] == 16`) captured 2026-08-06 while +/// the MG clasp electrodes were held, embedded verbatim as hex. Nothing in this file changes production +/// behaviour — it pins what `Whoop5Ecg` ACTUALLY does when handed these bytes, including the places +/// where what it does is wrong. +/// +/// What the records establish, and what they contradict, is written against each assertion. The short +/// version: +/// +/// • The record body is `17 header + 1500 blob + 1 count + 22 leadsOffI + 22 leadsOffQ + 1` = 1563 +/// bytes, with no leftovers — but only when the payload is taken from frame offset 17, four bytes +/// earlier than the record's real status header. +/// • `headerLength = 17` is FOUR bytes too long for this record. The header's last nine bytes +/// (`heartKeyProgress` through `numberOfECGSamples`) therefore align correctly at offset 17 and +/// decode real values; the first eight bytes do not, and read the record's unix timestamp and two +/// unmapped bytes instead of signal quality, the four booleans and the two classifier enums. +/// STILL OPEN — nothing here fixes it. +/// • The leads-off block is FIXED-SIZE — eleven i16 slots for I then eleven for Q, at the same frame +/// offsets under both observed counts, with the unused tail slots zeroed. Reading it as PACKED +/// shifted Q by one slot per unused slot and left a 5-byte remainder against `defaultMaxPadding`, +/// which discarded 227 of the capture's 351 populated records before any field was read. FIXED — +/// `Whoop5Ecg.leadsOffSlotCount`; 350 of the 351 now decode. +/// • The sample blob is 3 bytes per sample, and the payload is 18 signed bits big-endian inside that +/// 24-bit field — measured here by continuity, not assumed, and not attributed to any vendor +/// source. No scale or unit is claimed for the values, and none is applied. +/// • The SAMPLE region is fixed-size too. The capture's one partly-filled record +/// (`numberOfECGSamples == 245`, embedded here) puts its leads-off block at the SAME frame offset +/// 1534 as every other record, with bytes 769...1533 zeroed — so the blob is not `n * bytesPerSample` +/// bytes long, it is `n` valid slots inside a region sized like everyone else's. Reading the region +/// as exactly full landed the count byte on a zero at payload[752], read an empty leads-off block and +/// left 766 bytes of "padding", which is why this was the last populated record still discarded. +/// FIXED — `decodeRaw` now locates the block from the END of the payload; all 351 populated records +/// decode at width 3. +/// • STILL OPEN — a partly-filled record cannot determine its own sample WIDTH. 245 samples inside a +/// 1500-byte region is equally consistent with 3 bytes per sample and with 4, so +/// `rawBytesPerSampleCandidates` reports `[3, 4]` for it and the auto-width overload declines rather +/// than guessing. The 350 full records resolve to `[3]` unaided, and that is where the width comes +/// from — it is a property of the stream, not of one record. +final class Whoop5EcgRawHardwareTests: XCTestCase { + + // MARK: - Real captured records + + /// Record at unix 1786031627 — leads connected, `numberOfLeadsOffSamples == 11`. + /// This is the case whose leads-off block happens to be exactly full, so it is the one the current + /// decoder accepts. + private static let recordLeadsOffCount11 = """ + aa0128060100cde02f10036a61cd000bae746aeb51010a000100000000ffff00f401834ae78349e98345d5834703834c + 53834814834916834a0d83478e834d3e83510a8351bb834fda834a718347d383447c8344f3834ec6834c158347848348 + 87834785834a2d834978834c33834d39834c07834b178352dc83537e8351fb8352968350db835319835048834540834a + b3834de2834f1b834f0f835133834f31834f1a83515d834e178351f18351a38352628352558352fd834d4c834a698350 + de835762834e80834b218349338349e6835095835228834c578351b6834f53834ff58351f4834dcb834a6f8350dc8352 + 658350d9834b2f834ba8834d50834e73834cd2834d3c8356698355e08356508357a48359bb835af9835a5f8353ae8352 + 78835bde835c928358c08353a3835693835bae835b7a835c4a835e89835c8a835bb0835f2583608b83610f835ec9835f + ee83611283606f8363e68361cd836cf583649b835d0783637983654d8360c0835f2883608f835a2c8352e78350f98349 + f18340e5833e90833c878338e0833677832ec9833094833bd28333be833941833e328343d7834d4e83570c8359228362 + 0383622283647f83618c8365ce837351836e3e8367e28368fb836cb3836cd5836de9836d8383717a836f7a836dc4836f + 8783726783723c8370828375d0837291836de9836f4583740883785c8373eb837d748376af8374b283745f8371ad8373 + 9a83763e8373eb8375d58375b48374f1837c6e838486837fc0837d3b8379cd837402837747837f4e83765783755b837a + ab8386ab83876b837c6d8375ef8376cc837d41837b33837f4a837f37837c5583811a837d95837b77837cc1837dc88381 + 11837c56837883837a32837da8837f2d8381ec837c388382cb838541837fef8378bd8374ac83731f837914837d9f837b + 93837f35837e22838094838290837e6b837b12837dac837fdc838048837fda83808f8380018381ee83822e8384ed8387 + 7883836b837f858382428378e3837bf38382e683865183850583810883839283886583840c8388568389b4838af6838e + 148388b783875983893583861883865c838e6283915d838a58838d748391e583904b8393288397618391038391e98390 + e383905f838c82838b178390f683947e83963883963383981d839973839b94839974839894839b0f83952b839453839b + 78839d39839b708399028394bc839038838f648395be839a4a839ad0839f44839c18839b008399a98398ee839bbe839c + 6983967a8392c38396b28396088395218395d1839a16839fe3839e5f83997483948f83974c839afc8396a2839b40839e + 48839aec839903839a8f8399ef839f09839aa58397dc839c9483a4f383a2bd83a3e683a197839a968399c4839821839b + 8c839dd58398488398528394eb839806839b0d839ae2839a388396f1839521838de2839230839a1983a05083a3e58399 + fe83981b83982b8396be83a224839df2839a04839ecc8399c9839671839a6d83953a8396c1839a6583a05483a22e83a1 + 3483a2e983a60083997983984883990d839e6b839ee783a030839dc183a0b983a68583a0c483a40583a0c6839f1d839c + 5683a05e83a0ac839d93839bd1839ebc83a53083aa8b83a71083a49883a0ff83a11f83a22f83a5ac83a69983a54383a3 + 2b839b30839e0d839e2f83a040839fae839f99839b5b83a93f83aeb383a61783a78683a7d383a88383abf683aa6e83a6 + 0383a79983a4cf83a60983a5dd83a7d083a48b83a61683a8c883a88f83a6f683a80a83a8ef83a5b083a73483aa4683a9 + 1b83aca583b07883a4be839ed383a1f483a30783a3ea83a79c83a67d83a50383a70383b0c083a64283abfd83aed883a9 + 1583ac5783aeea83ad7b83ae2683a99283a64183abca83ac8783aca783b38083b0b183a53183a42a83a7d383a69283a8 + e083ab2983aba083b1d283ad2883a4cd83a62183ab2c83a9fe83a478839d8983a4cb83aca483a393839ede83a2e483ab + c383a92f83a46083a49983ad4c83ad5083a7a883a53683a59883aa4783a8db83a45583a47183a59383a6ae83ac7c83b0 + 1d83ae9783ae7383aba783a80483aaf483b04c83af0c83a95083a77d83a61183a61b83a48383ad7183aed083a8bc0b43 + 004300430043004300430043004300430043004300f8fff8fff8fff8fff8fff8fff8fff7fff7fff7fff7ff00313774b4 + """ + + /// Record at unix 1786031642 — leads connected, `numberOfLeadsOffSamples == 10`. Structurally + /// identical, but the one unused leads-off slot pushes the trailing remainder to 5 bytes, which the + /// default padding tolerance rejects. + private static let recordLeadsOffCount10 = """ + aa0128060100cde02f10037961cd001aae746aeb51030a000124000000ffff00f40183f47c83f61983f31b83f42a83f5 + 6083f8bc83f3a683f10a83f15883f2d583f71683f45783f58783f26e83f18983efdb83efc483f51583f5e683f6f283f4 + 7d83f59e83f76283f59883f66383f1d583f4d783fbbc83f59183f5a083f4f483f5c683f7ab83eed783ecbf83f19083f0 + 5a83f1bd83efa383f30b83f48083f00483f37583f3dd83f1e283f0d983f1a983f2f183f0f283f07283ef9983ef5d83f2 + 7c83f3ec83f39d83f2e083f01583eece83ed8b83f06883f10383f19683f42983f08b83f35783f30383edae83ec4083ee + 3583f05e83f16083f3f383ee9b83ed5083f03883ecec83ee8d83f31b83eee883eb4883e87883e0e283e9f483f0d183f0 + 7f83effe83ed8983eef183e9dd83ea0b83ed9783ef3083eb8883e9ad83f03983f6d983f33c83ee9583eb6a83ee9483f3 + 7f83f25483ee0083ee1283ebf383eda583ef0583f05683f3ab83ef1a83f2cb83f45583f48383f0c083eacf83ee5583f2 + 9483f16e83ec5c83eaa683ecb583ecd283f0cf83f38983f39183e9c583e7dd83e9f783e8a983e41783e34b83e1cd83df + 1083dde183d17283d22283c94183bfe583c02483bd9d83bfa683c39483c36183bde583c4c783c7ae83d37d83de9783e3 + 0683e62283e8bd83e78d83eb4983e8e683e99283eaac83ecac83ed1a83efe083efdc83ecdb83f35483f20983ef6783f2 + 8183f56983f26183f40283f82583f44d83eb8d83eb0f83eb3283edc283f80f83fae583f5c183f4f383fc3783fae583f3 + 1a83eeea83ff7883fe3583f5e883fac783ff5983f9ae83f60383f7b883fb8d83f9f083feb783fee18003788009ce8006 + f48002a38005f28006ad800522800cb5800f30800ca7800dda8009c68005f18004e180031180093e8010658010918009 + 558005f98008da801274800e0e8002648004ce8003d9800b73800dd4800d2680133b800b908004e783f8dc83fabe8004 + a6800c9a80180c8016b280180b80170b801340801a21801d1e801947801acd801ba480177c8020b68022a2802cba802c + ec802a6b8030688036da8039fd8040a68043f4804868804b7b804463804cec804fdd8058308055be8055618050d5804f + 85804ebc80556e8062e9805ce18058c1805908805632805eca806349805ee6805a13805b86805dd2806925806655805c + 4780587480620580629b8062358062a4806d0c8063c3805b65805f0d805e4e806b00806a8680656c805fbb805851805b + 4b8057f780596a8061a1806570805e02805e7f8063e58065d48061e680613e805f8b805e5e80579b80536c80543c8053 + e4805d83805aee80569880577f80567e805b73805074804fdd8050ec804f74805355804ef2804f46804f52804d718050 + 1f804f9b8058a28050758044c5804ccd804f49804f3a804b0e804bd5804da380451580467a804b6a804961804959804a + 26804c118041fc8041458042e38042918045b7804de88047b28044e68041498035ff80359d8038a3803b81803de78042 + 8a803b1c8033b68034c6803919803992802fc880334e8036a780350c80397b8041d1803998803d6c8032a68025428033 + 74803db9803761803f7980386c802dc0802f2c8035b08033b8803496802935801bac8021d6802e9b802d30802fbe802b + eb803439802d85802bfe802eb0802fb4802f938030ec802bd0802932802c4680339f802ef5802b368024cc8025e5802a + ba802a29802b80802f3780362c8034b38031e28033e2802eda80261b8022908024cd8029438023448027d3802e948027 + 5a8024088022928029038033dc80330f80290b802fbe8023858016cb8021fe80274f802304801ed68022ff802df38031 + 4380318a802c9680240780264a802766802a47802fa8802d808029958028d0801eeb8017d8801f00802854802ab28034 + 3b802ff78022b2801e018020fd8029f28023ac8023208019a68019c08021cc8028858025ed801afc800c5b801e09801f + fa801df4801df08016aa8012598017e2801c9480233a801f6f801d2f8011d080080e8014a2801d018022a98016218005 + 0b800589800dde8019cf8020168020a2800e2b8003ac800ba6801956801838800ebe800c68800d218009ee800e4b0a43 + 004300430043004300420042004200420042000000f7fff7fff7fff7fff7fff7fff6fff6fff6fff6ff000000fe104255 + """ + + /// Record at unix 1786031625 — the capture's one PARTLY FILLED record, `numberOfECGSamples == 245`. + /// + /// It is the first record of the second reading: the empty record before it is at 1786031624 and the + /// first full 500-sample one is at 1786031626, so the strap began the reading roughly half a second + /// into this record's second and wrote 245 of the region's 500 slots. Same 1584-byte frame, same + /// layout, same leads-off offsets as both full records above. + private static let recordPartlyFilled245 = """ + aa0128060100cde02f10036861cd0009ae746aeb51010a000100000000ffff00f50083ae1e8259e48210008210008210 + 008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210 + 008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210 + 008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210 + 00821000821000821000821000821000821000821000821000821000821000821000821000c21000c21000c21000c210 + 00c21085c25696c328a5c39fc1c3cda0c3e169c3e9d9c3ecc2c3ec8bc3ec7dc3efd1c3f2d9c3f434c3f49ac3f4c6c3f5 + b0c3f6f2c3f847c3f716c3f7dac3f776c3f66ec3f67fc3f8a3c3f980c3f8cac3f79dc3f85dc3f839c3f747c3f8b7c3f8 + fac3f829c3f964c3f923c3f7cec3fa53c3f88ac3f78dc3f8c2c3f8f0c3f73fc3f763c3f867c3f7d5c3f816c3f99bc3f7 + dfc3f7f2c3f8fbc3f830c3f777c3f800c3f7b3c3f864c3f6fdc3f79ec3f882c3f77ac3f748c3f88ec3f907c3f980c3f7 + f4c3f924c3f8c8c3f856c3f802c3f823c3f903c3f90fc3f950c3fab5c3fa82c3f98ec3f91ac3f8bdc3fb13c3fb7ec3fb + 3fc3fbd5c3fa08c3faf3c3fc74c3fa42c3fa90c3fbd5c3fc54c3fc01c3fcf6c3fb4dc3fc34c3fd5ac3fcccc3fdfac3fd + 40c3fbacc3fb30c3fc65c3fbf4c3fb99c3fb7dc3fbc4c3fc6fc3fcf4c3fbadc3fbcec3fb46c3fb58c3fc58c3fb0bc3f9 + 45c3fa54c3fa75c3fa37c3fab2c3fa6cc3faf0c3fab2c3fad5c3fb46c3fb1ac3fc9dc3fbd2c3faa7c3fa98c3fb85c3fd + b0c3fc78c3fb95c3fab7c3fc94c3fe67c3fe15c3fcffc3fa3bc3fd33c3ff86c00062c001dac0013cc3ff22c3fefcc3ff + 78c3fef1c00077c00148c000a1c00088c0011dc00263c0017ec000d6c002d9c0023cc001c3c00287c00322c00340c003 + 18c00460c002eec003e1c0059bc00561c00479c0056cc00640c005e2c00670c0063bc006444005d440043a4002eb4002 + a10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a45 + 004400450045004500450045004500460047000000f9fff9fffafffafffafffafffafffafffbfffcff0000009b973d34 + """ + + /// The payload offset at which `EcgStatusHeader`'s `numberOfECGSamples` lands on the record's real + /// sample count. NOT `Whoop5Ecg.puffinPayloadStart` (11): a type-47 historical record carries a + /// layout marker at [10], a u32 record index at [11] and a u32 unix second at [15] before its body, + /// exactly as `decodeWhoop5HistoricalV2021` documents for versions 20/21. + private static let ecgPayloadStart = 17 + + // MARK: - Helpers + + private func hexToBytes(_ h: String) -> [UInt8] { + var out = [UInt8]() + var nibble: UInt8? + for c in h { + guard let digit = c.hexDigitValue else { continue } + let v = UInt8(digit) + if let hi = nibble { out.append(hi << 4 | v); nibble = nil } else { nibble = v } + } + return out + } + + private var frame11: [UInt8] { hexToBytes(Self.recordLeadsOffCount11) } + private var frame10: [UInt8] { hexToBytes(Self.recordLeadsOffCount10) } + private var framePartial: [UInt8] { hexToBytes(Self.recordPartlyFilled245) } + + /// The sample encoding under test: a 24-bit BIG-endian field whose top 6 bits are a tag and whose + /// low 18 bits are a signed value. + private func sample18(_ blob: [UInt8], _ i: Int) -> Int { + let o = i * 3 + var v = (Int(blob[o] & 0x03) << 16) | (Int(blob[o + 1]) << 8) | Int(blob[o + 2]) + if v & 0x20000 != 0 { v -= 0x40000 } + return v + } + + /// The rival encoding: byte 0 is an opaque status byte and the value is the trailing 16 bits. + /// Kept so the continuity test below can actually discriminate rather than merely agree. + private func sample16(_ blob: [UInt8], _ i: Int) -> Int { + let o = i * 3 + var v = (Int(blob[o + 1]) << 8) | Int(blob[o + 2]) + if v & 0x8000 != 0 { v -= 0x10000 } + return v + } + + private func maxAbsDelta(_ values: [Int]) -> Int { + var m = 0 + for i in 1.. UInt32 { + UInt32(b[o]) | UInt32(b[o + 1]) << 8 | UInt32(b[o + 2]) << 16 | UInt32(b[o + 3]) << 24 + } + + // MARK: - The frames are real and well formed + + func testCapturedFramesPassBothWhoop5CRCs() { + for frame in [frame11, frame10, framePartial] { + XCTAssertEqual(frame.count, 1584) + XCTAssertEqual(frame[8], 47, "type-47 historical record") + XCTAssertEqual(frame[9], 16, "layout version 16") + let check = verifyFrame(frame, family: .whoop5) + XCTAssertTrue(check.ok, "real capture must pass the CRC16 header and CRC32 trailer") + XCTAssertEqual(check.length, 1576) + } + // The unix second the repo's own v20/v21 header model puts at [15] — this is what makes an ECG + // status header starting at 17 impossible: bytes 17 and 18 are its two high bytes. + XCTAssertEqual(u32le(frame11, 15), 1786031627) + XCTAssertEqual(u32le(frame10, 15), 1786031642) + XCTAssertEqual(frame11[17], 0x74) + XCTAssertEqual(frame11[18], 0x6A) + } + + // MARK: - What upstream concludes about the sample width + + func testRawBytesPerSampleCandidatesResolveToThreeOnRealRecord() { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart) else { + return XCTFail("CRC-gated payload extraction failed on a real frame") + } + XCTAssertEqual(payload.count, 1563) + // Unambiguous: exactly one width survives, and it is 3. + XCTAssertEqual(Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload), [3]) + XCTAssertNotNil(Whoop5Ecg.decodeRaw(payload: payload), + "the disambiguating overload must accept a single-candidate buffer") + } + + /// The offset is load-bearing, and wrong offsets fail closed rather than producing a plausible decode. + func testOtherPayloadStartsYieldNoCandidates() { + for start in [Whoop5Ecg.puffinPayloadStart, 19, 21] { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: start) else { + return XCTFail("payload extraction failed at \(start)") + } + XCTAssertEqual(Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload), [], + "offset \(start) must not admit any sample width") + } + } + + // MARK: - The full RawLabradorPacket shape against real bytes + + func testRawLabradorPacketAccountsForEveryByteOfTheRecord() { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed on a real record") + } + XCTAssertEqual(packet.header.numberOfECGSamples, 500) + XCTAssertEqual(packet.rawECGDataRaw.count, 1500) + XCTAssertEqual(packet.bytesPerSample, 3) + XCTAssertEqual(packet.unusedSampleBytes, 0, "a full record fills its region exactly") + XCTAssertEqual(packet.numberOfLeadsOffSamples, 11) + XCTAssertEqual(packet.leadsOffIRaw, [UInt16](repeating: 67, count: 11)) + XCTAssertEqual(packet.leadsOffQRaw, + [UInt16](repeating: 65528, count: 7) + [UInt16](repeating: 65527, count: 4)) + XCTAssertEqual(packet.padding, [0x00]) + + // Every byte of the 1563-byte payload is claimed by exactly one field. The leads-off block is + // charged at its fixed size — which for this record happens to equal the count. + let accounted = Whoop5Ecg.headerLength + + packet.rawECGDataRaw.count + + 1 + + Whoop5Ecg.leadsOffSlotCount * 2 + + Whoop5Ecg.leadsOffSlotCount * 2 + + packet.padding.count + XCTAssertEqual(accounted, payload.count, "no unaccounted bytes in the record body") + } + + // MARK: - Where the header actually is + + /// The nine header bytes from `heartKeyProgress` onward align correctly at offset 17 and carry real + /// values; the eight before them do not. Both halves are pinned so a future fix has to move both. + func testHeaderTailIsCorrectAndHeaderHeadIsNot() { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart), + let header = EcgStatusHeader(payload: payload) else { + return XCTFail("header parse failed") + } + // Correctly aligned tail — these agree with the record. + XCTAssertEqual(header.numberOfECGSamples, 500) + XCTAssertEqual(header.heartKeyProgress, .percent(0)) + XCTAssertEqual(header.heartKeyHRV, 65535, "0xFFFF unknown sentinel while the reading is running") + XCTAssertEqual(header.heartKeyAverageHR, 0) + XCTAssertEqual(header.heartKeyHR, 0) + XCTAssertEqual(header.heartKeyStressScore, 0) + XCTAssertEqual(header.heartKeyUnreadableReason, 0) + + // Misaligned head — `signalQualityRaw` and `statusFlags` are the unix timestamp's two high + // bytes, and the four booleans read unmapped preamble rather than 0/1 flags. + XCTAssertEqual(header.signalQualityRaw, 0x74, "this is frame[17], a unix byte, not a quality grade") + XCTAssertEqual(header.signalQuality, .unknown, "116 is outside the 0...3 enum") + XCTAssertEqual(header.statusFlags, 0x6A, "this is frame[18], the other unix byte") + XCTAssertEqual(payload[2], 0xEB, "'heartKeyStarted' reads a 235, which is not a Bool") + XCTAssertEqual(payload[3], 0x51, "'heartKeyIsRunning' reads an 81, which is not a Bool") + } + + /// The record's real signal-quality and progress bytes, located by their behaviour across the + /// capture: quality climbs 1 -> 3 and progress ramps 0 -> 100 over the 30-second reading. Progress + /// at frame[25] is what `EcgStatusHeader` already reads correctly; quality at frame[21] is not. + func testRealSignalQualityAndProgressBytes() { + XCTAssertEqual(frame11[21], 1, "signal quality: low, at the start of the reading") + XCTAssertEqual(frame10[21], 3, "signal quality: high, 15 seconds in") + XCTAssertEqual(frame11[25], 0, "progress 0%") + XCTAssertEqual(frame10[25], 36, "progress 36%") + // The repo's header maps frame[21] onto 'heartKeyIsStoppedAndComplete' — four bytes early. + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart), + let header = EcgStatusHeader(payload: payload) else { return XCTFail("header parse failed") } + XCTAssertTrue(header.heartKeyIsStoppedAndComplete, + "reads frame[21] == 1, which is really the signal-quality grade") + } + + // MARK: - Sample decode + + func testFirstTwelveSamplesDecodeAsSigned18BitBigEndian() { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + let blob = packet.rawECGDataRaw + let first12 = (0..<12).map { sample18(blob, $0) } + XCTAssertEqual(first12, [-46361, -46615, -47659, -47357, -45997, -47084, -46826, -46579, -47218, -45762, -44790, -44613]) + + // Every sample in this record carries tag 0b100000 — leads connected. + for i in 0..> 2, 0b100000, "sample \(i) tag") + } + } + + func testSecondRecordFirstTwelveSamples() { + guard let payload = Whoop5Ecg.innerPayload(frame10, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + let blob = packet.rawECGDataRaw + XCTAssertEqual((0..<12).map { sample18(blob, $0) }, [-2948, -2535, -3301, -3030, -2720, -1860, -3162, -3830, -3752, -3371, -2282, -2985]) + let all = (0..= 0 }.count, 304) + } + + /// The discriminating evidence for 18 bits over "status byte + 16-bit value". + /// + /// In the first record the trace sits on a large negative electrode offset that leaves the 16-bit + /// signed range. Read as 18 bits it is a continuous waveform; read as a trailing 16-bit value it + /// wraps full scale between adjacent 500 Hz samples. A 65428-count step between neighbouring samples + /// of a settling ECG is not physical, so the 18-bit reading is the only one the data supports. + func testEighteenBitReadingIsContinuousWhereSixteenBitWraps() { + guard let payload = Whoop5Ecg.innerPayload(frame11, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + let blob = packet.rawECGDataRaw + let n = Int(packet.header.numberOfECGSamples) + let as18 = (0.. [UInt16] { + (0..<11).map { UInt16(frame[start + $0 * 2]) | UInt16(frame[start + $0 * 2 + 1]) << 8 } + } + // Count 11: every slot carries a value. + XCTAssertEqual(slots(frame11, 1535), [UInt16](repeating: 67, count: 11)) + XCTAssertEqual(slots(frame11, 1557), + [UInt16](repeating: 65528, count: 7) + [UInt16](repeating: 65527, count: 4)) + // Count 10: ten values then a zeroed slot, in BOTH arrays, at the SAME offsets. + XCTAssertEqual(slots(frame10, 1535), [67, 67, 67, 67, 67, 66, 66, 66, 66, 66, 0]) + XCTAssertEqual(slots(frame10, 1557), + [65527, 65527, 65527, 65527, 65527, 65527, 65526, 65526, 65526, 65526, 0]) + // And exactly one byte follows the block before the CRC32 trailer. + XCTAssertEqual(1579 + 1 + 4, frame10.count) + } + + /// Regression: a real count=10 record is ACCEPTED. Under the packed reading its remainder was five + /// bytes against `defaultMaxPadding = 3`, so `rawBytesPerSampleCandidates` returned `[]` and the + /// record was discarded before any field was read — 227 of the capture's 351 populated records. + func testPopulatedRealRecordIsNotDiscarded() { + for frame in [frame11, frame10] { + guard let payload = Whoop5Ecg.innerPayload(frame, payloadStart: Self.ecgPayloadStart) else { + return XCTFail("payload extraction failed") + } + XCTAssertEqual(Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload), [3], + "a real, well-formed, CRC-valid record must resolve to width 3") + XCTAssertNotNil(Whoop5Ecg.decodeRaw(payload: payload)) + // Accepted on its OWN remainder, not by widening the tolerance. + XCTAssertEqual(Whoop5Ecg.decodeRaw(payload: payload)?.padding, [0x00]) + } + } + + /// With the count below the block's capacity, the ten valid values come off the front of each fixed + /// array. The packed reading started Q two bytes early — it picked up the unused eleventh I slot as + /// Q[0] and pushed the tenth real Q value out into the "padding". + func testPartiallyFilledLeadsOffBlockKeepsQAligned() { + guard let payload = Whoop5Ecg.innerPayload(frame10, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + XCTAssertEqual(packet.numberOfLeadsOffSamples, 10) + XCTAssertEqual(packet.leadsOffIRaw, [67, 67, 67, 67, 67, 66, 66, 66, 66, 66]) + XCTAssertEqual(packet.leadsOffQRaw, [65527, 65527, 65527, 65527, 65527, 65527, 65526, 65526, 65526, 65526]) + XCTAssertNotEqual(packet.leadsOffQRaw.first, 0, "no spurious leading slot") + XCTAssertEqual(packet.leadsOffQRaw.last, 65526, "the tenth real Q value, previously lost") + // The unused eleventh slot of each array is dropped, not carried, and is not padding either. + XCTAssertEqual(packet.leadsOffIRaw.count, 10) + XCTAssertEqual(packet.leadsOffQRaw.count, 10) + XCTAssertEqual(packet.padding, [0x00]) + } + + // MARK: - The fixed-size sample region + + /// The geometry, read straight off the frame with no decoder involved. This is the evidence the + /// end-anchored `decodeRaw` rests on: a record carrying 245 samples is byte-for-byte the same size as + /// a record carrying 500, its sample data stops at frame[769] = 34 + 245 * 3, everything from there + /// to the leads-off block is zero, and the block is at the SAME frame offsets as both full records'. + func testPartlyFilledRecordHasAFullSizeSampleRegion() { + XCTAssertEqual(framePartial.count, frame11.count) + XCTAssertEqual(framePartial.count, 1584) + // 245 samples, not 500 — and the record's own second sits between the last empty record and the + // first full one, so this is the reading starting part-way through a second. + XCTAssertEqual(UInt16(framePartial[32]) | UInt16(framePartial[33]) << 8, 245) + XCTAssertEqual(u32le(framePartial, 15), 1786031625) + XCTAssertEqual(u32le(frame11, 15), 1786031627) + + XCTAssertNotEqual(framePartial[768], 0, "the last byte of sample 244") + XCTAssertTrue(framePartial[769..<1534].allSatisfy { $0 == 0 }, + "the region's unused capacity is zero-filled, not absent") + // Same fixed leads-off offsets as the two full records. + XCTAssertEqual(framePartial[1534], 10) + func slots(_ frame: [UInt8], _ start: Int) -> [UInt16] { + (0..<11).map { UInt16(frame[start + $0 * 2]) | UInt16(frame[start + $0 * 2 + 1]) << 8 } + } + XCTAssertEqual(slots(framePartial, 1535), [69, 68, 69, 69, 69, 69, 69, 69, 70, 71, 0]) + XCTAssertEqual(slots(framePartial, 1557), + [65529, 65529, 65530, 65530, 65530, 65530, 65530, 65530, 65531, 65532, 0]) + XCTAssertEqual(framePartial[1579], 0) + } + + /// Regression: the last populated record the decoder discarded. Reading the region as exactly + /// `n * bytesPerSample` put the count byte on payload[752] — a zero inside the fill — which yielded an + /// empty leads-off block and 766 trailing bytes against `defaultMaxPadding = 3`. + func testPartlyFilledRecordDecodesAtTheStreamWidth() { + guard let payload = Whoop5Ecg.innerPayload(framePartial, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed on the partly-filled record") + } + XCTAssertEqual(payload.count, 1563, "same body length as a full record") + XCTAssertEqual(payload[752], 0, "where the region-is-exactly-full reading looked for the count") + + XCTAssertEqual(packet.header.numberOfECGSamples, 245) + XCTAssertEqual(packet.rawECGDataRaw.count, 735, "245 * 3 — only the VALID slots are carried") + XCTAssertEqual(packet.bytesPerSample, 3) + XCTAssertEqual(packet.unusedSampleBytes, 765) + XCTAssertEqual(packet.sampleRegionBytes, 1500, "the same region every full record carries") + XCTAssertEqual(packet.numberOfLeadsOffSamples, 10) + XCTAssertEqual(packet.leadsOffIRaw, [69, 68, 69, 69, 69, 69, 69, 69, 70, 71]) + XCTAssertEqual(packet.leadsOffQRaw, + [65529, 65529, 65530, 65530, 65530, 65530, 65530, 65530, 65531, 65532]) + XCTAssertEqual(packet.padding, [0x00], "accepted on its own remainder, not a widened tolerance") + + // The blob's ends, verbatim. No encoding claim is made for this record: it is the first half + // second of a reading, and its samples are still settling — see the tag census below. + XCTAssertEqual(Array(packet.rawECGDataRaw.prefix(3)), [0x83, 0xAE, 0x1E]) + XCTAssertEqual(Array(packet.rawECGDataRaw.suffix(3)), [0x40, 0x02, 0xA1]) + + // Every byte of the 1563-byte body is claimed by exactly one field, with BOTH fixed-size + // containers charged at their full size and the counts charged at what they declare. + let accounted = Whoop5Ecg.headerLength + + packet.rawECGDataRaw.count + + packet.unusedSampleBytes + + Whoop5Ecg.leadsOffBlockLength + + packet.padding.count + XCTAssertEqual(accounted, payload.count, "17 + 735 + 765 + 45 + 1") + } + + /// A partly-filled record does NOT determine its own sample width, and the enumerator says so rather + /// than picking one: 245 samples inside a 1500-byte region fits 3 bytes per sample and 4 alike. The + /// full records are where the width comes from — they resolve unaided. + func testPartlyFilledRecordReportsItsWidthAsAmbiguous() { + guard let payload = Whoop5Ecg.innerPayload(framePartial, payloadStart: Self.ecgPayloadStart) else { + return XCTFail("payload extraction failed") + } + XCTAssertEqual(Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload), [3, 4]) + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload), + "an ambiguous buffer must refuse to decode, not guess") + // 1 and 2 are ruled out by real sample bytes sitting in what they would call unused capacity. + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 1)) + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2)) + } + + /// The tags this record's samples carry, as a census rather than an interpretation. The two full + /// records are uniformly 0b100000; this one is not, which is the reason no encoding claim is made + /// for its values here — the electrode input is still settling half a second into the reading. + func testPartlyFilledRecordSampleTagsAreNotUniform() { + guard let payload = Whoop5Ecg.innerPayload(framePartial, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + var census = [UInt8: Int]() + for i in 0..> 2, default: 0] += 1 + } + XCTAssertEqual(census, [48: 176, 32: 65, 16: 4]) + } + + /// The count=10 record accounts for every byte too — with the leads-off block charged at its FIXED + /// size rather than at the count. + func testCountTenRecordAccountsForEveryByte() { + guard let payload = Whoop5Ecg.innerPayload(frame10, payloadStart: Self.ecgPayloadStart), + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) else { + return XCTFail("decodeRaw failed") + } + XCTAssertEqual(payload.count, 1563) + XCTAssertEqual(packet.header.numberOfECGSamples, 500) + XCTAssertEqual(packet.rawECGDataRaw.count, 1500) + XCTAssertEqual(packet.unusedSampleBytes, 0) + let accounted = Whoop5Ecg.headerLength + + packet.rawECGDataRaw.count + + packet.unusedSampleBytes + + 1 + + Whoop5Ecg.leadsOffSlotCount * 2 + + Whoop5Ecg.leadsOffSlotCount * 2 + + packet.padding.count + XCTAssertEqual(accounted, payload.count, "17 + 1500 + 0 + 1 + 22 + 22 + 1") + } +} diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgTests.swift index 043904808e..dd6b3cc120 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5EcgTests.swift @@ -45,6 +45,18 @@ final class Whoop5EcgTests: XCTestCase { values.flatMap { [UInt8($0 & 0xFF), UInt8($0 >> 8)] } } + /// The raw record's leads-off tail: the count byte, then `Whoop5Ecg.leadsOffSlotCount` i16 I slots, + /// then the same again for Q. The block is FIXED-SIZE on real hardware (see + /// `Whoop5EcgRawHardwareTests`), so these fixtures zero-fill the slots the count does not reach + /// instead of packing the arrays to the count. + private func leadsOffBlock(i: [UInt16], q: [UInt16]) -> [UInt8] { + precondition(i.count == q.count && i.count <= Whoop5Ecg.leadsOffSlotCount) + let pad = Whoop5Ecg.leadsOffSlotCount - i.count + return [UInt8(i.count)] + + u16le(i + [UInt16](repeating: 0, count: pad)) + + u16le(q + [UInt16](repeating: 0, count: pad)) + } + /// Wrap a payload in a real, CRC-correct puffin frame using the shipped builder, so the frame-level /// tests exercise the same envelope the strap speaks. private func puffinFrame(type: UInt8, payload: [UInt8]) -> [UInt8] { @@ -201,7 +213,7 @@ final class Whoop5EcgTests: XCTestCase { let rawBlob: [UInt8] = Array(0..<12) // 4 samples × 3 bytes let leadsOffI: [UInt16] = [1, 2] let leadsOffQ: [UInt16] = [3, 4] - let payload = header(samples: 4) + rawBlob + [2] + u16le(leadsOffI) + u16le(leadsOffQ) + let payload = header(samples: 4) + rawBlob + leadsOffBlock(i: leadsOffI, q: leadsOffQ) let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 3) XCTAssertNotNil(packet) @@ -214,21 +226,81 @@ final class Whoop5EcgTests: XCTestCase { XCTAssertEqual(packet?.header.numberOfECGSamples, 4) } + /// The SAMPLE region is fixed-size too: `numberOfECGSamples` says how many of its slots are valid, + /// and the rest is zero-filled capacity that still has to be stepped over to reach the leads-off + /// block. Only the valid bytes are carried; the unused ones are counted (see the real record in + /// `Whoop5EcgRawHardwareTests`, where the count is 245 inside a 500-slot region). + func testRawPartlyFilledSampleRegionIsSteppedOverNotCarried() { + let samples: [UInt8] = [1, 2, 3, 4, 5, 6] // 3 valid samples × 2 bytes + let unused = [UInt8](repeating: 0, count: 8) // 4 more slots the record did not fill + let payload = header(samples: 3) + samples + unused + leadsOffBlock(i: [5], q: [6]) + + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2) + XCTAssertEqual(packet?.rawECGDataRaw, samples, "only the valid slots are carried") + XCTAssertEqual(packet?.unusedSampleBytes, 8) + XCTAssertEqual(packet?.sampleRegionBytes, 14) + XCTAssertEqual(packet?.bytesPerSample, 2, "still count ÷ numberOfECGSamples") + XCTAssertEqual(packet?.numberOfLeadsOffSamples, 1) + XCTAssertEqual(packet?.leadsOffIRaw, [5]) + XCTAssertEqual(packet?.leadsOffQRaw, [6]) + XCTAssertEqual(packet?.padding, [], "the unused capacity is region, not trailing padding") + } + + /// The zero-fill is what makes the step-over safe. A non-zero byte between the declared samples and + /// the leads-off block means the block is not where this width says it is, so the decode fails closed + /// rather than skipping over bytes that might be data. + func testRawRejectsNonZeroBytesInTheUnusedSampleRegion() { + // The stray byte leads the unused span, so it stays inside that span at every end position the + // padding budget allows — a byte at the span's tail would simply become the count byte of a + // one-shorter record, which is a different (and legitimate) reading, not a violation. + let region: [UInt8] = [1, 2, 3, 4, 5, 6] + [0x09, 0, 0, 0, 0, 0, 0, 0] + let payload = header(samples: 3) + region + leadsOffBlock(i: [5], q: [6]) + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2)) + // With the same bytes declared as samples the record is fine — it is the SKIPPING that is gated. + let full = header(samples: 7) + region + leadsOffBlock(i: [5], q: [6]) + XCTAssertEqual(Whoop5Ecg.decodeRaw(payload: full, bytesPerSample: 2)?.rawECGDataRaw, region) + } + + /// A zero count still carries the full fixed block — the slots are simply all unused. func testRawWithNoLeadsOffSamples() { - let payload = header(samples: 2) + [0xAA, 0xBB, 0xCC, 0xDD] + [0] + let payload = header(samples: 2) + [0xAA, 0xBB, 0xCC, 0xDD] + leadsOffBlock(i: [], q: []) let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2) XCTAssertEqual(packet?.numberOfLeadsOffSamples, 0) XCTAssertEqual(packet?.leadsOffIRaw, []) XCTAssertEqual(packet?.leadsOffQRaw, []) XCTAssertEqual(packet?.rawECGDataRaw, [0xAA, 0xBB, 0xCC, 0xDD]) + XCTAssertEqual(packet?.padding, []) + } + + /// The valid values come off the FRONT of each fixed array, and the unused slots are dropped rather + /// than shifting Q or leaking into the padding — the defect the real capture exposed. + func testRawPartiallyFilledLeadsOffBlockKeepsQAligned() { + let payload = header(samples: 2) + [0, 0, 0, 0] + + leadsOffBlock(i: [11, 12, 13], q: [21, 22, 23]) + let packet = Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2) + XCTAssertEqual(packet?.numberOfLeadsOffSamples, 3) + XCTAssertEqual(packet?.leadsOffIRaw, [11, 12, 13]) + XCTAssertEqual(packet?.leadsOffQRaw, [21, 22, 23]) + XCTAssertEqual(packet?.padding, [], "the unused slots are part of the block, not trailing padding") } func testRawRejectsTruncatedLeadsOffArrays() { - // Declares 3 leads-off samples but only carries I (and only partly). + // Declares 3 leads-off samples but carries neither array in full. let payload = header(samples: 2) + [0, 0, 0, 0] + [3] + u16le([1, 2, 3]) XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2)) } + /// The block holds a fixed number of slots, so a count above it cannot describe this layout and must + /// fail closed rather than read past the block. + func testRawRejectsLeadsOffCountAboveTheBlockCapacity() { + // The blob is exactly full, so the block can only be where the count byte is — this pins the + // capacity check itself rather than letting the end-of-record search reject the buffer earlier. + let overCount = UInt8(Whoop5Ecg.leadsOffSlotCount + 1) + let payload = header(samples: 2) + [0, 0, 0, 0] + [overCount] + + [UInt8](repeating: 0, count: Whoop5Ecg.leadsOffSlotCount * 4) + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 2)) + } + func testRawRejectsMissingLeadsOffCountByte() { // The raw blob consumes the whole buffer, leaving no room for the count byte. let payload = header(samples: 2) + [0, 0, 0, 0] @@ -258,13 +330,14 @@ final class Whoop5EcgTests: XCTestCase { func testRawRejectsShortHeaderAndZeroWidth() { XCTAssertNil(Whoop5Ecg.decodeRaw(payload: [1, 2, 3], bytesPerSample: 2)) - let payload = header(samples: 2) + [0, 0, 0, 0] + [0] + let payload = header(samples: 2) + [0, 0, 0, 0] + leadsOffBlock(i: [], q: []) XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, bytesPerSample: 0)) } func testRawSampleWidthCandidatesAreEnumeratedNotGuessed() { // 4 samples × 2 bytes, then a 1-sample leads-off tail. Width 2 must be admitted. - let payload = header(samples: 4) + [UInt8](repeating: 0x11, count: 8) + [1] + u16le([7]) + u16le([8]) + let payload = header(samples: 4) + [UInt8](repeating: 0x11, count: 8) + + leadsOffBlock(i: [7], q: [8]) let candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload) XCTAssertTrue(candidates.contains(2), "width 2 must be structurally admissible") // Whatever the full candidate set is, the single-candidate decoder must agree with it: it either @@ -278,11 +351,12 @@ final class Whoop5EcgTests: XCTestCase { } func testRawAmbiguousBufferRefusesToDecode() { - // Hand-built so several widths parse: a long all-zero tail lets many split points look valid. - let payload = header(samples: 1) + [UInt8](repeating: 0, count: 6) - let candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload, maxPadding: 8) - XCTAssertGreaterThan(candidates.count, 1, "fixture is meant to be ambiguous") - XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload, maxPadding: 8)) + // Hand-built so several widths parse: with a single sample, an all-zero tail long enough to hold + // the fixed leads-off block leaves every width inside the padding tolerance. + let payload = header(samples: 1) + [UInt8](repeating: 0, count: 49) + let candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload: payload) + XCTAssertEqual(candidates, [1, 2, 3, 4], "fixture is meant to be ambiguous") + XCTAssertNil(Whoop5Ecg.decodeRaw(payload: payload)) } // MARK: - Frame level (CRC gating) @@ -329,7 +403,7 @@ final class Whoop5EcgTests: XCTestCase { } func testRawFrameDecodesThroughAValidPuffinEnvelope() { - let payload = header(samples: 2) + [1, 2, 3, 4] + [1] + u16le([5]) + u16le([6]) + let payload = header(samples: 2) + [1, 2, 3, 4] + leadsOffBlock(i: [5], q: [6]) let frame = puffinFrame(type: 0x2F, payload: payload) let packet = Whoop5Ecg.decodeRawFrame(frame, bytesPerSample: 2) XCTAssertEqual(packet?.rawECGDataRaw, [1, 2, 3, 4]) @@ -338,7 +412,7 @@ final class Whoop5EcgTests: XCTestCase { } func testRawFrameRejectsBadCRC() { - let payload = header(samples: 2) + [1, 2, 3, 4] + [0] + let payload = header(samples: 2) + [1, 2, 3, 4] + leadsOffBlock(i: [], q: []) var frame = puffinFrame(type: 0x2F, payload: payload) frame[frame.count - 2] ^= 0xFF XCTAssertNil(Whoop5Ecg.decodeRawFrame(frame, bytesPerSample: 2)) @@ -376,6 +450,191 @@ final class Whoop5EcgTests: XCTestCase { XCTAssertFalse(Whoop5Ecg.plausibleFilteredPayload([UInt8](repeating: 0, count: 64))) } + // MARK: - Structural triage: sample width + + /// THE DEFECT this widening fixes. The triage hardcoded `n * 2`, so a payload whose samples are 3 + /// bytes wide missed the length agreement by a mile, failed the padding budget, and was discarded + /// without a trace — while the probe truthfully reported that no frame passed. The raw side has + /// enumerated widths since it was written (`rawBytesPerSampleCandidates`), and a populated raw flash + /// record read off hardware was 1500 bytes against `numberOfECGSamples = 500`, i.e. 3. + func testTriageAcceptsAThreeBytePerSampleBufferAndNamesTheWidth() { + let payload = header(samples: 4) + [UInt8](repeating: 0x11, count: 12) // 4 samples × 3 bytes + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(payload), [3]) + XCTAssertTrue(Whoop5Ecg.plausibleFilteredPayload(payload)) + } + + /// The widening is a SUPERSET, never a replacement: 2 stays first in the candidate order and a + /// 2-byte buffer still passes exactly as it did. + func testTriageStillAcceptsTwoBytesPerSampleAndReportsItFirst() { + let payload = header(samples: 3) + i16le([1, 2, 3]) + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(payload), [2]) + XCTAssertTrue(Whoop5Ecg.plausibleFilteredPayload(payload)) + XCTAssertEqual(Whoop5Ecg.filteredWidthCandidates.first, 2) + } + + func testTriageAcceptsFourBytesPerSample() { + let payload = header(samples: 5) + [UInt8](repeating: 0x22, count: 20) + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(payload), [4]) + } + + /// When two widths both fit inside the pad4 budget the buffer genuinely does not determine the + /// answer, and the triage says so rather than picking one — the same contract + /// `rawBytesPerSampleCandidates` has. + func testTriageReportsEveryWidthAnAmbiguousBufferAdmits() { + let payload = header(samples: 2) + [UInt8](repeating: 0x33, count: 8) // 25 bytes total + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(payload), [3, 4]) + } + + /// Only the WIDTH assumption moved. Every other guard is unchanged, so a buffer that fails a field + /// check is still rejected even when its length agrees perfectly with one of the new widths. + func testWideningTheWidthDoesNotLoosenAnyOtherGuard() { + let threeByteBody = [UInt8](repeating: 0x11, count: 12) // exact under width 3 + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples: 4) + threeByteBody), [3]) + // …and each guard, one at a time, still rules it out. + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(started: 7, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(running: 2, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(stoppedAndComplete: 9, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(leadsOn: 3, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(signalQuality: 9, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(arrhythmiaResult: 9, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(arrhythmiaStatus: 9, samples: 4) + threeByteBody).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples: 0)).isEmpty) + } + + /// A genuinely malformed buffer is still rejected under EVERY admitted width — the widening must not + /// turn the triage into something that matches anything. + func testTriageStillRejectsBuffersNoWidthExplains() { + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples: 7) + i16le([1, 2, 3])).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples: 40) + i16le([1, 2])).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates( + header(samples: 1) + i16le([1]) + [UInt8](repeating: 0, count: 40)).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates([]).isEmpty) + XCTAssertFalse(Whoop5Ecg.plausibleFilteredPayload(header(samples: 7) + i16le([1, 2, 3]))) + } + + /// `numberOfECGSamples` is a u16 off the wire and the width comes from a caller, so the offset math + /// is overflow-checked rather than trusted — it would trap in Swift and wrap NEGATIVE in the Kotlin + /// twin. A rejection is the correct outcome, not a crash. + func testTriageWidthMathCannotOverflow() { + let payload = header(samples: 65_535) + [UInt8](repeating: 0, count: 8) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(payload, widths: [Int.max, 2, 3]).isEmpty) + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(payload, widths: [0, -1]).isEmpty) + } + + func testTriageWidthsAreReportedThroughACRCGatedFrameToo() { + let payload = header(samples: 4) + [UInt8](repeating: 0x11, count: 12) + var frame = puffinFrame(type: 0x28, payload: payload) + XCTAssertEqual(Whoop5Ecg.filteredBytesPerSampleCandidates(frame: frame), [3]) + frame[frame.count - 1] ^= 0xFF // a bad CRC never reaches a field read + XCTAssertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(frame: frame).isEmpty) + } + + // MARK: - Unclassified-frame census + + /// The census exists for the frames the triage says NO to. A heuristic that logs only its own hits + /// destroys the evidence for its own misses: the 2-byte assumption above discarded every 3-byte + /// frame, and the report then said, truthfully, that nothing passed. + func testCensusRecordsAFrameTheTriageRejects() { + // signalQualityRaw = 0xEE fails the very first guard, so the triage rejects this outright. + let frame = puffinFrame(type: 0x1A, payload: [UInt8](repeating: 0xEE, count: 20)) + XCTAssertFalse(Whoop5Ecg.plausibleFilteredFrame(frame), "fixture must be a triage MISS") + + var census = Whoop5EcgProbe.FrameCensus() + census.record(frame: frame) + XCTAssertEqual(census.framesSeen, 1) + XCTAssertEqual(census.buckets.count, 1) + XCTAssertEqual(census.buckets[0].typeByte, 0x1A) + XCTAssertEqual(census.buckets[0].count, 1) + + let sample = census.buckets[0].samples[0] + XCTAssertEqual(sample.frameLength, frame.count) + XCTAssertEqual(sample.payloadLength, 21) // 20 + the envelope's pad4 byte + XCTAssertEqual(sample.numberOfECGSamples, 0xEEEE) // read, not believed + XCTAssertTrue(sample.widths.isEmpty) + XCTAssertTrue(sample.headHex.hasPrefix("aa")) + XCTAssertEqual(census.lines[0], " type=0x1a frames=1") + XCTAssertTrue(census.lines[1].contains("widths=none")) + XCTAssertTrue(census.lines[1].contains("payload=21")) + } + + /// A HIT is censused too — the census is the whole record of the window, not a rejects bin. + func testCensusRecordsTriageHitsWithTheWidthThatAgreed() { + let frame = puffinFrame(type: 0x28, payload: header(samples: 4) + [UInt8](repeating: 0x11, count: 12)) + XCTAssertTrue(Whoop5Ecg.plausibleFilteredFrame(frame)) + var census = Whoop5EcgProbe.FrameCensus() + census.record(frame: frame) + XCTAssertEqual(census.buckets[0].samples[0].widths, [3]) + XCTAssertEqual(census.buckets[0].samples[0].numberOfECGSamples, 4) + XCTAssertTrue(census.lines[1].contains("widths=3")) + } + + /// A frame whose CRC does not check out yields no decoded field — the census records its shape and + /// its bytes, and says `payload=?` rather than reading a header out of an unverified buffer. + func testCensusReadsNoFieldOutOfAnUnverifiedFrame() { + var frame = puffinFrame(type: 0x28, payload: header(samples: 3) + i16le([1, 2, 3])) + frame[frame.count - 1] ^= 0xFF + var census = Whoop5EcgProbe.FrameCensus() + census.record(frame: frame) + XCTAssertNil(census.buckets[0].samples[0].payloadLength) + XCTAssertNil(census.buckets[0].samples[0].numberOfECGSamples) + XCTAssertTrue(census.buckets[0].samples[0].widths.isEmpty) + XCTAssertTrue(census.lines[1].contains("payload=? samples=? widths=none")) + } + + /// A 1 Hz stream must not be able to grow the report without bound — but nothing may be dropped in + /// silence either. Past the per-type sample cap the frames are COUNTED. + func testCensusCapsSamplesPerTypeButKeepsCounting() { + let frame = puffinFrame(type: 0x30, payload: [UInt8](repeating: 0xEE, count: 20)) + var census = Whoop5EcgProbe.FrameCensus() + for _ in 0..<50 { census.record(frame: frame) } + XCTAssertEqual(census.framesSeen, 50) + XCTAssertEqual(census.buckets.count, 1) + XCTAssertEqual(census.buckets[0].count, 50) + XCTAssertEqual(census.buckets[0].samples.count, Whoop5EcgProbe.FrameCensus.maxSamplesPerType) + XCTAssertTrue(census.lines.contains { $0.contains("+47 more of this type, not recorded") }) + } + + func testCensusCapsDistinctTypesAndCountsTheOverflow() { + var census = Whoop5EcgProbe.FrameCensus() + for type in 0..<(Whoop5EcgProbe.FrameCensus.maxTypes + 4) { + census.record(frame: puffinFrame(type: UInt8(type), payload: [UInt8](repeating: 0xEE, count: 20))) + } + XCTAssertEqual(census.buckets.count, Whoop5EcgProbe.FrameCensus.maxTypes) + XCTAssertEqual(census.framesBeyondTypeCap, 4) + XCTAssertEqual(census.framesSeen, Whoop5EcgProbe.FrameCensus.maxTypes + 4) + XCTAssertTrue(census.lines.contains { $0.contains("4 frame(s) of further type bytes past the") }) + } + + func testCensusIgnoresBuffersTooShortToCarryATypeByte() { + var census = Whoop5EcgProbe.FrameCensus() + census.record(frame: []) + census.record(frame: [UInt8](repeating: 0xAA, count: 8)) + XCTAssertTrue(census.isEmpty) + XCTAssertTrue(census.buckets.isEmpty) + } + + /// The census is only useful if the operator can COPY it: it belongs in the same report sheet as the + /// verdict, right under the triage result it is the check on. + func testReportCarriesTheCensusBesideTheTriageResult() { + let rejected = puffinFrame(type: 0x1A, payload: [UInt8](repeating: 0xEE, count: 20)) + var census = Whoop5EcgProbe.FrameCensus() + census.record(frame: rejected) + let text = Whoop5EcgProbe.report( + steps: [sent(139, arg: 1, .success), sent(124, arg: Whoop5Ecg.ControlSignal.start.rawValue, .success)], + ecgPacketsSeen: 0, candidateFrames: [], windowSeconds: 30, census: census) + // The old, evidence-free version of this run said only this line. Now the bytes are beside it. + XCTAssertTrue(text.contains("Candidate packet types: none — no frame passed the structural triage.")) + XCTAssertTrue(text.contains("Unclassified-frame census")) + XCTAssertTrue(text.contains("type=0x1a frames=1")) + XCTAssertTrue(text.contains("widths=none")) + XCTAssertTrue(text.contains(census.buckets[0].samples[0].headHex)) + } + + func testReportSaysSoWhenNoUnclassifiedFrameArrivedAtAll() { + let text = Whoop5EcgProbe.report(steps: [], ecgPacketsSeen: 0, candidateFrames: [], windowSeconds: 30) + XCTAssertTrue(text.contains("no unclassified frame arrived at all")) + } + // MARK: - Commands func testCommandOpcodesMatchTheRepoProtocolTable() { @@ -484,6 +743,7 @@ final class Whoop5EcgTests: XCTestCase { label: "\(name)(\(cmd))", outcome: outcome, requestsRealtimeData: Whoop5Ecg.requestsRealtimeData(cmd: cmd, arg: arg), + sentArgument: arg, replyHex: replyHex) } @@ -517,10 +777,10 @@ final class Whoop5EcgTests: XCTestCase { // the old precedence turned one loose match into an unhedged "not blocked". let failed = [sent(124, arg: 1, .failure)] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: failed, ecgPacketsSeen: 12, windowSeconds: 30), - .dataRequestRefused(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124)"])) + .dataRequestRefused(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1"])) let unsupported = [sent(139, arg: 1, .unsupported)] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: unsupported, ecgPacketsSeen: 12, windowSeconds: 30), - .opcodeUnsupported(commands: ["TOGGLE_LABRADOR_FILTERED(139)"])) + .opcodeUnsupported(commands: ["TOGGLE_LABRADOR_FILTERED(139) arg=1"])) // With no contrary result code, candidates are the verdict — as candidates, not as proof. let ok = [sent(124, arg: 1, .success)] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: ok, ecgPacketsSeen: 12, windowSeconds: 30), @@ -543,7 +803,7 @@ final class Whoop5EcgTests: XCTestCase { sent(124, arg: Whoop5Ecg.ControlSignal.start.rawValue, .failure), ] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .dataRequestRefused(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124)"])) + .dataRequestRefused(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1"])) } func testVerdictAllSuccessButSilentIsTheSilentNoOpCase() { @@ -578,7 +838,7 @@ final class Whoop5EcgTests: XCTestCase { let steps = [sent(123, arg: Whoop5Ecg.WristSelection.left.rawValue, .success, replyHex: "aa010c000100271124d77b81010100007ce76722")] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .noDataRequested(commands: ["SELECT_WRIST(123)"])) + .noDataRequested(commands: ["SELECT_WRIST(123) arg=1"])) let text = Whoop5EcgProbe.report(steps: steps, ecgPacketsSeen: 0, candidateFrames: [], windowSeconds: 30) XCTAssertTrue(text.contains("NOT A TEST")) @@ -596,7 +856,7 @@ final class Whoop5EcgTests: XCTestCase { let steps = [sent(123, arg: Whoop5Ecg.WristSelection.right.rawValue, .failure, replyHex: "aa010c000100271124217bcc000100000213163d")] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .commandRefused(commands: ["SELECT_WRIST(123)"])) + .commandRefused(commands: ["SELECT_WRIST(123) arg=0"])) let text = Whoop5EcgProbe.report(steps: steps, ecgPacketsSeen: 0, candidateFrames: [], windowSeconds: 30) XCTAssertTrue(text.contains("REFUSED")) @@ -612,9 +872,9 @@ final class Whoop5EcgTests: XCTestCase { sent(139, arg: 0, .success), ] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .noDataRequested(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124)", - "TOGGLE_LABRADOR_RAW_SAVE(125)", - "TOGGLE_LABRADOR_FILTERED(139)"])) + .noDataRequested(commands: ["TOGGLE_LABRADOR_DATA_GENERATION(124) arg=0", + "TOGGLE_LABRADOR_RAW_SAVE(125) arg=0", + "TOGGLE_LABRADOR_FILTERED(139) arg=0"])) } func testRawSaveAloneCannotUnlockTheSilentVerdict() { @@ -622,7 +882,7 @@ final class Whoop5EcgTests: XCTestCase { // so a raw-save-only run cannot be read as "accepted and then silent" (#891 hypothesis (b)). let steps = [sent(125, arg: 1, .success)] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .noDataRequested(commands: ["TOGGLE_LABRADOR_RAW_SAVE(125)"])) + .noDataRequested(commands: ["TOGGLE_LABRADOR_RAW_SAVE(125) arg=1"])) } func testAnUnacknowledgedDataRequestIsNotAcceptedButSilent() { @@ -633,8 +893,8 @@ final class Whoop5EcgTests: XCTestCase { sent(124, arg: Whoop5Ecg.ControlSignal.start.rawValue, .noReply), ] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .dataRequestNotAccepted(commands: ["TOGGLE_LABRADOR_FILTERED(139)", - "TOGGLE_LABRADOR_DATA_GENERATION(124)"])) + .dataRequestNotAccepted(commands: ["TOGGLE_LABRADOR_FILTERED(139) arg=1", + "TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1"])) } func testNoVerdictClaimsAFlagBlockWithoutADataRequest() { @@ -701,7 +961,7 @@ final class Whoop5EcgTests: XCTestCase { func testVerdictUnsupportedIsReportedAsItselfNotAsABlock() { let steps = [sent(139, arg: 1, .unsupported)] XCTAssertEqual(Whoop5EcgProbe.verdict(steps: steps, ecgPacketsSeen: 0, windowSeconds: 30), - .opcodeUnsupported(commands: ["TOGGLE_LABRADOR_FILTERED(139)"])) + .opcodeUnsupported(commands: ["TOGGLE_LABRADOR_FILTERED(139) arg=1"])) } func testVerdictSilenceIsNeverCalledABlock() { @@ -729,13 +989,49 @@ final class Whoop5EcgTests: XCTestCase { let text = Whoop5EcgProbe.report(steps: steps, ecgPacketsSeen: 0, candidateFrames: ["type=0x28 len=220"], windowSeconds: 30) XCTAssertTrue(text.contains("DATA REQUEST REFUSED")) - XCTAssertTrue(text.contains("SELECT_WRIST(123): SUCCESS(1)")) - XCTAssertTrue(text.contains("TOGGLE_LABRADOR_DATA_GENERATION(124): FAILURE(0)")) + XCTAssertTrue(text.contains("SELECT_WRIST(123) arg=1: SUCCESS(1)")) + XCTAssertTrue(text.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1: FAILURE(0)")) XCTAssertTrue(text.contains("type=0x28 len=220")) XCTAssertTrue(text.contains("aabb")) XCTAssertTrue(text.contains("not a medical measurement or a diagnosis")) } + /// A START run and a RESTART run send the SAME three opcodes and differ in exactly one byte, so + /// without the argument annotation their reports are character-for-character identical — and a + /// report is the artefact that gets copied out of the app and pasted into an issue, long after the + /// strap log that recorded `payload=0102` is gone. This pins that the two are distinguishable. + func testStartAndRestartRunsRenderDistinguishably() { + func run(_ control: Whoop5Ecg.ControlSignal) -> String { + Whoop5EcgProbe.report( + steps: [sent(139, arg: 1, .success), + sent(125, arg: 1, .success), + sent(124, arg: control.rawValue, .success)], + ecgPacketsSeen: 0, candidateFrames: [], windowSeconds: 30) + } + let startText = run(.start) + let restartText = run(.restart) + XCTAssertTrue(startText.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1: SUCCESS(1)")) + XCTAssertTrue(restartText.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=2: SUCCESS(1)")) + XCTAssertNotEqual(startText, restartText) + // Both are still the SAME verdict: restart asks for realtime data exactly like start does, so + // the annotation records what was sent without changing what the run is read as. + XCTAssertTrue(restartText.contains("Accepted but SILENT")) + XCTAssertTrue(startText.contains("Accepted but SILENT")) + } + + /// An UNSOLICITED reply has no known argument — nothing in the app sent it — and the report must say + /// nothing rather than invent a value. + func testAnUnknownArgumentIsOmittedRatherThanGuessed() { + let step = Whoop5EcgProbe.Step(label: "TOGGLE_LABRADOR_DATA_GENERATION(124)", + outcome: .success, + requestsRealtimeData: false) + XCTAssertNil(step.sentArgument) + XCTAssertEqual(step.labelWithArgument, "TOGGLE_LABRADOR_DATA_GENERATION(124)") + let text = Whoop5EcgProbe.report(steps: [step], ecgPacketsSeen: 0, + candidateFrames: [], windowSeconds: 30) + XCTAssertFalse(text.contains("arg=")) + } + /// #896 review: nobody is told to hold the clasp. An MG measures across the wrist electrode AND the /// two clasp indents, and lead state is not on the wire — so a run where the clasp was never touched /// returns zero packets for a reason that has nothing to do with the firmware. #891 asks other MG diff --git a/Packages/WhoopStore/Sources/WhoopStore/AppleStepHourStore.swift b/Packages/WhoopStore/Sources/WhoopStore/AppleStepHourStore.swift new file mode 100644 index 0000000000..13cf745a86 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/AppleStepHourStore.swift @@ -0,0 +1,42 @@ +import Foundation +import GRDB + +// MARK: - v27 store: hourly Apple Health step counts +// Mirrors OuraRawStore: an idempotent ON CONFLICT upsert keyed by the natural key (deviceId, ts), and +// a range read — all GRDB work via syncWrite/syncRead. `appleDaily.steps` already answers "how many +// steps that day"; this table answers "which HOURS were recorded", so the UI can show retroactively +// when the iPhone was off/dead/left behind (e.g. mid-hike) instead of a single flattened daily total. + +extension WhoopStore { + + /// Upsert hourly Apple Health step counts. Idempotent by (deviceId, ts): re-importing the same + /// hour overwrites its count rather than duplicating. Returns rows changed. + @discardableResult + public func upsertAppleStepHours(_ rows: [(ts: Int, steps: Int)], deviceId: String) async throws -> Int { + try syncWrite { db in + var n = 0 + for r in rows { + try db.execute(sql: """ + INSERT INTO appleStepHour (deviceId, ts, steps) + VALUES (?, ?, ?) + ON CONFLICT(deviceId, ts) DO UPDATE SET + steps = excluded.steps + """, arguments: [deviceId, r.ts, r.steps]) + n += db.changesCount + } + return n + } + } + + /// Hourly step counts for a device over `[fromTs, toTs]` inclusive, oldest hour first. + public func appleStepHours(deviceId: String, fromTs: Int, toTs: Int) async throws -> [(ts: Int, steps: Int)] { + try syncRead { db in + try Row.fetchAll(db, sql: """ + SELECT ts, steps FROM appleStepHour + WHERE deviceId = ? AND ts >= ? AND ts <= ? + ORDER BY ts ASC + """, arguments: [deviceId, fromTs, toTs]) + .map { (ts: $0["ts"], steps: $0["steps"]) } + } + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/CloudEditDeletes.swift b/Packages/WhoopStore/Sources/WhoopStore/CloudEditDeletes.swift new file mode 100644 index 0000000000..7e6e4b9d52 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/CloudEditDeletes.swift @@ -0,0 +1,36 @@ +import Foundation +import GRDB + +// MARK: - Cloud edit deletes: physical DELETEs paired with a CloudTombstoneStore.addTombstone call. +// A server-side journal delete must remove the local row(s) AND record a tombstone (else the next +// sync/backfill/live-BLE write resurrects them) — a caller applies both, in either order. Plain +// DELETEs, no upsert semantics; each returns the number of rows actually removed. + +extension WhoopStore { + + /// Physically delete HR samples for a device with `ts` in `[fromTs, toTs]` inclusive. + /// Returns rows deleted. Pair with `addTombstone(kind: "hrRange", ...)` so a later re-sync + /// (backfill replay, or the strap re-streaming an already-deleted window) can't resurrect them. + @discardableResult + public func deleteHrRange(deviceId: String, fromTs: Int, toTs: Int) async throws -> Int { + try syncWrite { db in + try db.execute(sql: """ + DELETE FROM hrSample WHERE deviceId = ? AND ts >= ? AND ts <= ? + """, arguments: [deviceId, fromTs, toTs]) + return db.changesCount + } + } + + /// Physically delete one metric point by natural key (deviceId, day, key). Returns rows deleted. + /// Pair with `addTombstone(kind: "metricPoint", ...)` so a later re-derive/re-upsert can't + /// resurrect it. + @discardableResult + public func deleteMetricPoint(deviceId: String, day: String, key: String) async throws -> Int { + try syncWrite { db in + try db.execute(sql: """ + DELETE FROM metricSeries WHERE deviceId = ? AND day = ? AND key = ? + """, arguments: [deviceId, day, key]) + return db.changesCount + } + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/CloudTombstoneStore.swift b/Packages/WhoopStore/Sources/WhoopStore/CloudTombstoneStore.swift new file mode 100644 index 0000000000..ba02476921 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/CloudTombstoneStore.swift @@ -0,0 +1,98 @@ +import Foundation +import GRDB + +// MARK: - v26 store: cloud edit tombstones (resurrection guards) +// A server-side journal delete (workout / HR range / metric point) is applied locally as a +// physical DELETE (see CloudEditDeletes) plus a tombstone row here, so a LATER re-sync/backfill/ +// live-BLE write can never silently resurrect the same data. Mirrors OuraRawStore/MetricSeriesStore: +// idempotent inserts, range/set reads, all GRDB work via the actor's syncWrite/syncRead helpers. +// +// The three `*TombstoneRows` helpers take an already-open `Database` connection rather than going +// through `syncRead`, so a guard running INSIDE another store's `syncWrite` block (upsertWorkouts, +// upsertMetricSeries, StreamStore.insert) queries on the SAME connection/transaction instead of +// opening a second one. The public `workoutTombstones`/etc. accessors below simply wrap them in +// `syncRead` for callers outside a write. + +extension WhoopStore { + + /// Workout tombstones for a device: `(startTs, sport)` pairs a resurrection guard must drop. + /// Shared by the public `workoutTombstones` read and the `upsertWorkouts` guard. + static func workoutTombstoneRows(_ db: Database, deviceId: String) throws -> [(startTs: Int, sport: String)] { + try Row.fetchAll(db, sql: """ + SELECT startTs, sport FROM cloudTombstone WHERE kind = 'workout' AND deviceId = ? + """, arguments: [deviceId]) + .map { (startTs: $0["startTs"], sport: $0["sport"]) } + } + + /// HR-range tombstones for a device: `(fromTs, toTs)` ranges a live/backfill insert must filter. + /// Shared by the public `hrTombstoneRanges` read and the `StreamStore.insert` guard. + static func hrTombstoneRangeRows(_ db: Database, deviceId: String) throws -> [(fromTs: Int, toTs: Int)] { + try Row.fetchAll(db, sql: """ + SELECT startTs, endTs FROM cloudTombstone WHERE kind = 'hrRange' AND deviceId = ? + """, arguments: [deviceId]) + .map { (fromTs: $0["startTs"], toTs: $0["endTs"]) } + } + + /// Metric-point tombstones for a device: `(day, key)` pairs a resurrection guard must drop. + /// Shared by the public `metricPointTombstones` read and the `upsertMetricSeries` guard. + static func metricPointTombstoneRows(_ db: Database, deviceId: String) throws -> [(day: String, key: String)] { + try Row.fetchAll(db, sql: """ + SELECT day, key FROM cloudTombstone WHERE kind = 'metricPoint' AND deviceId = ? + """, arguments: [deviceId]) + .map { (day: $0["day"], key: $0["key"]) } + } + + /// Record a cloud-edit tombstone. Idempotent by (kind, editSeq) — INSERT OR IGNORE, so replaying + /// the same journal entry (e.g. after a reconnect) is a no-op rather than a duplicate row. + /// `startTs`/`endTs`/`sport` apply to "workout"/"hrRange" kinds; `day`/`key` to "metricPoint" — + /// the caller passes nil for whichever fields don't apply to `kind`. + public func addTombstone(kind: String, deviceId: String, startTs: Int?, endTs: Int?, + sport: String?, day: String?, key: String?, editSeq: Int) async throws { + let now = Int(Date().timeIntervalSince1970) + try syncWrite { db in + try db.execute(sql: """ + INSERT OR IGNORE INTO cloudTombstone + (kind, deviceId, startTs, endTs, sport, day, key, editSeq, createdAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, arguments: [kind, deviceId, startTs, endTs, sport, day, key, editSeq, now]) + } + } + + /// Workout tombstones for a device (unordered). See `workoutTombstoneRows` for the guard-side use. + public func workoutTombstones(deviceId: String) async throws -> [(startTs: Int, sport: String)] { + try syncRead { db in try WhoopStore.workoutTombstoneRows(db, deviceId: deviceId) } + } + + /// HR-range tombstones for a device (unordered). See `hrTombstoneRangeRows` for the guard-side use. + public func hrTombstoneRanges(deviceId: String) async throws -> [(fromTs: Int, toTs: Int)] { + try syncRead { db in try WhoopStore.hrTombstoneRangeRows(db, deviceId: deviceId) } + } + + /// Metric-point tombstones for a device (unordered). See `metricPointTombstoneRows` for the guard-side use. + public func metricPointTombstones(deviceId: String) async throws -> [(day: String, key: String)] { + try syncRead { db in try WhoopStore.metricPointTombstoneRows(db, deviceId: deviceId) } + } + + /// Remove every cloud-edit tombstone for a device (used by a future full-disconnect flow — + /// `deleteAllData` deliberately does NOT cover `cloudTombstone`, see + /// `DeviceRegistryStore.deviceScopedTableExemptions`). Mirrors `deleteOuraRaw`. Returns rows deleted. + @discardableResult + public func deleteCloudTombstones(deviceId: String) async throws -> Int { + try syncWrite { db in + try db.execute(sql: "DELETE FROM cloudTombstone WHERE deviceId = ?", arguments: [deviceId]) + return db.changesCount + } + } +} + +/// Hashable natural key for the `upsertWorkouts` resurrection guard's pre-loaded tombstone Set. +struct WorkoutTombstoneKey: Hashable { + let startTs: Int + let sport: String +} + +/// Hashable natural key for the `upsertMetricSeries` resurrection guard's pre-loaded tombstone Set. +struct MetricPointTombstoneKey: Hashable { + let day: String + let key: String +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/ContentToken.swift b/Packages/WhoopStore/Sources/WhoopStore/ContentToken.swift new file mode 100644 index 0000000000..8cc0f82a30 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/ContentToken.swift @@ -0,0 +1,48 @@ +import Foundation +import GRDB + +extension WhoopStore { + /// Cheap composite fingerprint of the store's DATA tables — deliberately excluding `cursors` (see + /// below) — that changes if and only if something an upload would actually ship has changed. + /// `CloudSyncModel.performSync` (Cloud Sync v2) compares this against the token it saved after the + /// last successful upload FROM THIS DEVICE (`UserDefaults["cloudsync.lastUploadToken"]`) to skip a + /// redundant re-export + POST of an unchanged 100-300MB `.noopbak`. + /// + /// Format: `"hr::|sleep::|daily::|workout:|series:"` + /// — opaque, comparable only to itself, never parsed back apart. The `hr`/`sleep`/`daily` segments + /// mirror `hrFingerprint`'s (count, max) change-detector shape (Reads.swift): a SQL `COUNT(*)` plus a + /// `COALESCE(MAX(…), default)` aggregate over one table, computed entirely in SQLite with no row + /// materialized into Swift. `workout` and `metricSeries` use `COUNT` alone: `workout` rows are + /// rewritten IN PLACE by `fix_workout`/`delete_workout` (no column is a safe append-only max to key + /// on) and `metricSeries.day` is written far out of order by design — for both, a changed COUNT + /// already catches every insert/delete/tombstone, and an in-place correction that leaves a table's + /// own row count unchanged (e.g. `fix_workout`'s patch) still moves the `hrSample`/`dailyMetric` + /// segments in the same edit, so the composite token still changes. + /// + /// Deliberately EXCLUDES `cursors`: that table is sync BOOKKEEPING (`cloud_edits`, + /// `cloud_edits_recomputed`, `stagelock:…`, the `highwater:`/`read:` cursors) rather than user data. + /// A cursor write must never dirty the token — a pull that applies zero edits can still advance + /// `cloud_edits`, and an unrelated stream's highwater bump happens on nearly every sync; if either + /// counted, the token would read "changed" on almost every sync regardless of whether any actual + /// data moved, defeating the whole point of skipping the upload. (The saved comparison value itself, + /// `cloudsync.lastUploadToken`, lives in `UserDefaults` on `CloudSyncModel`, not in this store at + /// all — so writing it can never feed back into this computation either.) + public func contentToken() async throws -> String { + try syncRead { db in + let hrCount = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM hrSample") ?? 0 + let hrMaxTs = try Int.fetchOne(db, sql: "SELECT COALESCE(MAX(ts), 0) FROM hrSample") ?? 0 + + let sleepCount = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM sleepSession") ?? 0 + let sleepMaxEndTs = try Int.fetchOne(db, sql: "SELECT COALESCE(MAX(endTs), 0) FROM sleepSession") ?? 0 + + let dailyCount = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM dailyMetric") ?? 0 + let dailyMaxDay = try String.fetchOne(db, sql: "SELECT COALESCE(MAX(day), '') FROM dailyMetric") ?? "" + + let workoutCount = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM workout") ?? 0 + let seriesCount = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM metricSeries") ?? 0 + + return "hr:\(hrCount):\(hrMaxTs)|sleep:\(sleepCount):\(sleepMaxEndTs)|daily:\(dailyCount):\(dailyMaxDay)" + + "|workout:\(workoutCount)|series:\(seriesCount)" + } + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/Database.swift b/Packages/WhoopStore/Sources/WhoopStore/Database.swift index 9b30f625c7..e48dbf2c5e 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Database.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Database.swift @@ -501,6 +501,50 @@ extension WhoopStore { on: "ouraRaw", columns: ["deviceId", "endpoint", "day"]) } + // v26: cloud edit tombstones (Phase 3 cloud journal apply). When a server-side journal edit + // deletes a workout / HR range / metric point, NOOP physically deletes the local row(s) AND + // records a tombstone here — otherwise the NEXT sync/backfill/live-BLE write would silently + // resurrect the same row from the strap or a re-pulled cloud page. `editSeq` is the cloud + // journal's own sequence number; the (kind, editSeq) unique key makes replaying the same + // journal entry idempotent (INSERT OR IGNORE, never a duplicate tombstone). Additive only — + // a NEW table, no existing row touched. + migrator.registerMigration("v26-cloud-tombstone") { db in + try db.create(table: "cloudTombstone") { t in + t.autoIncrementedPrimaryKey("id") + t.column("kind", .text).notNull() // "workout" | "hrRange" | "metricPoint" + t.column("deviceId", .text).notNull() + t.column("startTs", .integer) // workout: startTs; hrRange: fromTs + t.column("endTs", .integer) // hrRange: toTs + t.column("sport", .text) // workout + t.column("day", .text) // metricPoint + t.column("key", .text) // metricPoint + t.column("editSeq", .integer).notNull() // server journal seq (idempotency) + t.column("createdAt", .integer).notNull() + t.uniqueKey(["kind", "editSeq"]) + } + try db.create(index: "idx_cloudTombstone_kind_device", on: "cloudTombstone", columns: ["kind", "deviceId"]) + } + + // v27: hourly Apple Health step counts. The daily `collect(.stepCount, …)` path in + // HealthKitBridge flattens a whole day to one `appleDaily.steps` total, so a dead/absent phone + // for part of a day (e.g. the phone died mid-hike) is invisible — steps just read low for the + // WHOLE day instead of showing exactly which hours had no recording. HealthKit retains hourly + // statistics historically, so this table lets a one-time backfill answer PAST days + // retroactively, not just from the day this migration ships. `ts` is the hour-BUCKET START + // (unix seconds), aligned to local-clock hour boundaries by the `HKStatisticsCollectionQuery` + // anchored at local midnight (see HealthKitBridge); `steps` is the cumulative sum within that + // hour. PK (deviceId, ts) mirrors every other per-sample table (hrSample, stepSample, …) and + // makes the hourly upsert idempotent. Additive only — a NEW table, no existing row touched, old + // readers unaffected. + migrator.registerMigration("v27-apple-step-hour") { db in + try db.create(table: "appleStepHour") { t in + t.column("deviceId", .text).notNull() // "apple-health" + t.column("ts", .integer).notNull() // hour-start unix seconds (local-hour aligned by HK) + t.column("steps", .integer).notNull() + t.primaryKey(["deviceId", "ts"]) + } + } + // v26 (Oura efficiency unit heal): the Oura API importer (OuraApiParser.swift) wrote Oura's // native 0-100 integer `efficiency` straight into sleepSession.efficiency / dailyMetric.efficiency, // but NOOP's own sleep pipeline (StrandAnalytics) stores that shared column as a 0-1 FRACTION @@ -567,6 +611,41 @@ extension WhoopStore { t.primaryKey(["deviceId", "ts"]) } } + // NOTE ON NUMBERING (upstream sync 2026-07-27): the two fork-local migrations below keep their + // ORIGINAL `v28-`/`v29-` identifiers even though upstream has since claimed those same ordinals + // (`v28-raw-imu`, `v29-score-input-provenance`). GRDB keys `registerMigration` by the identifier + // STRING, not by an ordinal, so the duplicate numbers are cosmetic and all six apply exactly once. + // Renumbering them past upstream's — the usual rule when sending a migration UPSTREAM, where no + // device has applied it yet — would be actively unsafe HERE: a renamed identifier reads as + // un-applied on a fork device that already ran it, and `CREATE TABLE phoneTimezone` would then + // throw "table already exists" and wedge the migrator on a live database. The same reasoning + // already governs `v26-cloud-tombstone`/`v27-apple-step-hour`, which collide with upstream's + // `v26-efficiency-heal`/`v27-ppg-waveform` and have shipped that way since the previous sync. + // v28: the phone's IANA timezone, recorded one row per LOCAL day. Every timestamp in the DB is + // epoch-UTC and nothing records which zone the phone was in when a day was lived, so any + // wall-clock interpretation of a historical night (bedtime, "3am", day boundaries) is guesswork + // once the traveller crosses zones (e.g. Pacific ↔ Eastern). This table stamps each local day + // with `TimeZone.current.identifier` so downstream analysis can localise that specific day's + // samples instead of assuming one fixed offset. `day` is the local calendar day ("YYYY-MM-DD") + // and PK, so a same-day re-record overwrites in place (idempotent upsert). Additive only — a NEW + // table, no existing row touched, old readers unaffected. The value also rides every upload as + // the `X-Phone-Timezone` header on POST /ingest so the server knows the CURRENT zone even before + // this table lands in a given mirror. Android Room twin deferred (Swift-only contributor); a Room + // migration adding `phoneTimezone` is needed before an Android release. + migrator.registerMigration("v28-phone-timezone") { db in + try db.create(table: "phoneTimezone") { t in + t.column("day", .text).primaryKey() // local calendar day "YYYY-MM-DD" + t.column("tzId", .text).notNull() // IANA identifier, e.g. "America/Los_Angeles" + } + } + // v29: persist nightly SDNN — the 5-min SDNN index (broad-variability twin of avgHrv=RMSSD), + // window-matched to a watch's short SDNN rather than the drift-inflated whole-night SD. Additive, + // nullable; computed by HRVAnalyzer.sdnnIndex during the nightly analysis. + migrator.registerMigration("v29-daily-avg-sdnn") { db in + try db.alter(table: "dailyMetric") { t in + t.add(column: "avgSdnn", .double) + } + } // #423: WHOOP 5/MG raw-IMU offload capture (100 Hz 6-axis). `samples` is a packed i16 LE BLOB of // the six wire columns (ax…az,gx…gz). Twin of the Android `rawImuSample` table (MIGRATION_20_21); @@ -710,6 +789,55 @@ extension WhoopStore { t.primaryKey(["deviceId", "ts"]) } } + + // v34: DURABLE per-sample SpO2 percentages from the 5/MG `@82` candidate byte. + // + // THE PROBLEM THIS FIXES. The strap emits its own computed SpO2 % on `@82` of the v18 record. That + // byte is already banked, raw, in `v18AuxSample.fields` (v31) — but `v18AuxSample` is CAPPED at + // `WhoopStore.v18AuxRetentionRows` (604,800 rows/device ≈ 7 days at 1 Hz), because the aux blob + // costs ~30 MB/day and an uncapped 30 MB/day table is not a thing a phone can carry. So every SpO2 + // reading rolls off after a week and is gone permanently: the strap trims its own history the + // moment an offload is acked, so there is no second copy anywhere. + // + // Raising the aux cap is the WRONG fix — it would buy a year of SpO2 at the price of ~11 GB of + // mostly-unrelated aux bytes. The right fix is a narrow sibling: the in-band SpO2 signal is only + // ~2,000 samples/week (a run of ~30 one-second samples once per ~1,200 s, sleep-gated), i.e. + // ~100k rows/year, which is trivially retainable forever. This table is that sibling. The aux + // capture stays capped and unchanged; this one is never pruned. + // + // WHY NOT EXTEND `spo2Sample` (v3). Wrong shape, and extending it would corrupt a live reader: + // 1. `spo2Sample` is `(deviceId, ts, red, ir)` — two NOT NULL raw ADC channels off the WHOOP 4.0 + // v24 layout. We have a computed PERCENTAGE and no red/IR at all, so every insert would have + // to fabricate `red`/`ir` values that were never measured. Absence-becomes-a-fabricated-0 is + // exactly the failure mode the rest of this schema is built to avoid. + // 2. `AnalyticsEngine.nightlySpo2RawMeans` averages `red`/`ir` over that table into + // `dailyMetric.spo2Red`/`spo2Ir`. Inserting rows carrying fake zeros would silently drag + // those means toward zero — a live, user-visible tile broken by a storage decision. + // 3. The two are different physical quantities from different device generations. A percentage + // and a pair of ADC counts sharing a table would need a discriminator column and every reader + // would have to remember to apply it. A separate table makes the distinction unforgeable. + // + // `pct` is the RAW in-band byte (70...100), stored verbatim. Run grouping and acquisition-ramp + // trimming are deliberately NOT applied here — they are read-time policies, they have already + // changed once on the cloud reader, and a store that baked in the old policy would have destroyed + // the evidence needed to change them again. The one thing applied on the way in is the + // `spo2CandidateInBand` demultiplex (see `Spo2PctSample`), which is not a policy: `@82` carries + // measurements, bit-7 status sentinels, sub-70 diagnostic codes and 0 on the same byte, and only + // the `70...100` band is a percentage of anything. Storing a sentinel here would be storing a + // number that is not the quantity the column is named for. + // + // NOT PRUNED, by design — the same "durable decoded biometric history" class as `hrSample` and + // `ppgWaveformSample`. The `v18AuxSample` sweep in `StreamStore.insert` deletes only + // `FROM v18AuxSample`, so this table is untouched by it; `Spo2PctDurableTests` pins that. + // PK (deviceId, ts) mirrors every other per-sample table and makes re-ingest idempotent. + migrator.registerMigration("v34-spo2-pct-durable") { db in + try db.create(table: "spo2PctSample") { t in + t.column("deviceId", .text).notNull() + t.column("ts", .integer).notNull() // strap-second, unix seconds + t.column("pct", .integer).notNull() // raw in-band @82 byte, 70...100 + t.primaryKey(["deviceId", "ts"]) + } + } return migrator } } diff --git a/Packages/WhoopStore/Sources/WhoopStore/DatabaseIntegrity.swift b/Packages/WhoopStore/Sources/WhoopStore/DatabaseIntegrity.swift index 63b5ea9a1c..4139a87023 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/DatabaseIntegrity.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/DatabaseIntegrity.swift @@ -35,6 +35,22 @@ public enum DatabaseIntegrity { func quickCheckRows(readonly: Bool) throws -> [String] { var config = Configuration() config.readonly = readonly + if !readonly { + // The read-write fallback is the ONE connection in this tree that can reach the live + // store without going through `WhoopStore(path:)`, and therefore without + // `StoreReplication`'s checkpointing policy (see `WalCheckpointing`). Today it is + // harmless only by a chain of reasoning about SQLite internals — autocheckpoint fires + // at the end of a WRITE transaction and this probe only reads, and the close-time + // checkpoint only runs for the LAST connection, which this never is while the app's + // pool is open. Under `.external` that chain is the whole safety argument for a + // replicator's resume offset, so it is replaced with the guarantee: this connection + // cannot restart the WAL because it is not permitted to checkpoint. + // + // No behaviour change for the common case (probing a staged backup file): the + // close-time checkpoint that cleans up the `-wal` this connection creates is a + // separate mechanism and is unaffected by `wal_autocheckpoint`. + config.prepareDatabase { try $0.execute(sql: "PRAGMA wal_autocheckpoint = 0") } + } let dbQueue = try DatabaseQueue(path: path, configuration: config) return try dbQueue.read { db in try String.fetchAll(db, sql: "PRAGMA quick_check(1)") } } diff --git a/Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift b/Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift index 3606072933..ab021aceef 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift @@ -99,6 +99,10 @@ public struct DeviceRegistryStore: Sendable { // v25-oura-raw: the opt-in Oura cloud-import raw archive is deviceId-keyed too, so "delete this // device's data" must clear it — else an imported Oura source's payloads would survive deletion. "ouraRaw", + // v27-apple-step-hour: per-hour Apple Health step counts are deviceId-keyed (apple-health) and + // are real per-device recordings like appleDaily/stepSample, not tombstone-class bookkeeping — + // a device-data wipe / "Remove Apple Health data" must clear them too. + "appleStepHour", // v27-ppg-waveform (issue #156 follow-up): the durable raw v26 optical PPG waveform is // deviceId-keyed exactly like every other per-second stream above — must be cleared too, or a // "delete all of this device's data" leaves the raw waveform behind (the same privacy defect @@ -110,13 +114,34 @@ public struct DeviceRegistryStore: Sendable { // v31-deep-capture-channels: the banked 5/MG v18 auxiliary fields are deviceId-keyed per-second // rows like every stream above, so a "delete all of this device's data" must clear them too. "v18AuxSample", + // v34-spo2-pct-durable: the durable `@82` SpO2 percentages are deviceId-keyed per-second rows, + // and they OUTLIVE the aux table they are forked from (never pruned) — which makes covering them + // here more important, not less. A wipe that cleared `v18AuxSample` but left this behind would + // leave years of blood-oxygen readings on the device after the user asked for them to be gone. + "spo2PctSample", ] + /// deviceId-keyed tables deliberately EXCLUDED from `deviceScopedTables` — a normal "delete this + /// device's data" wipe must NOT touch them, even though they carry a `deviceId` column. + /// + /// - `cloudTombstone` (v26-cloud-tombstone) is the guard's memory of cloud-journal deletes (a + /// workout / HR range / metric point the user removed on the cloud side). If a device-data wipe + /// cleared it, the very next BLE backfill or cloud re-sync could silently resurrect data the user + /// already deleted — tombstones must outlive per-device data wipes. A dedicated + /// `CloudTombstoneStore.deleteCloudTombstones(deviceId:)` exists for sites that DO mean to forget + /// a device's cloud-edit history (a full disconnect), never `deleteAllData`. + /// + /// `DeviceRegistryStoreTests.testDeviceScopedTablesCoversEveryDeviceIdKeyedTable` treats this list + /// as the only legitimate reason a deviceId-keyed table can be absent from `deviceScopedTables`; + /// any other uncovered table still fails that test. + static let deviceScopedTableExemptions = ["cloudTombstone"] + /// Permanently delete every recorded sample/derived row belonging to one device, across all /// `deviceId`-keyed tables, in a single transaction (all-or-nothing). The `pairedDevice` registry /// row is left intact — the caller archives/removes that separately. Tables are deleted defensively /// with `DELETE FROM WHERE deviceId = ?`; a missing table would throw, but every table here - /// is created unconditionally by the migrator, so the set is stable. + /// is created unconditionally by the migrator, so the set is stable. `cloudTombstone` is + /// deliberately NOT among these tables — see `deviceScopedTableExemptions`. public func deleteAllData(deviceId: String) throws { try dbQueue.write { db in for table in Self.deviceScopedTables { diff --git a/Packages/WhoopStore/Sources/WhoopStore/JournalWorkoutAppleCache.swift b/Packages/WhoopStore/Sources/WhoopStore/JournalWorkoutAppleCache.swift index 217716b229..ffae03a3e4 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/JournalWorkoutAppleCache.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/JournalWorkoutAppleCache.swift @@ -138,11 +138,18 @@ extension WhoopStore { } /// Upsert workouts. Natural key (deviceId, startTs, sport). Returns rows changed. + /// + /// Resurrection guard: a cloud-journal delete tombstones a workout's (startTs, sport) — see + /// `CloudTombstoneStore`. A row matching a tombstone is dropped BEFORE the insert, so a later + /// re-import (backfill replay, cloud re-pull) can't resurrect a workout the user deleted. @discardableResult public func upsertWorkouts(_ rows: [WorkoutRow], deviceId: String) async throws -> Int { try syncWrite { db in + let tombstoned = Set(try WhoopStore.workoutTombstoneRows(db, deviceId: deviceId) + .map { WorkoutTombstoneKey(startTs: $0.startTs, sport: $0.sport) }) var n = 0 for r in rows { + if tombstoned.contains(WorkoutTombstoneKey(startTs: r.startTs, sport: r.sport)) { continue } try db.execute(sql: """ INSERT INTO workout (deviceId, startTs, endTs, sport, source, durationS, energyKcal, @@ -249,6 +256,17 @@ extension WhoopStore { } } + /// True when a workout exists at the exact natural key (deviceId, startTs, sport). A cheap + /// existence check for a resurrection/staleness guard (`CloudEditApplier`'s `fix_workout` + /// handler) that only needs a boolean, not a fully decoded `WorkoutRow`. + public func workoutExists(deviceId: String, startTs: Int, sport: String) async throws -> Bool { + try syncRead { db in + try Row.fetchOne(db, sql: """ + SELECT 1 FROM workout WHERE deviceId = ? AND startTs = ? AND sport = ? LIMIT 1 + """, arguments: [deviceId, startTs, sport]) != nil + } + } + /// Apple-Health daily aggregates for days in [from, to] (lexicographic compare), oldest first. public func appleDaily(deviceId: String, from: String, to: String) async throws -> [AppleDaily] { try syncRead { db in diff --git a/Packages/WhoopStore/Sources/WhoopStore/LabMarkerStore.swift b/Packages/WhoopStore/Sources/WhoopStore/LabMarkerStore.swift index ef6e26f18e..4789d07dbe 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/LabMarkerStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/LabMarkerStore.swift @@ -214,8 +214,18 @@ extension WhoopStore { /// from the CURRENT `labMarker` rows. Latest-numeric-per-day wins; if no numeric /// reading remains for a cell, its projected day is deleted (so a removed/last /// reading never leaves a stale projected value behind). + /// + /// Resurrection guard: a cloud-journal delete tombstones a metric point's (day, key) — see + /// `CloudTombstoneStore`. Every cell this projects lands in `metricSeries` under the constant + /// `labBookSourceId` (not the marker's own `deviceId`), so tombstones are loaded for THAT id — a + /// tombstoned cell is skipped entirely, so a later marker upsert/delete can't resurrect a point + /// the user deleted on the cloud side. private func reprojectCells(_ db: Database, cells: Set) throws { + let tombstoned = Set(try WhoopStore.metricPointTombstoneRows(db, deviceId: WhoopStore.labBookSourceId) + .map { MetricPointTombstoneKey(day: $0.day, key: $0.key) }) for cell in cells { + if tombstoned.contains(MetricPointTombstoneKey(day: cell.day, key: cell.markerKey)) { continue } + // Latest NUMERIC reading for this (markerKey, day): greatest takenAt with a // non-null value. Matches LabBookProjection.project(.latest) on numeric rows. let latest = try Double.fetchOne(db, sql: """ diff --git a/Packages/WhoopStore/Sources/WhoopStore/MetricSeriesStore.swift b/Packages/WhoopStore/Sources/WhoopStore/MetricSeriesStore.swift index 2b1e5152e5..c489034047 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/MetricSeriesStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/MetricSeriesStore.swift @@ -27,6 +27,10 @@ extension WhoopStore { /// Upsert metric points. Natural key (deviceId, day, key). Returns rows changed. /// Idempotent: re-upserting the same (deviceId, day, key) updates `value` in place rather than /// creating a duplicate. + /// + /// Resurrection guard: a cloud-journal delete tombstones a metric point's (day, key) — see + /// `CloudTombstoneStore`. A row matching a tombstone is dropped BEFORE the insert, so a later + /// re-derive/re-upsert can't resurrect a point the user deleted. @discardableResult public func upsertMetricSeries(_ rows: [MetricPoint], deviceId: String) async throws -> Int { try syncWrite { db in @@ -35,9 +39,18 @@ extension WhoopStore { } /// Transaction-sharing primitive used by computed-score persistence. + /// + /// The resurrection guard lives HERE, not in the async wrapper, so the two writers cannot drift: + /// `ScoreInputProvenanceStore` persists computed headline scores through this same primitive, and a + /// guard that only covered the wrapper would let a score the user deleted in the cloud journal come + /// back the next time it was re-derived. `metricPointTombstoneRows` reads on the caller's `db`, so it + /// shares the enclosing transaction rather than opening a second connection. static func upsertMetricSeries(_ rows: [MetricPoint], deviceId: String, in db: Database) throws -> Int { + let tombstoned = Set(try WhoopStore.metricPointTombstoneRows(db, deviceId: deviceId) + .map { MetricPointTombstoneKey(day: $0.day, key: $0.key) }) var n = 0 for r in rows { + if tombstoned.contains(MetricPointTombstoneKey(day: r.day, key: r.key)) { continue } try db.execute(sql: """ INSERT INTO metricSeries (deviceId, day, key, value) diff --git a/Packages/WhoopStore/Sources/WhoopStore/MetricsCache.swift b/Packages/WhoopStore/Sources/WhoopStore/MetricsCache.swift index 1c0fc3098b..525fc4f4ef 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/MetricsCache.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/MetricsCache.swift @@ -68,19 +68,27 @@ public struct DailyMetric: Equatable, Codable { // call site is unaffected. public let spo2Red: Int? // mean raw red PPG ADC during detected sleep public let spo2Ir: Int? // mean raw IR PPG ADC during detected sleep + /// Nightly SDNN (ms — the Task Force 5-min SDNN INDEX: mean of per-5-min-segment SDNN, ddof=1). The + /// BROAD autonomic-variability twin of `avgHrv` (which is RMSSD for the strap: the fast, vagal, + /// "recovered today?" metric). The 5-min index — not a single whole-night SD — is stored deliberately: + /// whole-night SD is dominated by the slow HR drift across sleep stages (reads 2-3× high) and is not + /// comparable to a watch's short-window SDNN; the index is. v29 column, nullable: WHOOP/on-device nights + /// compute it from the night's R-R (`HRVAnalyzer.sdnnIndex`); Apple rows mirror their own SDNN reading; + /// Oura/other imports carry no SDNN so it stays nil. + public let avgSdnn: Double? public init(day: String, totalSleepMin: Double?, efficiency: Double?, deepMin: Double?, remMin: Double?, lightMin: Double?, disturbances: Int?, restingHr: Int?, avgHrv: Double?, recovery: Double?, strain: Double?, exerciseCount: Int?, spo2Pct: Double? = nil, skinTempDevC: Double? = nil, respRateBpm: Double? = nil, steps: Int? = nil, activeKcalEst: Double? = nil, - spo2Red: Int? = nil, spo2Ir: Int? = nil) { + spo2Red: Int? = nil, spo2Ir: Int? = nil, avgSdnn: Double? = nil) { self.day = day; self.totalSleepMin = totalSleepMin; self.efficiency = efficiency self.deepMin = deepMin; self.remMin = remMin; self.lightMin = lightMin self.disturbances = disturbances; self.restingHr = restingHr; self.avgHrv = avgHrv self.recovery = recovery; self.strain = strain; self.exerciseCount = exerciseCount self.spo2Pct = spo2Pct; self.skinTempDevC = skinTempDevC; self.respRateBpm = respRateBpm self.steps = steps; self.activeKcalEst = activeKcalEst - self.spo2Red = spo2Red; self.spo2Ir = spo2Ir + self.spo2Red = spo2Red; self.spo2Ir = spo2Ir; self.avgSdnn = avgSdnn } /// The freshest STRICTLY-PRIOR day that carries at least one overnight vital (HRV / resting HR / @@ -388,8 +396,8 @@ extension WhoopStore { (deviceId, day, totalSleepMin, efficiency, deepMin, remMin, lightMin, disturbances, restingHr, avgHrv, recovery, strain, exerciseCount, spo2Pct, skinTempDevC, respRateBpm, steps, activeKcalEst, - spo2Red, spo2Ir) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + spo2Red, spo2Ir, avgSdnn) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(deviceId, day) DO UPDATE SET totalSleepMin = excluded.totalSleepMin, efficiency = excluded.efficiency, @@ -408,13 +416,14 @@ extension WhoopStore { steps = excluded.steps, activeKcalEst = excluded.activeKcalEst, spo2Red = excluded.spo2Red, - spo2Ir = excluded.spo2Ir + spo2Ir = excluded.spo2Ir, + avgSdnn = excluded.avgSdnn """, arguments: [deviceId, d.day, d.totalSleepMin, d.efficiency, d.deepMin, d.remMin, d.lightMin, d.disturbances, d.restingHr, d.avgHrv, d.recovery, d.strain, d.exerciseCount, d.spo2Pct, d.skinTempDevC, d.respRateBpm, d.steps, d.activeKcalEst, - d.spo2Red, d.spo2Ir]) + d.spo2Red, d.spo2Ir, d.avgSdnn]) n += db.changesCount } return n @@ -464,7 +473,7 @@ extension WhoopStore { SELECT day, totalSleepMin, efficiency, deepMin, remMin, lightMin, disturbances, restingHr, avgHrv, recovery, strain, exerciseCount, spo2Pct, skinTempDevC, respRateBpm, steps, activeKcalEst, - spo2Red, spo2Ir FROM dailyMetric + spo2Red, spo2Ir, avgSdnn FROM dailyMetric WHERE deviceId = ? AND day >= ? AND day <= ? ORDER BY day ASC """, arguments: [deviceId, from, to]) @@ -478,8 +487,24 @@ extension WhoopStore { spo2Pct: $0["spo2Pct"], skinTempDevC: $0["skinTempDevC"], respRateBpm: $0["respRateBpm"], steps: $0["steps"], activeKcalEst: $0["activeKcalEst"], - spo2Red: $0["spo2Red"], spo2Ir: $0["spo2Ir"]) + spo2Red: $0["spo2Red"], spo2Ir: $0["spo2Ir"], avgSdnn: $0["avgSdnn"]) } } } + + /// Cheap existence check across EVERY device/source (no `deviceId` filter, unlike the reads + /// above): does the store have any cached daily-metric row at all? Every ingest path that puts + /// real data into this store — `WhoopImporter`, `OuraSyncWriter`, `AppleHealthImport`, + /// `XiaomiImporter`, `ShortcutHealthImport`, `HealthKitBridge`, and the on-device + /// `IntelligenceEngine` recompute that runs after a live BLE offload — upserts into `dailyMetric` + /// as part of that ingest. So a genuinely fresh/never-populated store (no BLE pairing, no import + /// ever run — the shape of the empty macOS test-host database that got auto-uploaded and replaced + /// the production mirror) has zero rows here. Used by `CloudSyncUploader.upload` to refuse + /// exporting such a store. `SELECT 1 … LIMIT 1` rather than `COUNT(*)`: this only needs existence, + /// so SQLite can stop at the first matching row instead of scanning/counting the whole table. + public func hasAnyDailyMetrics() async throws -> Bool { + try syncRead { db in + try Int.fetchOne(db, sql: "SELECT 1 FROM dailyMetric LIMIT 1") != nil + } + } } diff --git a/Packages/WhoopStore/Sources/WhoopStore/PhoneTimezoneStore.swift b/Packages/WhoopStore/Sources/WhoopStore/PhoneTimezoneStore.swift new file mode 100644 index 0000000000..d314319d7a --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/PhoneTimezoneStore.swift @@ -0,0 +1,37 @@ +import Foundation +import GRDB + +// MARK: - v28 store: the phone's IANA timezone, one row per local day +// Every timestamp in the DB is epoch-UTC and nothing else records which zone the phone was in when a +// day was lived, so a wall-clock reading of a historical night is guesswork once the user crosses time +// zones. This table stamps each local calendar day with `TimeZone.current.identifier` so downstream +// analysis can localise that specific day. Idempotent upsert keyed by `day`: recording the same day +// again overwrites the identifier in place rather than duplicating. + +extension WhoopStore { + + /// Upsert the phone's IANA timezone identifier for a local calendar day. Idempotent by `day`: + /// re-recording the same day overwrites its identifier rather than inserting a duplicate. Returns + /// rows changed. + @discardableResult + public func upsertPhoneTimezone(day: String, tzId: String) async throws -> Int { + try syncWrite { db in + try db.execute(sql: """ + INSERT INTO phoneTimezone (day, tzId) + VALUES (?, ?) + ON CONFLICT(day) DO UPDATE SET + tzId = excluded.tzId + """, arguments: [day, tzId]) + return db.changesCount + } + } + + /// The phone's recorded IANA timezone identifier for a local calendar day, or nil if none stored. + public func phoneTimezone(day: String) async throws -> String? { + try syncRead { db in + try String.fetchOne(db, sql: """ + SELECT tzId FROM phoneTimezone WHERE day = ? + """, arguments: [day]) + } + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/StoreReplication.swift b/Packages/WhoopStore/Sources/WhoopStore/StoreReplication.swift new file mode 100644 index 0000000000..acb9a2fe6a --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/StoreReplication.swift @@ -0,0 +1,107 @@ +import Foundation + +/// Process-wide default for the WAL-checkpointing parameters of `WhoopStore(path:)`. +/// +/// ## The hazard this exists to remove +/// +/// `WalCheckpointing.external` is a **per-connection** `PRAGMA wal_autocheckpoint = 0`, so it only +/// achieves anything if *every* connection to the file has it. This app opens the same database from +/// two independent places, each with its own `DatabasePool` — `Repository.ensureStore()` and +/// `BLEManager.bootstrapStore()` (see `StoreOpenGate`) — and both open through the same +/// `WhoopStore(path:)` initialiser. +/// +/// Passing `.external` as an argument at those two call sites would work, and would keep working +/// exactly until someone adds a third opener. The failure that follows is silent in the worst way: +/// the new pool auto-checkpoints at ~4 MB, restarts the WAL underneath the replicator, the +/// replicator finds its resume offset overwritten, and every push degrades to a full snapshot of the +/// whole database. Nothing throws. Nothing logs. The only symptom is that the upload never gets +/// smaller — i.e. the exact thing the replicator was adopted to fix, failing in a way that looks +/// like the replicator simply not being very good. +/// +/// So the mode is not an argument that call sites remember to pass. It is the **default value** of +/// `WhoopStore(path:)`'s parameters, resolved here at each call. Every opener that exists, and every +/// opener anyone writes later, is correct without knowing this type exists. Explicitly passing +/// `walCheckpointing:` still overrides it, which is what the tests do. +/// +/// ## Ordering +/// +/// `configure(...)` must run before the first store opens; afterwards it applies only to stores +/// opened later, which is a real (if narrow) way to get a half-configured process. That case is not +/// silent: it sets `configuredAfterFirstOpen`, which a diagnostics screen can read and a test can +/// assert on, and it logs. On iOS the correct call site is the top of `StrandiOSApp.init()`, ahead +/// of `AppModel()` — every store open in this app is `async` and therefore strictly later. +/// +/// ## Default +/// +/// `.automatic` + `.standard`, i.e. byte-for-byte the behaviour of a build that never calls +/// `configure`. Upstream links this file and gets SQLite's own autocheckpoint, unchanged. +public enum StoreReplication { + private static let lock = NSLock() + nonisolated(unsafe) private static var _walCheckpointing: WalCheckpointing = .automatic + nonisolated(unsafe) private static var _walBackstop: WalBackstopPolicy = .standard + nonisolated(unsafe) private static var _openCount = 0 + nonisolated(unsafe) private static var _configuredAfterFirstOpen = false + + /// The checkpointing mode a `WhoopStore(path:)` gets when the caller does not pass one. + public static var walCheckpointing: WalCheckpointing { + lock.lock(); defer { lock.unlock() } + return _walCheckpointing + } + + /// The WAL ceiling policy a `WhoopStore(path:)` gets when the caller does not pass one. Inert + /// under `.automatic`; under `.external` it is what bounds WAL growth if the replicator stops. + public static var walBackstop: WalBackstopPolicy { + lock.lock(); defer { lock.unlock() } + return _walBackstop + } + + /// How many stores have been opened through `WhoopStore(path:)` in this process. Only interesting + /// relative to `configure` — see `configuredAfterFirstOpen`. + public static var openedStoreCount: Int { + lock.lock(); defer { lock.unlock() } + return _openCount + } + + /// True if `configure` ran after a store had already opened, meaning at least one live pool is + /// still using the previous mode. Under `.external` that pool will keep auto-checkpointing and + /// will restart the WAL under the replicator. Surface it; do not let it be inferred from an + /// unexplained snapshot rate. + public static var configuredAfterFirstOpen: Bool { + lock.lock(); defer { lock.unlock() } + return _configuredAfterFirstOpen + } + + /// Set the process-wide default. Call once, before any store opens. + /// + /// Choosing `.external` is an assertion that something outside this package checkpoints the WAL. + /// `walBackstop` is what keeps that assertion from being load-bearing — see `WalCheckpointing`. + public static func configure(walCheckpointing: WalCheckpointing, + walBackstop: WalBackstopPolicy = .standard) { + lock.lock() + _walCheckpointing = walCheckpointing + _walBackstop = walBackstop + let late = _openCount > 0 + if late { _configuredAfterFirstOpen = true } + lock.unlock() + if late { + NSLog("StoreReplication: configure() ran AFTER %d store(s) were already open — those " + + "pools keep their previous checkpointing mode. Under .external they will restart " + + "the WAL under the replicator.", openedStoreCount) + } + } + + /// Restore the shipped defaults. Tests only — production configures once and never reverts. + public static func resetForTesting() { + lock.lock(); defer { lock.unlock() } + _walCheckpointing = .automatic + _walBackstop = .standard + _openCount = 0 + _configuredAfterFirstOpen = false + } + + /// Called by `WhoopStore(path:)` on every successful open, so a late `configure` can be detected. + static func noteStoreOpened() { + lock.lock(); defer { lock.unlock() } + _openCount += 1 + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift b/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift index 5d431a1b36..5d550f0fa0 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift @@ -158,12 +158,27 @@ extension WhoopStore { // over potentially millions of historical rows). cachedStatement persists the compiled // statement on the connection across insert() calls too. Each loop is guarded so empty // streams (the common live case) compile nothing. - if !streams.hr.isEmpty { + // + // Resurrection guard: a cloud-journal delete tombstones an HR range — see + // `CloudTombstoneStore`. Samples inside a tombstoned range are dropped BEFORE the insert, + // so a re-sync (backfill replay, or the strap re-streaming an already-deleted window) + // can't resurrect them. This runs on the LIVE BLE path, so it's one SELECT per call + // (not per row), and only when there's HR data to filter. + var hrRows = streams.hr + if !hrRows.isEmpty { + let tombstones = try WhoopStore.hrTombstoneRangeRows(db, deviceId: deviceId) + if !tombstones.isEmpty { + hrRows = hrRows.filter { s in + !tombstones.contains { $0.fromTs <= s.ts && s.ts <= $0.toTs } + } + } + } + if !hrRows.isEmpty { let stmt = try db.cachedStatement(sql: """ INSERT INTO hrSample (deviceId, ts, bpm) VALUES (?, ?, ?) ON CONFLICT(deviceId, ts) DO NOTHING """) - for s in streams.hr { + for s in hrRows { try stmt.execute(arguments: [deviceId, s.ts, s.bpm]) hr += db.changesCount } @@ -335,6 +350,46 @@ extension WhoopStore { v18Written += 1 } } + // DURABLE SpO2 (v34) — the in-band `@82` percentages, forked out of the SAME v18 aux samples + // banked just above into a narrow table that is never pruned. The aux blob is capped at + // ~7 days (see the sweep below); this is the copy that survives, and without it every reading + // older than a week is destroyed with no second copy anywhere (the strap trims its own history + // as soon as an offload is acked). + // + // READ OFF THE DECODED STRUCT, NOT THE BLOB. `s.auxByte82` is the slot value; the alternative + // — indexing byte 23 of the packed `fields` blob, which is what the cloud reader does — is + // correct ONLY while every slot ahead of `auxByte82` in `V18AuxSlot` order is present, because + // `V18AuxCodec` is a presence-bitmap format that omits absent slots from the body entirely. + // With all slots present the arithmetic happens to land on 23 (5-byte header + 4+1+1+1+1+2+1+1 + // +2+2+2 = 18 body bytes ahead of it), which is why reading `fields[23]` yields physiologically + // valid values in production — but a single absent earlier slot silently shifts it onto a + // neighbouring byte and the read becomes garbage that still looks like a plausible percentage. + // `V18AuxCodecOffsetTests` pins both halves of that. Here the decoded struct is already in + // hand, so the fragile path is not merely avoided, it is unreachable. + // + // Separate loop rather than a branch inside the one above, so the `blob.isEmpty` skip there + // can never silently drop an SpO2 reading. (In practice a present `auxByte82` guarantees a + // non-empty blob, but coupling the two would make that a load-bearing invariant nobody stated.) + // + // Gate: `spo2CandidateInBand` only. `@82` is multiplexed — bit-7 sentinels (0x80/0x88/0x90/ + // 0xA0/0xA8), sub-70 diagnostic codes, and 0-for-not-emitted share the byte with real + // percentages, and `101...127` is an empty band that keeps the two separable. Nothing else is + // applied: no run grouping, no acquisition-ramp trim, no smoothing. Those are read-time + // policies (they have already changed once on the cloud reader) and baking one in here would + // destroy the evidence needed to change it again. + // + // ON CONFLICT DO NOTHING keeps the FIRST value banked for a second, matching every other + // per-second stream's dedupe rule and making a re-offload of the same window idempotent. + if !streams.v18Aux.isEmpty { + let stmt = try db.cachedStatement(sql: """ + INSERT INTO spo2PctSample (deviceId, ts, pct) VALUES (?, ?, ?) + ON CONFLICT(deviceId, ts) DO NOTHING + """) + for s in streams.v18Aux { + guard let v = s.auxByte82, spo2CandidateInBand.contains(v) else { continue } + try stmt.execute(arguments: [deviceId, s.ts, v]) + } + } return (hr, rr, ev, bat, spo2, skin, resp, grav) } @@ -604,6 +659,34 @@ extension WhoopStore { try syncRead { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM v18AuxSample") ?? 0 } } + /// Durable in-band `@82` SpO2 percentages (v34) in `[from, to]` for one device, ascending by ts. + /// + /// Unlike `v18AuxSamples`, this reaches back over the WHOLE recorded history: `spo2PctSample` is never + /// pruned, while the aux table it is forked from is capped at ~7 days. Empty for a WHOOP 4.0 (no v18 + /// aux stream) and for any window offloaded before v34 — the v31..v33 aux rows still on the device do + /// carry the byte, but only for whatever slice of the last seven days survives, and this reader + /// deliberately does NOT fall back to them: a series that silently changed source partway through + /// would be unusable as evidence. Backfilling those survivors is a one-shot migration's job, not a + /// read path's. + /// + /// Values are RAW in-band bytes. Run grouping and acquisition-ramp trimming are the CALLER's policy — + /// see `AnalyticsEngine.nightlySpo2Pct` for the nightly one. + public func spo2PctSamples(deviceId: String, from: Int, to: Int, limit: Int = 200_000) async throws + -> [Spo2PctSample] { + try syncRead { db in + try Row.fetchAll(db, sql: """ + SELECT ts, pct FROM spo2PctSample + WHERE deviceId = ? AND ts >= ? AND ts <= ? + ORDER BY ts LIMIT ? + """, arguments: [deviceId, from, to, limit]) + .map { Spo2PctSample(ts: $0["ts"], pct: $0["pct"]) } + } + } + + public func spo2PctCountForTest() async throws -> Int { + try syncRead { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM spo2PctSample") ?? 0 } + } + public func ppgHrCountForTest() async throws -> Int { try syncRead { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM ppgHrSample") ?? 0 } } diff --git a/Packages/WhoopStore/Sources/WhoopStore/TimestampHeal.swift b/Packages/WhoopStore/Sources/WhoopStore/TimestampHeal.swift index 8093834237..97781700a1 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/TimestampHeal.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/TimestampHeal.swift @@ -55,7 +55,12 @@ extension WhoopStore { let rawTables = ["hrSample", "rrInterval", "event", "battery", "spo2Sample", "skinTempSample", "respSample", "gravitySample", "stepSample", "ppgHrSample", - "sleepStateSample", "ppgWaveformSample", "rawImuSample", "v18AuxSample"] + "sleepStateSample", "ppgWaveformSample", "rawImuSample", "v18AuxSample", + // v34: the durable SpO2 percentages are keyed by the SAME type-47 `ts` as + // every stream above and forked from the same records, so a bad-clock strap + // can garbage them identically. They are never pruned, so an unhealed + // out-of-bounds row here would persist forever rather than rolling off. + "spo2PctSample"] var rawDeleted = 0 for table in rawTables { try db.execute(sql: "DELETE FROM \(table) WHERE ts < ? OR ts > ?", diff --git a/Packages/WhoopStore/Sources/WhoopStore/WalBackstop.swift b/Packages/WhoopStore/Sources/WhoopStore/WalBackstop.swift new file mode 100644 index 0000000000..b884c9dcec --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/WalBackstop.swift @@ -0,0 +1,222 @@ +import Foundation +import GRDB + +/// The size ceiling that bounds WAL growth when `WalCheckpointing.external` has disabled SQLite's +/// own autocheckpoint. +/// +/// ## Why a ceiling is mandatory, not optional +/// +/// `.external` exists so a page-level replicator (Litestream, or its embeddable Rust port `liters`) +/// can be the *only* checkpointer. That replicator checkpoints from inside its own push loop: +/// `liters` runs a PASSIVE checkpoint once the WAL passes `min_checkpoint_page_n = 1000` pages +/// (~4 MB) and an emergency TRUNCATE at `truncate_page_n = 121_359` pages (~500 MB) — but **both +/// live inside `push()`**. If the replicator never runs, neither fires, and nothing else in the +/// process will: `WhoopStore.checkpointWAL()` is called only from bulk-import and backup paths, and +/// the hot BLE ingest path never checkpoints at all. +/// +/// Measured on a NOOP-shaped workload (2M-row base, one transaction per `Collector` flush), each +/// commit dirties ~14 pages regardless of payload, so the cost tracks the **commit count**, not the +/// data volume: at the live lane's ~30 s cadence (2,880 commits/day) the WAL grows ~166 MB/day and +/// ~1.17 GB/week with nothing checkpointing. That is a strictly worse failure than the full +/// re-upload `.external` was adopted to avoid. +/// +/// ## Choosing `ceilingBytes` — the trade +/// +/// Firing the backstop is not free. A checkpoint taken while the replicator is asleep truncates the +/// WAL out from under its resume offset; on the next push it finds the resume frame gone and may +/// have to upload a **full snapshot** of the whole database. (`liters`' `meta.rs` deliberately does +/// not persist `synced_to_wal_end` across a writer close/reopen, so after an app relaunch a +/// truncation is always "unexpected".) So the ceiling must be high enough never to be reached in +/// healthy operation, and low enough that a dead replicator cannot exhaust storage. +/// +/// `standard` is **64 MiB**, chosen against the measured 6.9 MB/hour: +/// +/// | property | value | why it matters | +/// |---|---|---| +/// | vs. `liters`' own PASSIVE threshold (~4 MB) | 16× | a *healthy* replicator keeps the WAL near 4 MB, so it never comes close — a firing is evidence of a problem, never routine | +/// | time to reach at 166 MB/day | ~9.3 h | longer than any normal gap between foregrounds/BG refreshes, so it fires zero times in healthy operation | +/// | vs. `liters`' emergency TRUNCATE (~500 MB) | 1/8 | the app-side floor always acts first, which is required: `liters`' threshold cannot act while `liters` is asleep | +/// | vs. the 766 MB production database | 8.4% | peak disk stays ~1.08× the database, not ~1.65× | +/// | TRUNCATE duration | sub-second on iPhone NVMe | cannot strand an app suspension (`0xdead10cc`) | +/// | crash-recovery WAL scan at launch | ≤ ~16k frames | a multi-hundred-MB WAL makes every cold launch pay a full WAL scan | +/// +/// Lower ceilings (16–32 MB) fire every 2–5 h and would routinely cost a full snapshot. Higher ones +/// (256 MB+) make cold launch and checkpoint latency user-visible and double peak disk. 64 MiB is +/// the point where the backstop is a genuine backstop: it only ever fires when the thing it is +/// backing up has stopped. +public struct WalBackstopPolicy: Sendable, Equatable { + /// Force a checkpoint once the `-wal` sibling reaches this many bytes. `0` disables the backstop + /// entirely — only correct if something else provably bounds the WAL. + public let ceilingBytes: Int64 + + /// After an attempt that did not bring the WAL back under the ceiling — a `TRUNCATE` checkpoint + /// can be blocked by a `DatabasePool` reader holding an open snapshot — wait at least this long + /// before trying again, so a pinned WAL cannot turn every commit into a checkpoint attempt. + public let minRetryInterval: TimeInterval + + public init(ceilingBytes: Int64, minRetryInterval: TimeInterval = 60) { + self.ceilingBytes = ceilingBytes + self.minRetryInterval = minRetryInterval + } + + /// 64 MiB. See the type's documentation for the derivation. + public static let standard = WalBackstopPolicy(ceilingBytes: 64 * 1024 * 1024) + + /// No backstop. Passing this with `.external` asserts that the caller has another bound on WAL + /// growth; there is no such bound inside this package. + public static let disabled = WalBackstopPolicy(ceilingBytes: 0) + + public var isEnabled: Bool { ceilingBytes > 0 } +} + +/// The backstop's decision logic, with no I/O and no SQLite — so the interesting behaviour +/// (threshold, in-flight suppression, retry backoff) is unit-testable without a database. +struct WalBackstopCore { + let policy: WalBackstopPolicy + + private(set) var checkpointInFlight = false + private(set) var nextAttemptAt: TimeInterval = 0 + /// Number of checkpoints this backstop has started. Telemetry: a non-zero count on a device + /// running the replicator means the replicator stopped running. + private(set) var firings = 0 + /// Attempts that finished with the WAL still at or above the ceiling (blocked checkpoint). + private(set) var ineffectiveAttempts = 0 + + init(policy: WalBackstopPolicy) { + self.policy = policy + } + + /// Called once per committed transaction with the current `-wal` size. + /// Returns `true` when the caller should start a checkpoint. + mutating func shouldCheckpoint(walBytes: Int64, now: TimeInterval) -> Bool { + guard policy.isEnabled else { return false } + guard !checkpointInFlight else { return false } + guard walBytes >= policy.ceilingBytes else { return false } + guard now >= nextAttemptAt else { return false } + checkpointInFlight = true + firings += 1 + return true + } + + /// Called when the checkpoint started by `shouldCheckpoint` has finished, with the `-wal` size + /// measured afterwards. A checkpoint that did not get the WAL back under the ceiling arms the + /// retry backoff instead of letting the next commit try again immediately. + mutating func didFinishCheckpoint(walBytes: Int64, now: TimeInterval) { + checkpointInFlight = false + if walBytes >= policy.ceilingBytes { + ineffectiveAttempts += 1 + nextAttemptAt = now + policy.minRetryInterval + } else { + nextAttemptAt = 0 + } + } +} + +/// Watches the `-wal` file from inside the write path and forces a checkpoint when it crosses the +/// policy ceiling. +/// +/// ## Why a `TransactionObserver` +/// +/// The trigger has to be somewhere the write path passes through, or the backstop is only as good +/// as the caller's memory. GRDB notifies `databaseDidCommit(_:)` for **every** committed +/// transaction on the connection, regardless of what `observes(eventsOfKind:)` returns — so this +/// observer costs one `stat(2)` per commit (~1 µs) and *nothing* per row: returning `false` from +/// `observes` means GRDB never even installs the SQLite update hook for us. +/// +/// A `DatabasePool` only commits on its single writer connection, so exactly one observer fires. +/// +/// ## Why the checkpoint is dispatched, not run inline +/// +/// `databaseDidCommit` runs on the writer's serial queue inside statement execution. Calling +/// `writeWithoutTransaction` there would re-enter that queue. The checkpoint is therefore handed to +/// a private utility queue, which then blocks on the writer normally — after the commit that +/// triggered it has finished. +final class WalBackstopMonitor: TransactionObserver, @unchecked Sendable { + private let walPath: String + private let lock = NSLock() + private var core: WalBackstopCore + /// Weak: the pool's writer connection holds this observer (weakly, `.observerLifetime`) and the + /// store holds it strongly. Capturing the pool strongly here would close a retain cycle through + /// the store. + private weak var pool: DatabasePool? + private let queue = DispatchQueue(label: "whoopstore.wal-backstop", qos: .utility) + + init(databasePath: String, policy: WalBackstopPolicy) { + self.walPath = databasePath + "-wal" + self.core = WalBackstopCore(policy: policy) + } + + /// Installs the observer on `pool`'s writer connection. `.observerLifetime` (GRDB's default) + /// keeps the connection's reference weak, so the store's strong reference is what keeps this + /// alive — and there is no cycle. + func attach(to pool: DatabasePool) { + self.pool = pool + pool.add(transactionObserver: self, extent: .observerLifetime) + } + + /// Size of the `-wal` sibling, or 0 if it does not exist. A raw `stat(2)`, deliberately not + /// `FileManager.attributesOfItem` — this runs on every commit and must not allocate. + private func walSizeBytes() -> Int64 { + var st = stat() + guard walPath.withCString({ stat($0, &st) }) == 0 else { return 0 } + return Int64(st.st_size) + } + + // MARK: Telemetry (read by tests and by the device-trial recorder) + + /// Checkpoints this backstop has forced since the store opened. + var firings: Int { lock.withLock { core.firings } } + /// Forced checkpoints that finished with the WAL still over the ceiling. + var ineffectiveAttempts: Int { lock.withLock { core.ineffectiveAttempts } } + /// Current `-wal` size in bytes. Safe to call from anywhere. + var currentWalBytes: Int64 { walSizeBytes() } + + // MARK: - TransactionObserver + + /// `false` for every event kind: this observer wants transaction boundaries, not row changes. + /// GRDB skips installing the update hook entirely when no observer wants events, so per-row + /// cost is zero — while `databaseDidCommit` is still delivered for every transaction. + func observes(eventsOfKind eventKind: DatabaseEventKind) -> Bool { false } + + func databaseDidChange(with event: DatabaseEvent) {} + + func databaseDidCommit(_ db: Database) { + let now = Date().timeIntervalSinceReferenceDate + let walBytes = walSizeBytes() + let start = lock.withLock { core.shouldCheckpoint(walBytes: walBytes, now: now) } + guard start else { return } + queue.async { [self] in runCheckpoint() } + } + + func databaseDidRollback(_ db: Database) {} + + // MARK: - Checkpointing + + /// Mirrors `WhoopStore.checkpointWAL()`: a TRUNCATE checkpoint, run outside any transaction. + /// Best-effort — under a `DatabasePool` a reader holding an open snapshot can block it, which is + /// what `minRetryInterval` exists for. + private func runCheckpoint() { + if let pool { + try? pool.writeWithoutTransaction { db in + try db.execute(sql: "PRAGMA wal_checkpoint(TRUNCATE)") + } + } + let after = walSizeBytes() + let now = Date().timeIntervalSinceReferenceDate + lock.withLock { core.didFinishCheckpoint(walBytes: after, now: now) } + } + + /// Runs any pending checkpoint synchronously. Tests only: production fires from `databaseDidCommit` + /// on a private queue, and this is how a test waits for it deterministically. + func drainForTest() { + queue.sync {} + } +} + +private extension NSLock { + func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} diff --git a/Packages/WhoopStore/Sources/WhoopStore/WhoopStore.swift b/Packages/WhoopStore/Sources/WhoopStore/WhoopStore.swift index 10165bbfd7..8b76fdd7f4 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/WhoopStore.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/WhoopStore.swift @@ -9,6 +9,74 @@ public enum WhoopStoreInfo { public static let schemaVersion = 18 } +/// Who is responsible for checkpointing the WAL back into the main database file. +/// +/// SQLite's `wal_autocheckpoint` is a **per-connection** setting, not a database property, so this +/// is applied inside `Configuration.prepareDatabase` and therefore reaches every connection a +/// `DatabasePool` opens. +/// +/// ## Why this exists +/// +/// A page-level replicator (Litestream and its embeddable Rust port, `liters`) ships **WAL frames**, +/// so it must observe every transaction. To stop the WAL being restarted underneath it, it holds a +/// long-running read transaction — which starves every *other* checkpointer, including SQLite's own +/// autocheckpoint. If a foreign connection nevertheless succeeds in restarting the WAL, the +/// replicator finds its resume offset overwritten and must fall back to a **full snapshot of the +/// entire database**. On iOS that would be a multi-hundred-megabyte upload, on every app launch. +/// +/// Passing `.external` is what lets such a replicator be the single checkpointer. +/// +/// ## The obligation `.external` carries — read before using it +/// +/// With autocheckpoint off, **nothing in this package bounds WAL growth**. `checkpointWAL()` is only +/// called from bulk-import and backup paths; the hot BLE ingest path never checkpoints. Measured on +/// a NOOP-shaped workload (a 2M-row base, `Collector.flushStandardHR()`'s ~30 s commit cadence, one +/// transaction per flush): +/// +/// | cadence | commits/day | WAL after 1 day | after 7 days | +/// |---|---|---|---| +/// | every 10 s | 8,640 | 341 MB | — | +/// | **every 30 s (NOOP today)** | **2,880** | **166 MB** | **1.17 GB** | +/// | every 5 min | 288 | 37 MB | — | +/// +/// The cost is driven by the **commit count**, not the data volume: each commit dirties ~14 pages +/// (the file header's change counter, each table's rightmost leaf, each index leaf, the interior +/// spine) whether it carries 30 rows or 300. Only ~11 MB/day of that is new data; the rest is +/// re-written pages that autocheckpoint would normally reclaim. +/// +/// So an `.external` caller MUST guarantee a checkpointer actually runs. If it stops running — the +/// user disables sync, the network is down for a week, the replicator is never started, or it +/// crashes on launch — the WAL grows without limit and will eventually exhaust device storage. That +/// is a strictly worse failure than the full re-upload it was meant to avoid. +/// +/// ## How that obligation is discharged +/// +/// It is **not** left to the caller. Choosing `.external` installs a `WalBackstopMonitor` on the +/// pool's writer connection: one `stat(2)` of the `-wal` sibling per committed transaction, and a +/// forced `wal_checkpoint(TRUNCATE)` once it crosses `WalBackstopPolicy.ceilingBytes` (64 MiB by +/// default — see `WalBackstopPolicy` for how that number was chosen). Growth is therefore bounded +/// even when the replicator never runs at all. +/// +/// Opting out is possible (`walBackstop: .disabled`) but is an assertion that something else bounds +/// the WAL. Nothing in this package does. +/// +/// ## The multi-pool caveat, which is load-bearing +/// +/// `StoreOpenGate` gives every opener its **own** `DatabasePool` on the same file — in this app both +/// `Repository` and `BLEManager` open one (see `StoreOpenGate`'s note). `wal_autocheckpoint` is +/// per-connection, so a second store opened with `.automatic` on the same file will happily restart +/// the WAL under a replicator that the first store's `.external` was protecting. **Every** opener of +/// a replicated file must pass `.external`, or the option does nothing. +public enum WalCheckpointing: Sendable { + /// SQLite's default: each connection auto-checkpoints at ~1000 WAL pages (~4 MB). This is the + /// behaviour of every build that does not opt out, and the only behaviour upstream ships. + case automatic + /// Disable `wal_autocheckpoint` on every connection this store opens, because an external + /// component checkpoints instead. See the obligation documented above — which the + /// `walBackstop` parameter of `init(path:walCheckpointing:walBackstop:)` discharges by default. + case external +} + /// Serializes `DatabasePool` creation + migration so two concurrent opens of the SAME file can never /// run their GRDB migrators at once (#261). /// @@ -68,8 +136,24 @@ public actor WhoopStore { /// (the Pool serializes writes and runs reads in parallel under WAL). public nonisolated var registryWriter: any DatabaseWriter { dbWriter } - private init(dbWriter: any DatabaseWriter) throws { + /// Bounds WAL growth when `walCheckpointing` is `.external`. `nil` for `.automatic` (SQLite's own + /// autocheckpoint is the bound) and for a store with no backstop policy. Held strongly here + /// because GRDB's `.observerLifetime` registration keeps only a weak reference — this property is + /// what keeps the observer alive for the life of the store. + let walBackstop: WalBackstopMonitor? + + /// The checkpointing mode this store actually opened with, whether it came from an explicit + /// argument or from `StoreReplication`. Worth surfacing on a diagnostics screen next to + /// `walBackstopFirings`: "external + 0 firings" and "automatic" are very different states that + /// otherwise look identical from outside. + public nonisolated let walCheckpointing: WalCheckpointing + + private init(dbWriter: any DatabaseWriter, + walCheckpointing: WalCheckpointing = .automatic, + walBackstop: WalBackstopMonitor? = nil) throws { self.dbWriter = dbWriter + self.walCheckpointing = walCheckpointing + self.walBackstop = walBackstop try WhoopStore.makeMigrator().migrate(dbWriter) } @@ -77,8 +161,12 @@ public actor WhoopStore { /// `StoreOpenGate` (below) opened the pool and migrated it under the process-wide open lock, so /// re-migrating here would be a redundant (and, if it raced a sibling opener, failing) second run. /// (#261) - private init(preMigrated dbWriter: any DatabaseWriter) { + private init(preMigrated dbWriter: any DatabaseWriter, + walCheckpointing: WalCheckpointing = .automatic, + walBackstop: WalBackstopMonitor? = nil) { self.dbWriter = dbWriter + self.walCheckpointing = walCheckpointing + self.walBackstop = walBackstop } /// Open (creating if needed) a database at `path` and run migrations. @@ -87,7 +175,25 @@ public actor WhoopStore { /// /// Open + migrate runs through `StoreOpenGate` so two concurrent openers of the SAME file never /// run their GRDB migrators at once (#261) — see that actor's note for the failure it prevents. - public init(path: String) async throws { + /// + /// Both checkpointing parameters default to the process-wide `StoreReplication` policy, which is + /// itself `.automatic` + `.standard` unless something called `StoreReplication.configure`. So a + /// build that never configures replication behaves exactly as every build does today. + /// + /// The default deliberately comes from a process-wide value rather than being written out at each + /// call site: `wal_autocheckpoint` is per-connection, this app opens the same file from two + /// independent places, and an opener that forgets `.external` silently defeats it for all the + /// others. See `StoreReplication` for the full argument. Only an embedder that has taken over + /// checkpointing should select `.external` — see `WalCheckpointing` for the obligation that + /// carries. Passing the arguments explicitly still overrides the policy, which is what tests do. + /// + /// `walBackstop` is inert under `.automatic` (SQLite's autocheckpoint already bounds the WAL) and + /// is what discharges `.external`'s obligation: it forces a checkpoint if the `-wal` sibling + /// crosses its ceiling no matter what the external replicator is doing. Pass `.disabled` only if + /// something outside this package provably bounds WAL growth. + public init(path: String, + walCheckpointing: WalCheckpointing = StoreReplication.walCheckpointing, + walBackstop: WalBackstopPolicy = StoreReplication.walBackstop) async throws { var config = Configuration() config.prepareDatabase { db in // `DatabasePool` puts the database in WAL mode itself (reads run as concurrent snapshots @@ -99,10 +205,29 @@ public actor WhoopStore { try db.execute(sql: "PRAGMA cache_size = -16000") // ~16 MB page cache try db.execute(sql: "PRAGMA mmap_size = 268435456") // 256 MB memory-mapped I/O try db.execute(sql: "PRAGMA temp_store = MEMORY") + // `wal_autocheckpoint` is a PER-CONNECTION setting, so this has to run inside + // `prepareDatabase` — it is applied to every connection the pool opens (the writer and + // each reader), not once at open. Setting it anywhere else would leave the pool's other + // connections still auto-checkpointing, which is the exact bug this guards against. + if case .external = walCheckpointing { + try db.execute(sql: "PRAGMA wal_autocheckpoint = 0") + } } config.busyMode = .timeout(5) let pool = try await StoreOpenGate.shared.openAndMigrate(path: path, configuration: config) - self.init(preMigrated: pool) + + // Only `.external` needs a backstop: under `.automatic` SQLite's own autocheckpoint already + // bounds the WAL, and installing an observer there would be pure cost for upstream builds. + var monitor: WalBackstopMonitor? + if case .external = walCheckpointing, walBackstop.isEnabled { + let m = WalBackstopMonitor(databasePath: path, policy: walBackstop) + m.attach(to: pool) + monitor = m + } + // Recorded after the open succeeds, so a `configure` that lands between a failed open and a + // successful retry is not reported as late. See `StoreReplication.configuredAfterFirstOpen`. + StoreReplication.noteStoreOpened() + self.init(preMigrated: pool, walCheckpointing: walCheckpointing, walBackstop: monitor) } /// Move aside a database file that has our data tables but no GRDB migration bookkeeping — the @@ -180,6 +305,68 @@ public actor WhoopStore { } } + /// Write a complete, consistent, single-file copy of this database at `path` — **without + /// checkpointing the live one.** + /// + /// ## Why a file-level backup cannot just checkpoint when a replicator owns the WAL + /// + /// A `.noopbak` archives the main `.sqlite` alone, with no `-wal` sidecar, so a backup has to get + /// every committed page into one file somehow. The obvious way is `checkpointWAL()`. Under + /// `WalCheckpointing.external` that is exactly what must not happen, and — this is the part that + /// is easy to get wrong — **a gentler checkpoint mode does not help.** + /// + /// Measured against a real `liters` writer and Apple's `libsqlite3` (`LitersRoundTripTests`, + /// 2026-07-28), with no replicator read lock held: + /// + /// | foreign checkpoint | `-wal` afterwards | next push | + /// |---|---|---| + /// | `TRUNCATE` | 0 bytes | **snapshot**, "wal truncated by another process" | + /// | `FULL` | 168,952 bytes — unchanged | **snapshot**, same reason | + /// + /// `FULL` looks harmless because the `-wal` file keeps its size, and it is not: once a checkpoint + /// has fully backfilled the WAL and no reader still needs those frames, **SQLite restarts the WAL + /// on the next write transaction** — new salt, frame 1 — and the replicator's resume offset is + /// gone just the same. `PASSIVE` has the identical endpoint whenever it happens to complete. The + /// property that matters is not the pragma's name; it is "did the WAL end up fully backfilled + /// with nothing pinning it", and every checkpoint mode reaches that state. + /// + /// ## What this does instead + /// + /// SQLite's Online Backup API (via GRDB's `backup(to:)`) reads the source through a read + /// transaction, so it sees the WAL's contents and copies them into the destination — and it never + /// checkpoints, never restarts, and never touches the source's `-wal` at all. The replicator's + /// resume point survives untouched, and the caller gets a single file that carries every + /// committed row. + /// + /// The cost is one full-size write. That is why this is not the default path: under `.automatic` + /// a checkpoint is free and correct, and this is reserved for the case where a checkpoint is not + /// available (see `CloudSyncUploader.defaultExporter`, its only production caller). + /// + /// Removes `path` and its `-wal`/`-shm` siblings first, and leaves no sidecars behind, so the + /// result is one self-contained file ready to archive. + public func writeConsistentCopy(to path: String) async throws { + try writeConsistentCopyImpl(to: path) + } + + /// Non-async for the same reason `checkpointWALImpl` is: it runs on the actor's executor, off the + /// main thread, and GRDB's calls here are synchronous and blocking. + private func writeConsistentCopyImpl(to path: String) throws { + let fm = FileManager.default + for suffix in ["", "-wal", "-shm"] { try? fm.removeItem(atPath: path + suffix) } + + let destination = try DatabaseQueue(path: path) + try dbWriter.backup(to: destination) + // Close before anyone reads the file: GRDB flushes on close, and the caller's next act is to + // hand this path to a ZIP writer. `close()` throws only if a statement is still live, which + // cannot be the case here. + try destination.close() + + // A `DatabaseQueue` on a fresh file is in rollback-journal mode, but the copied page 1 header + // carries the source's WAL marker, so a `-wal`/`-shm` pair can be left behind. They hold + // nothing — everything was flushed by `close()` — and a `.noopbak` must be one file. + for suffix in ["-wal", "-shm"] { try? fm.removeItem(atPath: path + suffix) } + } + /// Permanently delete every recorded sample/derived row for one device across all `deviceId`-keyed /// tables (16+ `DELETE FROM
WHERE deviceId = ?` in one GRDB transaction). Wraps the /// synchronous `DeviceRegistryStore.deleteAllData` so the heavy multi-table write runs on the actor's @@ -220,6 +407,38 @@ public actor WhoopStore { return found ? total : nil } + // MARK: - WAL backstop + + /// Size of the `-wal` sibling alone, in bytes; `0` when there is no WAL and `nil` for an + /// in-memory store. A single `stat(2)` — cheap enough to poll. Unlike `databaseFileSizeBytes()` + /// this isolates the component that `WalCheckpointing.external` puts at risk. + public nonisolated func walFileSizeBytes() -> Int64? { + let base = dbWriter.path + guard base != ":memory:", !base.isEmpty else { return nil } + var st = stat() + guard (base + "-wal").withCString({ stat($0, &st) }) == 0 else { return 0 } + return Int64(st.st_size) + } + + /// How many times the WAL backstop has forced a checkpoint because the `-wal` crossed its + /// ceiling. `0` when no backstop is installed (`.automatic`, or `.disabled`). + /// + /// **This is the number to watch on a replicated device.** A healthy external replicator keeps + /// the WAL near its own ~4 MB threshold, so any non-zero value means the replicator stopped + /// running long enough for the WAL to grow 16× past that — and that each firing may have cost + /// the replicator its incremental resume point. + public nonisolated var walBackstopFirings: Int { walBackstop?.firings ?? 0 } + + /// Forced checkpoints that finished with the WAL still over the ceiling — i.e. a `TRUNCATE` + /// blocked by a pool reader holding an open snapshot. Persistently non-zero means the ceiling is + /// not actually being enforced and a reader is pinning the WAL. + public nonisolated var walBackstopIneffectiveAttempts: Int { walBackstop?.ineffectiveAttempts ?? 0 } + + /// The installed backstop, or `nil` when none is (`.automatic`, or `.disabled`). `nonisolated` + /// for the same reason `registryWriter` is: the monitor does its own locking. Used by tests and + /// by the fork-side sync telemetry. + nonisolated var walBackstopMonitor: WalBackstopMonitor? { walBackstop } + // MARK: - Introspection (used by tests) public func tableNames() async throws -> Set { diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/AppleStepHourStoreTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/AppleStepHourStoreTests.swift new file mode 100644 index 0000000000..a3bedb49cb --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/AppleStepHourStoreTests.swift @@ -0,0 +1,56 @@ +import XCTest +import GRDB +@testable import WhoopStore + +final class AppleStepHourStoreTests: XCTestCase { + func testUpsertIsIdempotentByNaturalKey() async throws { + let store = try await WhoopStore.inMemory() + let n1 = try await store.upsertAppleStepHours([(ts: 1_000, steps: 120)], deviceId: "apple-health") + XCTAssertEqual(n1, 1) + + // Re-import the same hour with a revised count → overwrite in place, still one row. + let n2 = try await store.upsertAppleStepHours([(ts: 1_000, steps: 250)], deviceId: "apple-health") + XCTAssertEqual(n2, 1) + + let rows = try await store.appleStepHours(deviceId: "apple-health", fromTs: 0, toTs: 10_000) + XCTAssertEqual(rows.count, 1) + XCTAssertEqual(rows.first?.ts, 1_000) + XCTAssertEqual(rows.first?.steps, 250) + } + + func testRangeReadReturnsOldestFirstWithinBoundsInclusive() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertAppleStepHours([ + (ts: 3_600, steps: 100), // hour 1 + (ts: 7_200, steps: 200), // hour 2 + (ts: 10_800, steps: 300), // hour 3 + ], deviceId: "apple-health") + + // Inclusive bounds: both edges of the range are included. + let rows = try await store.appleStepHours(deviceId: "apple-health", fromTs: 3_600, toTs: 10_800) + XCTAssertEqual(rows.map(\.ts), [3_600, 7_200, 10_800]) + XCTAssertEqual(rows.map(\.steps), [100, 200, 300]) + + // A narrower window excludes rows outside it. + let narrow = try await store.appleStepHours(deviceId: "apple-health", fromTs: 3_601, toTs: 10_799) + XCTAssertEqual(narrow.map(\.ts), [7_200]) + + // A different device sees no rows. + let other = try await store.appleStepHours(deviceId: "my-whoop", fromTs: 0, toTs: 100_000) + XCTAssertTrue(other.isEmpty) + } + + func testUpsertPartitionsByDevice() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertAppleStepHours([(ts: 1_000, steps: 50)], deviceId: "apple-health") + _ = try await store.upsertAppleStepHours([(ts: 1_000, steps: 999)], deviceId: "other-device") + + let mine = try await store.appleStepHours(deviceId: "apple-health", fromTs: 0, toTs: 10_000) + XCTAssertEqual(mine.count, 1) + XCTAssertEqual(mine.first?.steps, 50) + + let others = try await store.appleStepHours(deviceId: "other-device", fromTs: 0, toTs: 10_000) + XCTAssertEqual(others.count, 1) + XCTAssertEqual(others.first?.steps, 999) + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/CloudTombstoneTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/CloudTombstoneTests.swift new file mode 100644 index 0000000000..cc78b2a2e1 --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/CloudTombstoneTests.swift @@ -0,0 +1,93 @@ +import XCTest +@testable import WhoopStore +import WhoopProtocol + +final class CloudTombstoneTests: XCTestCase { + func testWorkoutTombstoneBlocksResurrection() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertWorkouts([WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", durationS: 100, energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, zonesJSON: nil, notes: nil)], deviceId: "d1") + try await store.addTombstone(kind: "workout", deviceId: "d1", startTs: 100, endTs: nil, sport: "running", day: nil, key: nil, editSeq: 7) + _ = try await store.deleteWorkouts(deviceId: "d1", sport: "running", from: 100, to: 100) + // Re-import attempts to resurrect — the guard must drop it. + let n = try await store.upsertWorkouts([WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", durationS: 100, energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, zonesJSON: nil, notes: nil)], deviceId: "d1") + XCTAssertEqual(n, 0) + // Idempotent tombstone insert (same kind+editSeq). + try await store.addTombstone(kind: "workout", deviceId: "d1", startTs: 100, endTs: nil, sport: "running", day: nil, key: nil, editSeq: 7) + } + func testHrRangeTombstoneFiltersInserts() async throws { + let store = try await WhoopStore.inMemory() + try await store.addTombstone(kind: "hrRange", deviceId: "d1", startTs: 1000, endTs: 2000, sport: nil, day: nil, key: nil, editSeq: 8) + try await store.insert(Streams(hr: [HRSample(ts: 1500, bpm: 60), HRSample(ts: 2500, bpm: 61)]), deviceId: "d1") + let survivors = try await store.hrSamples(deviceId: "d1", from: 0, to: 10_000, limit: 10) + XCTAssertEqual(survivors.map(\.ts), [2500]) + } + func testDeleteHelpersAndMetricGuard() async throws { + let store = try await WhoopStore.inMemory() + try await store.insert(Streams(hr: [HRSample(ts: 1200, bpm: 60)]), deviceId: "d1") + // Hoisted out of XCTAssertEqual's autoclosure: this toolchain rejects `await` inside it. + let deletedHr = try await store.deleteHrRange(deviceId: "d1", fromTs: 1000, toTs: 2000) + XCTAssertEqual(deletedHr, 1) + _ = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "ref_sleep_score", value: 80)], deviceId: "d1") + try await store.addTombstone(kind: "metricPoint", deviceId: "d1", startTs: nil, endTs: nil, sport: nil, day: "2026-07-01", key: "ref_sleep_score", editSeq: 9) + let deletedMetric = try await store.deleteMetricPoint(deviceId: "d1", day: "2026-07-01", key: "ref_sleep_score") + XCTAssertEqual(deletedMetric, 1) + let n = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "ref_sleep_score", value: 81)], deviceId: "d1") + XCTAssertEqual(n, 0) // guard drops the tombstoned point + } + + // Finding 1: LabMarkerStore.reprojectCells wrote metricSeries directly, bypassing the + // metricPoint tombstone guard every other writer (upsertMetricSeries) respects. A lab-marker + // upsert that reprojects a tombstoned (day, key) cell must NOT resurrect it. + func testLabMarkerReprojectionRespectsMetricPointTombstone() async throws { + let store = try await WhoopStore.inMemory() + // The lab-book projection always lands under the constant `labBookSourceId`, not the + // marker's own deviceId (see LabMarkerStore) — the tombstone must be recorded against that + // same id to guard the cell that reprojectCells actually writes. + try await store.addTombstone(kind: "metricPoint", deviceId: WhoopStore.labBookSourceId, + startTs: nil, endTs: nil, sport: nil, + day: "2026-01-10", key: "ldl", editSeq: 11) + // Upserting a numeric reading for that exact (day, key) triggers reprojectCells — the guard + // must drop the cell instead of resurrecting it into metricSeries. + _ = try await store.upsertLabMarkers([ + LabMarkerRow(id: "a", deviceId: "my-whoop", markerKey: "ldl", category: "bloodPanel", + day: "2026-01-10", takenAt: 1_736_500_000, value: 3.4, valueText: nil, + unit: "mmol/L", source: "manual", note: nil, referenceText: nil), + ]) + let proj = try await store.metricSeries(deviceId: WhoopStore.labBookSourceId, + key: "ldl", from: "2026-01-01", to: "2026-12-31") + XCTAssertEqual(proj.count, 0, "tombstoned cell must stay absent from metricSeries") + // The marker row itself is untouched — only the metricSeries projection is guarded. + let markers = try await store.labMarkers(deviceId: "my-whoop", markerKey: "ldl") + XCTAssertEqual(markers.count, 1) + } + + // Finding 2: cloudTombstone must survive "Delete this device's data" (DeviceRegistryStore + // .deleteAllData, reachable from DevicesView's keep-paired wipe) or the guard's memory is erased + // and a later BLE backfill/cloud re-sync can resurrect data the user deleted on the cloud side. + func testDeviceDataWipePreservesTombstoneUntilExplicitlyDeleted() async throws { + let store = try await WhoopStore.inMemory() + let workout = WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", + durationS: 100, energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, + distanceM: nil, zonesJSON: nil, notes: nil) + _ = try await store.upsertWorkouts([workout], deviceId: "d1") + try await store.addTombstone(kind: "workout", deviceId: "d1", startTs: 100, endTs: nil, + sport: "running", day: nil, key: nil, editSeq: 21) + + try await store.deleteAllData(deviceId: "d1") + + let workoutsAfterWipe = try await store.workouts(deviceId: "d1", from: 0, to: 1000, limit: 10) + XCTAssertEqual(workoutsAfterWipe.count, 0, "device-data wipe clears the workout") + let tombstonesAfterWipe = try await store.workoutTombstones(deviceId: "d1") + XCTAssertEqual(tombstonesAfterWipe.count, 1, "device-data wipe must NOT erase the tombstone") + + // A later re-sync/backfill re-importing the same workout must still be blocked. + let resurrected = try await store.upsertWorkouts([workout], deviceId: "d1") + XCTAssertEqual(resurrected, 0, "tombstone still blocks resurrection after a device-data wipe") + + // A dedicated full-disconnect flow (not deleteAllData) can forget the tombstone. + let deletedCount = try await store.deleteCloudTombstones(deviceId: "d1") + XCTAssertEqual(deletedCount, 1) + let tombstonesAfterDelete = try await store.workoutTombstones(deviceId: "d1") + XCTAssertEqual(tombstonesAfterDelete.count, 0) + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/ConsistentCopyTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/ConsistentCopyTests.swift new file mode 100644 index 0000000000..a8418c6a8a --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/ConsistentCopyTests.swift @@ -0,0 +1,153 @@ +import XCTest +import GRDB +import WhoopProtocol +@testable import WhoopStore + +/// `writeConsistentCopy(to:)` exists because a file-level backup and a page replicator both want to +/// decide what happens to the WAL, and only one of them can. It has to satisfy both: +/// +/// 1. **The copy must be whole.** A `.noopbak` is one `.sqlite` with no `-wal` sidecar, so a copy +/// that misses what the WAL held is silent data loss on restore. Asserted by opening the copy +/// somewhere its siblings do not exist and counting rows. +/// 2. **The live WAL must be untouched.** Not "not truncated" — *untouched*. A checkpoint that +/// fully backfills the WAL is just as fatal as one that truncates it, because SQLite restarts a +/// fully-backfilled WAL on the next write, which is what invalidates a replicator's resume +/// offset. So this asserts the `-wal` is byte-identical afterwards, and that a subsequent write +/// still *appends* rather than restarting. +/// +/// The end-to-end consequence — that a real `liters` writer ships a delta rather than a snapshot +/// after one of these — is asserted in `LitersRoundTripTests`, which has an actual replicator. +final class ConsistentCopyTests: XCTestCase { + + private var paths: [String] = [] + + override func tearDown() { + for path in paths { + for suffix in ["", "-wal", "-shm"] { try? FileManager.default.removeItem(atPath: path + suffix) } + } + paths = [] + super.tearDown() + } + + private func tempPath(_ tag: String = "src") -> String { + let path = NSTemporaryDirectory() + "whoopstore-copy-\(tag)-\(UUID().uuidString).sqlite" + paths.append(path) + return path + } + + private func fileSize(_ path: String) -> Int64 { + ((try? FileManager.default.attributesOfItem(atPath: path))?[.size] as? NSNumber)?.int64Value ?? 0 + } + + @discardableResult + private func writeTransactions(_ store: WhoopStore, count: Int, rowsEach: Int = 400, + startingAt ts0: Int = 0) async throws -> Int { + var ts = ts0 + for _ in 0.. Int { + XCTAssertFalse(FileManager.default.fileExists(atPath: path + "-wal"), + "a staged copy must leave no -wal sidecar; a .noopbak carries one file") + let queue = try DatabaseQueue(path: path) + return try queue.read { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM hrSample") ?? -1 } + } + + // MARK: - The copy is complete + + /// Everything committed — including the rows that exist only in the WAL, which is most of them + /// on a replicated store because nothing checkpoints it. + func testTheCopyCarriesEveryCommittedRowIncludingTheOnesStillInTheWal() async throws { + let source = tempPath() + let store = try await WhoopStore(path: source, walCheckpointing: .external, walBackstop: .disabled) + try await writeTransactions(store, count: 20) + + XCTAssertGreaterThan(store.walFileSizeBytes() ?? 0, 0, + "the test is only meaningful if rows are sitting in the WAL") + + let copy = tempPath("dst") + try await store.writeConsistentCopy(to: copy) + + XCTAssertEqual(try hrRows(inFileAlone: copy), 20 * 400) + } + + /// The same under `.automatic`, so the method is not quietly coupled to the replication mode. + func testTheCopyIsCompleteUnderAutomaticCheckpointingToo() async throws { + let source = tempPath() + let store = try await WhoopStore(path: source) + try await writeTransactions(store, count: 10) + + let copy = tempPath("dst") + try await store.writeConsistentCopy(to: copy) + + XCTAssertEqual(try hrRows(inFileAlone: copy), 10 * 400) + } + + // MARK: - The live WAL is untouched + + /// THE CORE ASSERTION. Byte-identical, not merely "not empty": the replicator's resume offset is + /// a position in this file, and any checkpoint — `TRUNCATE`, `FULL`, `PASSIVE` — ends with the + /// WAL fully backfilled and about to be restarted by the next write. + func testTheCopyDoesNotTouchTheLiveWal() async throws { + let source = tempPath() + let store = try await WhoopStore(path: source, walCheckpointing: .external, walBackstop: .disabled) + try await writeTransactions(store, count: 20) + + let walBefore = fileSize(source + "-wal") + let mainBefore = fileSize(source) + XCTAssertGreaterThan(walBefore, 0) + + let copy = tempPath("dst") + try await store.writeConsistentCopy(to: copy) + + XCTAssertEqual(fileSize(source + "-wal"), walBefore, + "the live -wal must be byte-identical — this is the replicator's transport") + XCTAssertEqual(fileSize(source), mainBefore, + "the live database file must not be backfilled either; a fully-backfilled WAL " + + "is restarted by the next write, which is what costs a replicator its resume point") + } + + /// The consequence of the assertion above, stated as behaviour rather than as file sizes: writes + /// after a copy still *append* to the WAL. If the copy had checkpointed, the next write would + /// restart it and the `-wal` would shrink. + func testWritesAfterACopyStillAppendToTheSameWal() async throws { + let source = tempPath() + let store = try await WhoopStore(path: source, walCheckpointing: .external, walBackstop: .disabled) + var ts = try await writeTransactions(store, count: 20) + + let copy = tempPath("dst") + try await store.writeConsistentCopy(to: copy) + let walAfterCopy = fileSize(source + "-wal") + + _ = try await writeTransactions(store, count: 5, startingAt: ts) + ts += 1 + + XCTAssertGreaterThan(fileSize(source + "-wal"), walAfterCopy, + "the WAL must have grown, not restarted") + } + + // MARK: - Housekeeping + + /// Called once per fallback sync, forever, over a path the caller reuses on failure paths. An + /// existing file (and any stale sidecars) must be replaced, not appended to or refused. + func testWritingOverAnExistingCopyReplacesIt() async throws { + let source = tempPath() + let store = try await WhoopStore(path: source, walCheckpointing: .external, walBackstop: .disabled) + try await writeTransactions(store, count: 5) + + let copy = tempPath("dst") + try await store.writeConsistentCopy(to: copy) + XCTAssertEqual(try hrRows(inFileAlone: copy), 5 * 400) + + _ = try await writeTransactions(store, count: 5, startingAt: 5 * 400) + try await store.writeConsistentCopy(to: copy) + XCTAssertEqual(try hrRows(inFileAlone: copy), 10 * 400, + "the second copy must replace the first, not be refused because the file exists") + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/ContentTokenTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/ContentTokenTests.swift new file mode 100644 index 0000000000..762187c449 --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/ContentTokenTests.swift @@ -0,0 +1,75 @@ +import XCTest +import WhoopProtocol +@testable import WhoopStore + +final class ContentTokenTests: XCTestCase { + func testTokenIsStableAndDeterministicOnAnEmptyStore() async throws { + let store = try await WhoopStore.inMemory() + let a = try await store.contentToken() + let b = try await store.contentToken() + XCTAssertEqual(a, b) + XCTAssertEqual(a, "hr:0:0|sleep:0:0|daily:0:|workout:0|series:0") + } + + func testInsertingHrSampleChangesToken() async throws { + let store = try await WhoopStore.inMemory() + let before = try await store.contentToken() + _ = try await store.insert(Streams(hr: [HRSample(ts: 100, bpm: 60)]), deviceId: "dev1") + let after = try await store.contentToken() + XCTAssertNotEqual(before, after) + } + + func testInsertingSleepSessionChangesToken() async throws { + let store = try await WhoopStore.inMemory() + let before = try await store.contentToken() + _ = try await store.upsertSleepSessions( + [CachedSleepSession(startTs: 1000, endTs: 5000, efficiency: nil, restingHr: nil, + avgHrv: nil, stagesJSON: nil)], deviceId: "dev1") + let after = try await store.contentToken() + XCTAssertNotEqual(before, after) + } + + func testInsertingDailyMetricChangesToken() async throws { + let store = try await WhoopStore.inMemory() + let before = try await store.contentToken() + _ = try await store.upsertDailyMetrics( + [DailyMetric(day: "2026-05-23", totalSleepMin: 420, efficiency: 0.9, deepMin: nil, + remMin: nil, lightMin: nil, disturbances: nil, restingHr: nil, avgHrv: nil, + recovery: nil, strain: nil, exerciseCount: nil)], deviceId: "dev1") + let after = try await store.contentToken() + XCTAssertNotEqual(before, after) + } + + func testInsertingWorkoutChangesToken() async throws { + let store = try await WhoopStore.inMemory() + let before = try await store.contentToken() + _ = try await store.upsertWorkouts( + [WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", durationS: 100, + energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, + zonesJSON: nil, notes: nil)], deviceId: "dev1") + let after = try await store.contentToken() + XCTAssertNotEqual(before, after) + } + + func testInsertingMetricSeriesPointChangesToken() async throws { + let store = try await WhoopStore.inMemory() + let before = try await store.contentToken() + _ = try await store.upsertMetricSeries( + [MetricPoint(day: "2026-05-20", key: "restingHr", value: 54)], deviceId: "dev1") + let after = try await store.contentToken() + XCTAssertNotEqual(before, after) + } + + // The whole point of this token: sync bookkeeping (cursors) must never look like a data change, + // or every pull/highwater bump would force a full re-upload regardless of whether anything a + // backup actually carries moved. + func testCursorWriteDoesNotChangeToken() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.insert(Streams(hr: [HRSample(ts: 100, bpm: 60)]), deviceId: "dev1") + let before = try await store.contentToken() + try await store.setCursor("cloud_edits", 42) + try await store.setCursor("highwater:hr", 999) + let after = try await store.contentToken() + XCTAssertEqual(before, after) + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/DatabaseIntegrityTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/DatabaseIntegrityTests.swift index 165ff56b57..e67738e0ff 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/DatabaseIntegrityTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/DatabaseIntegrityTests.swift @@ -1,5 +1,6 @@ import XCTest import GRDB +import WhoopProtocol @testable import WhoopStore /// #1014 defence-in-depth: real-file coverage for the `PRAGMA quick_check` integrity gate the @@ -106,4 +107,35 @@ final class DatabaseIntegrityTests: XCTestCase { XCTAssertEqual(DatabaseIntegrity.verdict(fromRows: ["ok", "Page 9 is never used"]), "Page 9 is never used") } + + // MARK: - The probe must not checkpoint a replicated store + + /// `quickCheckFailure` opens its own GRDB connection, which is the one connection in this package + /// that reaches a live database without going through `WhoopStore(path:)` — and therefore without + /// `StoreReplication`'s checkpointing policy. Under `WalCheckpointing.external` a checkpoint here + /// would restart the WAL under the replicator and cost it its incremental resume point, turning + /// the next push into a full snapshot of the whole database. + /// + /// Asserted as an observable property (the `-wal` does not shrink), not as "the pragma was set", + /// so it holds whichever of the read-only or read-write branch the probe actually took. + func testProbingALiveExternallyCheckpointedStoreDoesNotShrinkItsWal() async throws { + let path = NSTemporaryDirectory() + "dbintegrity-wal-\(UUID().uuidString).sqlite" + defer { for s in ["", "-wal", "-shm"] { try? FileManager.default.removeItem(atPath: path + s) } } + + // Backstop disabled so the ONLY thing that could shrink the WAL is the probe. + let store = try await WhoopStore(path: path, walCheckpointing: .external, walBackstop: .disabled) + var ts = 0 + for _ in 0..<10 { + var hr: [HRSample] = [] + for _ in 0..<400 { ts += 1; hr.append(HRSample(ts: ts, bpm: 60 + ts % 40)) } + _ = try await store.insert(Streams(hr: hr), deviceId: "dev") + } + let before = try XCTUnwrap(store.walFileSizeBytes()) + XCTAssertGreaterThan(before, 0, "the test needs a non-empty WAL to be meaningful") + + XCTAssertNil(DatabaseIntegrity.quickCheckFailure(atPath: path), "the store is healthy") + + XCTAssertEqual(store.walFileSizeBytes(), before, + "the integrity probe must not checkpoint a store an external replicator owns") + } } diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/DeviceRegistryStoreTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/DeviceRegistryStoreTests.swift index ed7b01f3e9..6aba8c4a5d 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/DeviceRegistryStoreTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/DeviceRegistryStoreTests.swift @@ -117,9 +117,11 @@ final class DeviceRegistryStoreTests: XCTestCase { } // Regression guard (audit finding): every table with a `deviceId` column MUST appear in - // `deviceScopedTables`, or `deleteAllData` silently leaves that device's rows behind — a privacy - // defect for a delete-means-gone app. Enumerate the live schema and fail if any deviceId-keyed table - // is uncovered, so a future migration that adds one can't reintroduce the gap. + // `deviceScopedTables` OR the explicit `deviceScopedTableExemptions` allowlist, or `deleteAllData` + // silently leaves that device's rows behind — a privacy defect for a delete-means-gone app. + // Enumerate the live schema and fail if any deviceId-keyed table is uncovered by either list, so a + // future migration that adds one can't reintroduce the gap (and can't silently opt out of the wipe + // either — a genuine exemption must be named in `deviceScopedTableExemptions` with a reason). func testDeviceScopedTablesCoversEveryDeviceIdKeyedTable() throws { let dbq = try makeDB() let uncovered = try dbq.read { db -> [String] in @@ -131,14 +133,16 @@ final class DeviceRegistryStoreTests: XCTestCase { for table in tables { let cols = try Row.fetchAll(db, sql: "PRAGMA table_info(\(table))") let hasDeviceId = cols.contains { ($0["name"] as String?) == "deviceId" } - if hasDeviceId && !DeviceRegistryStore.deviceScopedTables.contains(table) { + let covered = DeviceRegistryStore.deviceScopedTables.contains(table) + || DeviceRegistryStore.deviceScopedTableExemptions.contains(table) + if hasDeviceId && !covered { missing.append(table) } } return missing } XCTAssertTrue(uncovered.isEmpty, - "deviceId-keyed tables missing from deviceScopedTables (deleteAllData would skip them): \(uncovered)") + "deviceId-keyed tables missing from deviceScopedTables/deviceScopedTableExemptions (deleteAllData would skip them): \(uncovered)") } func testDayOwnershipUpsertAndRead() throws { diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/JournalWorkoutAppleCacheTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/JournalWorkoutAppleCacheTests.swift index 98e390d403..3fc7251483 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/JournalWorkoutAppleCacheTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/JournalWorkoutAppleCacheTests.swift @@ -259,6 +259,29 @@ final class JournalWorkoutAppleCacheTests: XCTestCase { XCTAssertEqual(other.count, 1, "other device untouched") } + func testWorkoutExists() async throws { + let store = try await WhoopStore.inMemory() + let w = WorkoutRow(startTs: 100, endTs: 700, sport: "running", source: "whoop", durationS: 600, + energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, + zonesJSON: nil, notes: nil) + try await store.upsertWorkouts([w], deviceId: "devA") + + let exists = try await store.workoutExists(deviceId: "devA", startTs: 100, sport: "running") + XCTAssertTrue(exists) + + // A mismatch on any single key column must miss. + let wrongSport = try await store.workoutExists(deviceId: "devA", startTs: 100, sport: "cycling") + XCTAssertFalse(wrongSport) + let wrongStart = try await store.workoutExists(deviceId: "devA", startTs: 200, sport: "running") + XCTAssertFalse(wrongStart) + let wrongDevice = try await store.workoutExists(deviceId: "devB", startTs: 100, sport: "running") + XCTAssertFalse(wrongDevice) + + _ = try await store.deleteWorkouts(deviceId: "devA", sport: "running", from: 100, to: 100) + let afterDelete = try await store.workoutExists(deviceId: "devA", startTs: 100, sport: "running") + XCTAssertFalse(afterDelete, "a deleted workout must no longer report as existing") + } + // MARK: - appleDaily func testAppleDailyUpsertReadAndIdempotency() async throws { diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/MetricsCacheTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/MetricsCacheTests.swift index 2f41d3062c..3485f101eb 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/MetricsCacheTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/MetricsCacheTests.swift @@ -364,6 +364,24 @@ final class MetricsCacheTests: XCTestCase { XCTAssertEqual(rows.map { $0.day }, ["2026-05-20"]) } + // MARK: - empty-store guard (incident: an empty test-host DB must never look "has data") + + func testHasAnyDailyMetricsFalseWhenEmptyTrueAfterUpsert() async throws { + let store = try await WhoopStore.inMemory() + // A freshly-created store (no BLE pairing, no import ever run) — the exact shape of the + // production incident's empty macOS test-host database — must read false. + let beforeAnyData = try await store.hasAnyDailyMetrics() + XCTAssertFalse(beforeAnyData) + + try await store.upsertDailyMetrics([ + DailyMetric(day: "2026-05-23", totalSleepMin: nil, efficiency: nil, deepMin: nil, remMin: nil, + lightMin: nil, disturbances: nil, restingHr: nil, avgHrv: nil, recovery: nil, + strain: nil, exerciseCount: nil), + ], deviceId: "devA") + let afterUpsert = try await store.hasAnyDailyMetrics() + XCTAssertTrue(afterUpsert) + } + // MARK: - windowed computed-daily delete (#277 local-day re-bucketing migration) func testDeleteDailyMetricsInRangeKeepsImportedAndOutOfRange() async throws { @@ -485,6 +503,63 @@ final class MetricsCacheTests: XCTestCase { XCTAssertNil(bareRow.activeKcalEst) } + // MARK: - v29 whole-night SDNN column (avgSdnn, alongside avgHrv=RMSSD) + + func testV29AvgSdnnColumnPresent() async throws { + let store = try await WhoopStore.inMemory() + let cols = try await store.columnNamesForTest(table: "dailyMetric") + XCTAssertTrue(cols.contains("avgSdnn"), "dailyMetric missing v29 avgSdnn column") + } + + func testV29AvgSdnnRoundTrip() async throws { + let store = try await WhoopStore.inMemory() + // avgHrv carries RMSSD; avgSdnn carries the whole-night SDNN — distinct values on the same row. + let d = DailyMetric(day: "2026-05-29", totalSleepMin: 415, efficiency: 0.9, + deepMin: 88, remMin: 108, lightMin: 219, disturbances: 2, + restingHr: 50, avgHrv: 62.0, recovery: 0.73, strain: 10.5, + exerciseCount: 1, spo2Pct: 96.2, skinTempDevC: 0.0, respRateBpm: 14.7, + steps: 7_900, activeKcalEst: 2_180.0, avgSdnn: 88.4) + try await store.upsertDailyMetrics([d], deviceId: "devA") + + let rows = try await store.dailyMetrics(deviceId: "devA", from: "2026-05-01", to: "2026-05-31") + XCTAssertEqual(rows.count, 1) + let row = try XCTUnwrap(rows.first) + XCTAssertEqual(try XCTUnwrap(row.avgHrv), 62.0, accuracy: 0.001, "RMSSD must survive intact") + XCTAssertEqual(try XCTUnwrap(row.avgSdnn), 88.4, accuracy: 0.001, "SDNN must persist distinctly") + + // Omitting avgSdnn keeps it nil (defaulted init — old call sites unchanged). + let bare = DailyMetric(day: "2026-05-30", totalSleepMin: nil, efficiency: nil, + deepMin: nil, remMin: nil, lightMin: nil, disturbances: nil, + restingHr: nil, avgHrv: 55.0, recovery: nil, strain: nil, exerciseCount: nil) + try await store.upsertDailyMetrics([bare], deviceId: "devA") + let bareRows = try await store.dailyMetrics(deviceId: "devA", from: "2026-05-30", to: "2026-05-30") + let bareRow = try XCTUnwrap(bareRows.first) + XCTAssertEqual(try XCTUnwrap(bareRow.avgHrv), 55.0, accuracy: 0.001) + XCTAssertNil(bareRow.avgSdnn, "avgSdnn defaults to nil when the source has no whole-night SDNN") + } + + func testV29AvgSdnnUpsertUpdates() async throws { + let store = try await WhoopStore.inMemory() + // Insert with nil SDNN, then re-upsert the same day with SDNN populated (WHOOP backfill path). + let d1 = DailyMetric(day: "2026-05-31", totalSleepMin: 400, efficiency: 0.88, + deepMin: 80, remMin: 100, lightMin: 220, disturbances: 3, + restingHr: 54, avgHrv: 60.0, recovery: 0.65, strain: 13.0, exerciseCount: 0) + try await store.upsertDailyMetrics([d1], deviceId: "devA") + let preRows = try await store.dailyMetrics(deviceId: "devA", from: "2026-05-31", to: "2026-05-31") + var row = try XCTUnwrap(preRows.first) + XCTAssertNil(row.avgSdnn) + + let d2 = DailyMetric(day: "2026-05-31", totalSleepMin: 400, efficiency: 0.88, + deepMin: 80, remMin: 100, lightMin: 220, disturbances: 3, + restingHr: 54, avgHrv: 60.0, recovery: 0.65, strain: 13.0, exerciseCount: 0, + avgSdnn: 91.2) + try await store.upsertDailyMetrics([d2], deviceId: "devA") + let rows = try await store.dailyMetrics(deviceId: "devA", from: "2026-05-31", to: "2026-05-31") + XCTAssertEqual(rows.count, 1, "upsert must not duplicate") + row = try XCTUnwrap(rows.first) + XCTAssertEqual(try XCTUnwrap(row.avgSdnn), 91.2, accuracy: 0.001) + } + // MARK: - read highwater cursor (distinct prefix from upload highwater) func testReadHighwaterRoundTripsUnderDistinctPrefix() async throws { diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/PhoneTimezoneStoreTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/PhoneTimezoneStoreTests.swift new file mode 100644 index 0000000000..3222094889 --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/PhoneTimezoneStoreTests.swift @@ -0,0 +1,45 @@ +import XCTest +import GRDB +@testable import WhoopStore + +final class PhoneTimezoneStoreTests: XCTestCase { + + /// v28 creates the phoneTimezone table with `day` as the sole primary key. + func testV28CreatesPhoneTimezoneTable() async throws { + let store = try await WhoopStore.inMemory() + let tables = try await store.tableNames() + XCTAssertTrue(tables.contains("phoneTimezone"), "missing v28 phoneTimezone table") + let cols = try await store.primaryKeyColumns("phoneTimezone") + XCTAssertEqual(cols, ["day"]) + } + + func testUpsertIsIdempotentByDay() async throws { + let store = try await WhoopStore.inMemory() + let n1 = try await store.upsertPhoneTimezone(day: "2026-07-14", tzId: "America/Los_Angeles") + XCTAssertEqual(n1, 1) + + // Same day, phone now on Eastern → overwrite in place, still one row. + let n2 = try await store.upsertPhoneTimezone(day: "2026-07-14", tzId: "America/New_York") + XCTAssertEqual(n2, 1) + + let tz = try await store.phoneTimezone(day: "2026-07-14") + XCTAssertEqual(tz, "America/New_York") + } + + func testDistinctDaysCoexist() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertPhoneTimezone(day: "2026-07-13", tzId: "America/Los_Angeles") + _ = try await store.upsertPhoneTimezone(day: "2026-07-14", tzId: "America/New_York") + + let d13 = try await store.phoneTimezone(day: "2026-07-13") + let d14 = try await store.phoneTimezone(day: "2026-07-14") + XCTAssertEqual(d13, "America/Los_Angeles") + XCTAssertEqual(d14, "America/New_York") + } + + func testMissingDayReturnsNil() async throws { + let store = try await WhoopStore.inMemory() + let tz = try await store.phoneTimezone(day: "1999-01-01") + XCTAssertNil(tz) + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json b/Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json index 72081fadf5..585c4e13ef 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json @@ -1,6 +1,6 @@ { - "_readme": "SHARED Room<->GRDB SCHEMA ORACLE (#775). Two byte-identical copies: Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json and android/app/src/test/resources/schema_oracle.json. SchemaOracleTests.swift compares GRDB's PRAGMA table_info/index_list against it; SchemaOracleTest.kt compares Room's exported schema JSON against it. `columns` is the iOS/GRDB shape in GRDB column order (macOS is the reference implementation); every way Android differs is spelled out in an `android` / `iosAbsent` / `androidColumnOrder` override naming a key in `divergenceReasons`. Adding a column, reordering one, or changing a type/nullability on one platform only fails both suites until it is either fixed or written down here.", - "roomVersion": 25, + "_readme": "SHARED Room<->GRDB SCHEMA ORACLE (#775). Two byte-identical copies: Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json and android/app/src/test/resources/schema_oracle.json. SchemaOracleTests.swift compares GRDB's PRAGMA table_info/index_list against it; SchemaOracleTest.kt compares Room's exported schema JSON against it. `columns` is the iOS/GRDB shape in GRDB column order (macOS is the reference implementation); every way Android differs is spelled out in an `android` / `iosAbsent` / `androidColumnOrder` override naming a key in `divergenceReasons`. Adding a column, reordering one, or changing a type/nullability on one platform only fails both suites until it is either fixed or written down here. `grdbMigrationLineages` is the same idea for migration IDENTIFIERS: a fork that numbered its own migrations before upstream reached those numbers declares them there instead of renumbering \u2014 GRDB keys migrations by identifier string, so renaming one wedges a database that already ran it.", + "roomVersion": 26, "grdbMigrations": [ "v1", "v2", @@ -27,15 +27,38 @@ "v23-daily-spo2-raw", "v24-rr-seq", "v25-oura-raw", + "v26-cloud-tombstone", + "v27-apple-step-hour", "v26-efficiency-heal", "v27-ppg-waveform", + "v28-phone-timezone", + "v29-daily-avg-sdnn", "v28-raw-imu", "v29-score-input-provenance", "v30-rr-ord", - "v31-deep-capture-channels" + "v31-deep-capture-channels", + "v34-spo2-pct-durable" ], + "grdbMigrationLineages": { + "fork-ios-only": { + "reason": "This fork (vishk23/noop) carries four iOS-only migrations that were numbered from v26 before upstream's own v26..v29 existed, so both lineages claim v26, v27, v28 and v29. They are NOT renumbered to sit after upstream's: GRDB keys migrations by identifier STRING, so renaming one makes it read as un-applied on a device that already ran it, and the migrator then re-runs its body against a schema that already has the table \u2014 wedging a live database. The identifiers are frozen exactly as first shipped. What still carries the vN <-> Room-version mapping is the BASELINE lineage (grdbMigrations minus every lineage declared here), which both suites still check is exactly v1..vN with no gaps and no repeats; a lineage's own vN must strictly increase; and full-identifier uniqueness is enforced across everything, because a genuinely duplicated identifier makes GRDB silently skip the second body.", + "migrations": [ + "v26-cloud-tombstone", + "v27-apple-step-hour", + "v28-phone-timezone", + "v29-daily-avg-sdnn" + ] + }, + "fork-ahead-of-upstream": { + "reason": "This fork numbers a migration AHEAD of upstream on purpose, which is the mirror image of the `fork-ios-only` case and needs its own entry rather than being folded into it. Local main carries migrations through v31; upstream ryanbr/noop already holds `v32-rr-src-channel` and `v33-workout-steps`, which this branch has not merged yet. Numbering this one v32 would therefore collide with an upstream identifier the very next time the fork merges upstream \u2014 and because GRDB keys migrations by identifier STRING, that collision is not a merge conflict a human sees: two different bodies claiming the same name means GRDB silently SKIPS the second, so one of the two tables is never created and the failure surfaces later as a missing-table error on a real device. Starting at v34 leaves upstream's v32/v33 free to arrive untouched. It is declared here rather than renumbered because the baseline lineage must stay exactly v1..vN, and because once this identifier has run on any device renaming it re-runs its body against a schema that already has the table.", + "migrations": [ + "v34-spo2-pct-durable" + ] + } + }, "divergenceReasons": { "android-orphan-synced-column": "REAL COLUMN DRIFT: `synced` exists on Android only. GRDB's v10 note is explicit that stepSample gets no `synced` column ('unused; see StreamStore'), and v12's ppgHrSample never had one either; the Room entities copied the flag from the older per-second entities. Nothing reads it on Android \u2014 the per-row upload flag belongs to an upload path that does not exist there \u2014 so it is dead width, not divergent data. Removing it needs a Room table rebuild.", + "avg-sdnn-ios-only": "FORK-ONLY COLUMN: `dailyMetric.avgSdnn` is appended by the fork's GRDB `v29-daily-avg-sdnn` \u2014 the 5-minute SDNN index, the broad-variability twin of `avgHrv` (RMSSD), window-matched to a watch's short SDNN rather than the drift-inflated whole-night SD, and computed by `HRVAnalyzer.sdnnIndex` during the nightly analysis. Additive and nullable, so it costs an existing row nothing. Room has no field for it; giving it one means a @Database version bump plus a migration, so the absence is recorded here rather than left to read as unexplained drift.", "battery-alter-append-order": "REAL COLUMN-ORDER DRIFT (CLAUDE.md: 'column order in a Room CREATE TABLE must match the entity field order'). GRDB appended `synced` (v5) then `charging` (v6) with ALTER TABLE, which can only append; the Room entity declares `charging` before `synced`. Same columns, same types, different positions \u2014 so any positional copy (INSERT INTO battery SELECT * FROM \u2026) swaps them. Fixing it means reordering the Room entity's fields, which changes Room's identity hash and therefore needs a version bump plus a no-op migration.", "grdb-boolean-affinity": "GRDB's .boolean column type is declared BOOLEAN, which SQLite gives NUMERIC affinity; Room maps Kotlin Boolean to INTEGER. For the 0/1 values both sides write, NUMERIC and INTEGER affinity store bit-identical INTEGERs, so a .noopbak round-trips. Closing it means a GRDB table rebuild.", "paired-device-alter-append-order": "REAL COLUMN-ORDER DRIFT, same shape as battery. GRDB's v16-paired-device-peripheral appended `peripheralId` at the end; the Room entity declares it fifth, between `nickname` and `sourceKind`.", @@ -115,6 +138,35 @@ ], "indices": [] }, + "appleStepHour": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v27-apple-step-hour`): hourly Apple Health step counts. The daily HealthKit collect path flattens a whole day into one `appleDaily.steps` total, so hours the phone was dead or absent read as low steps for the entire day instead of as the specific hours with no recording. Android has no HealthKit source, so no Room twin.", + "columns": [ + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "ts", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "steps", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "deviceId", + "ts" + ], + "indices": [] + }, "battery": { "platform": "both", "columns": [ @@ -178,6 +230,85 @@ "androidColumnOrderDivergence": "battery-alter-append-order", "indices": [] }, + "cloudTombstone": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v26-cloud-tombstone`): records deletions accepted from the NOOP cloud journal, keyed by (kind, editSeq) for idempotency, so a later re-import cannot resurrect a row the user deleted. Backs Strand/CloudSync/, which has no Android counterpart.", + "columns": [ + { + "name": "id", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "kind", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "startTs", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "endTs", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "sport", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "day", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "key", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "editSeq", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "id" + ], + "indices": [ + { + "name": "idx_cloudTombstone_kind_device", + "unique": false, + "columns": [ + "kind", + "deviceId" + ] + } + ] + }, "cursors": { "platform": "ios_only", "reason": "iOS offload/upload bookmarks (GRDB v2). Android keeps the equivalent bookmarks in SharedPreferences, not in Room.", @@ -322,6 +453,14 @@ "affinity": "INTEGER", "notNull": false, "default": null + }, + { + "name": "avgSdnn", + "affinity": "REAL", + "notNull": false, + "default": null, + "androidAbsent": true, + "divergence": "avg-sdnn-ios-only" } ], "primaryKey": [ @@ -1054,6 +1193,28 @@ "androidColumnOrderDivergence": "paired-device-alter-append-order", "indices": [] }, + "phoneTimezone": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v28-phone-timezone`): the IANA timezone the phone was in for each local calendar day, so day-bucketed metrics stay correct across travel. Android has no twin.", + "columns": [ + { + "name": "day", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "tzId", + "affinity": "TEXT", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "day" + ], + "indices": [] + }, "ppgHrSample": { "platform": "both", "columns": [ @@ -1537,6 +1698,34 @@ ], "indices": [] }, + "spo2PctSample": { + "platform": "both", + "columns": [ + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "ts", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "pct", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "deviceId", + "ts" + ], + "indices": [] + }, "spo2Sample": { "platform": "both", "columns": [ diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/SchemaOracleTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/SchemaOracleTests.swift index 5cba234060..30853408ae 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/SchemaOracleTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/SchemaOracleTests.swift @@ -32,6 +32,15 @@ final class SchemaOracleTests: XCTestCase { let grdbMigrations: [String] let divergenceReasons: [String: String] let tables: [String: OracleTable] + /// Absent upstream, where there is only the baseline lineage. See `MigrationLineage`. + let grdbMigrationLineages: [String: MigrationLineage]? + var lineages: [String: MigrationLineage] { grdbMigrationLineages ?? [:] } + } + /// A set of migration identifiers numbered independently of the baseline lineage — a fork that + /// reached `vN` before upstream did. Declaring one is what lets the two coexist without a renumber. + struct MigrationLineage: Decodable { + let reason: String + let migrations: [String] } struct OracleTable: Decodable { let platform: String // "both" | "ios_only" | "android_only" @@ -201,23 +210,58 @@ final class SchemaOracleTests: XCTestCase { /// entirely, because the first already recorded that name in `grdb_migrations`. /// /// So: identifiers unique, and their `vN` prefixes exactly 1...N with no gaps and no repeats. + /// + /// The one sanctioned exception is a DECLARED lineage. A fork that numbered its own migrations + /// before upstream reached those numbers cannot renumber its way out — GRDB keys by identifier + /// string, so renaming one makes it read as un-applied on a device that already ran it and the + /// migrator re-runs its body against a schema that already has the table. Such a fork lists its + /// identifiers under `grdbMigrationLineages`; everything NOT listed is the baseline lineage and is + /// still held to exactly v1...vN. So the escape hatch costs a written-down reason and cannot be + /// taken by accident — an undeclared collision still fails, which is what catches #369 vs #475. func testGrdbMigrationIdentifiersAreUniqueAndSequential() throws { + let oracle = try loadOracle() let ids = WhoopStore.makeMigrator().migrations XCTAssertEqual(Set(ids).count, ids.count, "duplicate GRDB migration identifier — GRDB would silently SKIP the second body") - var numbers: [Int] = [] - for id in ids { - let digits = id.dropFirst().prefix { $0.isNumber } - guard id.hasPrefix("v"), let n = Int(digits) else { - return XCTFail("migration identifier '\(id)' is not of the form v[-slug]") + func version(of id: String) -> Int? { + guard id.hasPrefix("v"), let n = Int(id.dropFirst().prefix(while: \.isNumber)) else { return nil } + return n + } + for id in ids where version(of: id) == nil { + return XCTFail("migration identifier '\(id)' is not of the form v[-slug]") + } + + // A declared lineage must be true (every member registered) and self-consistent (its own vN + // strictly increasing), so the ledger can only shrink deliberately — same rule the divergence + // reasons live by. + var claimedBy: [String: String] = [:] + for (name, lineage) in oracle.lineages.sorted(by: { $0.key < $1.key }) { + XCTAssertFalse(lineage.migrations.isEmpty, "lineage '\(name)' declares no migrations — delete it") + for id in lineage.migrations { + if let other = claimedBy.updateValue(name, forKey: id) { + return XCTFail("'\(id)' is claimed by both lineage '\(other)' and '\(name)'") + } + guard ids.contains(id) else { + return XCTFail("lineage '\(name)' declares '\(id)', which is not registered — a lineage " + + "entry that stopped being true must be deleted, not left to rot") + } + } + let numbers = lineage.migrations.compactMap(version(of:)) + for (a, b) in zip(numbers, numbers.dropFirst()) where b <= a { + return XCTFail("lineage '\(name)' goes v\(a) then v\(b) — a lineage numbers its own " + + "migrations, so its vN must strictly increase") } - numbers.append(n) } - for (offset, n) in numbers.enumerated() where n != offset + 1 { - return XCTFail("GRDB migration '\(ids[offset])' claims v\(n) but is #\(offset + 1) in " - + "registration order — two migrations claiming the same vN, or a gap, makes the " - + "GRDB-name <-> Room-version mapping ambiguous. Renumber before merging.") + + let baseline = ids.filter { claimedBy[$0] == nil } + for (offset, id) in baseline.enumerated() where version(of: id) != offset + 1 { + return XCTFail("GRDB migration '\(id)' claims v\(version(of: id)!) but is #\(offset + 1) in the " + + "BASELINE lineage (registration order minus every lineage schema_oracle.json " + + "declares) — two migrations claiming the same vN, or a gap, makes the " + + "GRDB-name <-> Room-version mapping ambiguous. Renumber before merging, or, if " + + "the identifier already shipped and renaming it would wedge a live database, " + + "declare it in grdbMigrationLineages with a reason.") } } diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/Spo2PctDurableTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/Spo2PctDurableTests.swift new file mode 100644 index 0000000000..52c75de8e9 --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/Spo2PctDurableTests.swift @@ -0,0 +1,256 @@ +import XCTest +import GRDB +import WhoopProtocol +@testable import WhoopStore + +/// v34: the durable `@82` SpO2 percentage stream. +/// +/// The 5/MG emits its own computed SpO2 % on `@82` of the v18 record. That byte has been banked since v31 +/// — inside `v18AuxSample.fields`, a table CAPPED at 604,800 rows/device (~7 days at 1 Hz) because the aux +/// blob costs ~30 MB/day. So every reading rolled off after a week and was gone permanently: the strap +/// trims its own history the moment an offload is acked, leaving no second copy anywhere. +/// +/// `spo2PctSample` is the narrow, never-pruned sibling that fixes that. ~2,000 samples/week — ~100k +/// rows/year — is nothing to keep forever. These tests pin the two properties the fix rests on: the right +/// values get in (and only those), and the aux sweep cannot touch them. +final class Spo2PctDurableTests: XCTestCase { + + private func store() async throws -> WhoopStore { + let s = try await WhoopStore.inMemory() + try await s.upsertDevice(id: "dev1", mac: nil, name: nil) + return s + } + + /// One v18 aux sample carrying `@82` = `v` at `ts`, and nothing else. + private func aux(_ ts: Int, _ v: Int?) -> V18AuxSample { V18AuxSample(ts: ts, auxByte82: v) } + + // MARK: - Migration shape + + func testV34CreatesTheDurableTable() async throws { + let s = try await store() + let tables = try await s.tableNames() + XCTAssertTrue(tables.contains("spo2PctSample")) + let pk = try await s.primaryKeyColumns("spo2PctSample") + let cols = try await s.columnNamesForTest(table: "spo2PctSample") + XCTAssertEqual(pk, ["deviceId", "ts"]) + XCTAssertEqual(Set(cols), ["deviceId", "ts", "pct"]) + } + + /// The v3 `spo2Sample` table is NOT the home for this and must be left exactly as it was: its + /// `red`/`ir` are NOT NULL raw ADC channels off the WHOOP 4.0 v24 layout, and + /// `AnalyticsEngine.nightlySpo2RawMeans` averages them into a live tile. Storing percentages there + /// would have meant fabricating red/ir zeros on every insert and dragging that mean toward zero. + func testTheWhoop4RawTableIsUntouched() async throws { + let s = try await store() + let cols = try await s.columnNamesForTest(table: "spo2Sample") + // Its own columns, unchanged (`synced` is the outbox flag a later migration added). + XCTAssertEqual(Set(cols), ["deviceId", "ts", "red", "ir", "synced"]) + // And emphatically no percentage column: the two quantities never share a table. + XCTAssertFalse(cols.contains("pct")) + } + + // MARK: - In-band values persist + + func testInBandValuesPersistVerbatim() async throws { + let s = try await store() + _ = try await s.insert(Streams(v18Aux: [aux(100, 97), aux(101, 94), aux(102, 88)]), + deviceId: "dev1") + let rows = try await s.spo2PctSamples(deviceId: "dev1", from: 0, to: 1_000) + XCTAssertEqual(rows.map(\.ts), [100, 101, 102]) + XCTAssertEqual(rows.map(\.pct), [97, 94, 88]) + } + + /// The band is INCLUSIVE at both ends — 70 and 100 are real readings, and a gate that quietly dropped + /// the endpoints would bias every night's median upward at exactly the readings that matter most. + func testBandBoundariesAreInclusive() async throws { + let s = try await store() + _ = try await s.insert(Streams(v18Aux: [aux(200, 70), aux(201, 100)]), deviceId: "dev1") + let rows = try await s.spo2PctSamples(deviceId: "dev1", from: 0, to: 1_000) + XCTAssertEqual(rows.map(\.pct), [70, 100]) + } + + // MARK: - Out-of-band values are rejected + + /// `@82` is MULTIPLEXED: the same byte carries measurements, bit-7 status sentinels, sub-70 diagnostic + /// codes, and 0 for "not emitted". Only `70...100` is a percentage of anything. A sentinel banked as a + /// percentage would not merely be wrong, it would be *plausible* — 0x88 is 136, which no reader would + /// flag, and 0x08 is 8, which reads as catastrophic hypoxia. Rejection has to happen at the boundary. + func testStatusSentinelsAndDiagnosticCodesAreRejected() async throws { + let s = try await store() + let rejected = [0x00, 0x08, 0x45, 0x80, 0x88, 0x90, 0xA0, 0xA8, 0xFF] + let samples = rejected.enumerated().map { aux(300 + $0.offset, $0.element) } + _ = try await s.insert(Streams(v18Aux: samples), deviceId: "dev1") + let banked = try await s.spo2PctCountForTest() + let auxRows = try await s.v18AuxCountForTest() + XCTAssertEqual(banked, 0, "no sentinel or diagnostic code may be banked as a percentage") + // The raw bytes are still captured in the aux table — this gate demultiplexes, it does not discard. + XCTAssertEqual(auxRows, rejected.count) + } + + /// `101...127` is the EMPTY BAND — nothing has ever been observed there across 626,725 production + /// records. It is what makes measurements and bit-7 sentinels separable by value alone, with no side + /// channel. A value appearing there would mean the demultiplex assumption has broken, so it must not + /// be admitted on the strength of merely being "close to 100". + func testTheEmptyBandAbove100IsRejected() async throws { + let s = try await store() + let samples = (101...127).enumerated().map { aux(400 + $0.offset, $0.element) } + _ = try await s.insert(Streams(v18Aux: samples), deviceId: "dev1") + let banked = try await s.spo2PctCountForTest() + XCTAssertEqual(banked, 0) + } + + /// An aux record with no `@82` slot at all banks no percentage — absence stays absence. + func testAbsentSlotBanksNothing() async throws { + let s = try await store() + _ = try await s.insert(Streams(v18Aux: [V18AuxSample(ts: 500, statusWord: 7)]), deviceId: "dev1") + let banked = try await s.spo2PctCountForTest() + let auxRows = try await s.v18AuxCountForTest() + XCTAssertEqual(banked, 0) + XCTAssertEqual(auxRows, 1) + } + + // MARK: - Idempotence + + /// A re-offload of a window already banked must not duplicate or mutate it. ON CONFLICT DO NOTHING + /// keeps the FIRST value for a second, matching every other per-second stream's dedupe rule. + func testReInsertIsIdempotentAndKeepsTheFirstValue() async throws { + let s = try await store() + _ = try await s.insert(Streams(v18Aux: [aux(600, 96), aux(601, 95)]), deviceId: "dev1") + _ = try await s.insert(Streams(v18Aux: [aux(600, 96), aux(601, 95)]), deviceId: "dev1") + // Same seconds, different values — the first-seen reading survives. + _ = try await s.insert(Streams(v18Aux: [aux(600, 71), aux(601, 99)]), deviceId: "dev1") + let rows = try await s.spo2PctSamples(deviceId: "dev1", from: 0, to: 1_000) + XCTAssertEqual(rows.map(\.ts), [600, 601]) + XCTAssertEqual(rows.map(\.pct), [96, 95]) + } + + /// Two straps' readings for the same second are different rows — the PK is (deviceId, ts). + func testPerDeviceScoping() async throws { + let s = try await store() + try await s.upsertDevice(id: "dev2", mac: nil, name: nil) + _ = try await s.insert(Streams(v18Aux: [aux(700, 97)]), deviceId: "dev1") + _ = try await s.insert(Streams(v18Aux: [aux(700, 92)]), deviceId: "dev2") + let d1 = try await s.spo2PctSamples(deviceId: "dev1", from: 0, to: 1_000) + let d2 = try await s.spo2PctSamples(deviceId: "dev2", from: 0, to: 1_000) + XCTAssertEqual(d1.map(\.pct), [97]) + XCTAssertEqual(d2.map(\.pct), [92]) + } + + // MARK: - Retention: the whole point of the table + + /// THE LOAD-BEARING TEST. The aux sweep must not reach this table. Retention is forced to 1 row and + /// the prune to every row, so the aux capture is cut to its newest row — while every percentage + /// survives. This is the entire reason v34 exists: before it, seven-day-old SpO2 was destroyed with + /// no recoverable copy. + func testSpo2SurvivesTheAuxPrune() async throws { + let s = try await store() + let samples = (0..<40).map { aux(1_000 + $0, 90 + ($0 % 8)) } + _ = try await s.insert(Streams(v18Aux: samples), deviceId: "dev1", + v18AuxRetentionRows: 1, v18AuxPruneEveryRows: 1) + + // The aux table was swept down to its cap... + let auxRows = try await s.v18AuxCountForTest() + XCTAssertEqual(auxRows, 1) + // ...and not one percentage was lost. + let rows = try await s.spo2PctSamples(deviceId: "dev1", from: 0, to: 10_000) + XCTAssertEqual(rows.count, 40) + XCTAssertEqual(rows.map(\.ts), Array(1_000..<1_040)) + XCTAssertEqual(rows.map(\.pct), (0..<40).map { 90 + ($0 % 8) }) + } + + /// Repeated sweeps across many batches never erode the durable rows. + func testRepeatedPrunesNeverErodeTheDurableRows() async throws { + let s = try await store() + for batch in 0..<5 { + let samples = (0..<10).map { aux(2_000 + batch * 10 + $0, 95) } + _ = try await s.insert(Streams(v18Aux: samples), deviceId: "dev1", + v18AuxRetentionRows: 2, v18AuxPruneEveryRows: 1) + } + let banked = try await s.spo2PctCountForTest() + let auxRows = try await s.v18AuxCountForTest() + XCTAssertEqual(banked, 50) + XCTAssertLessThanOrEqual(auxRows, 12) + } + + // MARK: - Privacy + + /// A never-pruned table makes delete-means-gone MORE important, not less: without this the wipe would + /// leave years of blood-oxygen readings behind after the user asked for them to be erased. + func testDeleteAllDataClearsTheDurableRows() async throws { + let s = try await store() + _ = try await s.insert(Streams(v18Aux: [aux(3_000, 96)]), deviceId: "dev1") + let before = try await s.spo2PctCountForTest() + XCTAssertEqual(before, 1) + XCTAssertTrue(DeviceRegistryStore.deviceScopedTables.contains("spo2PctSample")) + try await s.deleteAllData(deviceId: "dev1") + let after = try await s.spo2PctCountForTest() + XCTAssertEqual(after, 0) + } +} + +/// The blob-offset question, pinned. +/// +/// The cloud reader takes `fields[23]` of the STORED `v18AuxSample.fields` blob and gets physiologically +/// valid SpO2 out of it. That works — but it is an ARITHMETIC COINCIDENCE of a fully-populated record, not +/// a property of the format. `V18AuxCodec` is a presence-bitmap codec: absent slots are omitted from the +/// body entirely, so every slot's byte offset depends on which slots ahead of it were present. +/// +/// These tests establish both halves, because only the pair is decisive. The first alone would license +/// the fragile read; the second is what says the device write path must not use it. +final class V18AuxCodecOffsetTests: XCTestCase { + + /// A record carrying every slot: `auxByte82` lands at blob byte 23. Header is 5 bytes (version + + /// u32 bitmap), then the eleven slots ahead of it occupy 4+1+1+1+1+2+1+1+2+2+2 = 18 bytes, so it sits + /// at 5 + 18 = 23 and the whole blob is 32 bytes. THIS is why the cloud's `fields[23]` reads real + /// percentages: in production every one of those slots is present. + func testFullyPopulatedRecordPutsAuxByte82AtBlobByte23() { + let full = V18AuxSample( + ts: 1, recordIndex: 0x11223344, rrCount: 4, cardiacFlags: 0x21, hrQualityFlags: 0x82, + heartRateAlt: 58, rrPacked: 0x0505, cardiacStatus: 0x31, stepCadence: 0x41, + statusWord: 0x0606, statusWord1: 0x0707, statusWord2: 0x0808, auxByte82: 93, + opticalBaselineA: 0x51, opticalBaselineB: 0x52, opticalAmpA: 0x53, opticalAmpB: 0x54, + unknownF32Bits: 0x55667788) + let blob = [UInt8](V18AuxCodec.pack(full)) + + XCTAssertEqual(blob.count, 32, "5-byte header + 27 body bytes") + XCTAssertEqual(Int(blob[23]), 93, "byte 23 of a fully-populated blob IS @82") + // And it agrees with the decoded struct, which is the only contract that actually holds. + XCTAssertEqual(V18AuxCodec.unpack(Data(blob), ts: 1).auxByte82, 93) + // Derive the offset from the slot table rather than trusting the literal, so a future slot + // inserted ahead of @82 fails HERE with a readable number instead of silently moving the cloud's + // read onto a neighbouring byte. + let ahead = V18AuxSlot.allCases + .filter { $0.rawValue < V18AuxSlot.auxByte82.rawValue } + .reduce(0) { $0 + $1.width } + XCTAssertEqual(V18AuxCodec.headerBytes + ahead, 23) + } + + /// ONE absent slot ahead of it and byte 23 is a different field entirely — while the decoded value is + /// still correct. The blob read does not degrade loudly; it returns a neighbouring byte that can look + /// exactly like a plausible saturation. This is why `StreamStore` reads `s.auxByte82` off the decoded + /// struct and never indexes the blob. + func testOneAbsentEarlierSlotMovesAuxByte82OffByte23() { + // Identical to the record above except `recordIndex` (4 bytes, slot 0) is absent. + let partial = V18AuxSample( + ts: 1, rrCount: 4, cardiacFlags: 0x21, hrQualityFlags: 0x82, + heartRateAlt: 58, rrPacked: 0x0505, cardiacStatus: 0x31, stepCadence: 0x41, + statusWord: 0x0606, statusWord1: 0x0707, statusWord2: 0x0808, auxByte82: 93, + opticalBaselineA: 0x51, opticalBaselineB: 0x52, opticalAmpA: 0x53, opticalAmpB: 0x54, + unknownF32Bits: 0x55667788) + let blob = [UInt8](V18AuxCodec.pack(partial)) + + XCTAssertEqual(blob.count, 28, "four fewer body bytes without recordIndex") + // @82 has slid four bytes earlier... + XCTAssertEqual(Int(blob[19]), 93) + // ...and byte 23 now holds opticalAmpB, four slots further along. Reading it as a percentage + // would yield 0x54 = 84: in band, unremarkable, and completely fabricated. Note it does NOT + // fail loudly — an optical amplitude byte and a saturation percentage occupy the same numeric + // range, so the bad read is indistinguishable from a good one at the call site. + XCTAssertEqual(Int(blob[23]), 0x54) + XCTAssertNotEqual(Int(blob[23]), 93) + XCTAssertTrue(spo2CandidateInBand.contains(Int(blob[23])), + "the wrong byte is still 'in band' — which is exactly why this read is unsafe") + // The decoded read is unaffected — it is driven by the bitmap, not by an offset. + XCTAssertEqual(V18AuxCodec.unpack(Data(blob), ts: 1).auxByte82, 93) + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/StoreReplicationTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/StoreReplicationTests.swift new file mode 100644 index 0000000000..47e7c3552f --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/StoreReplicationTests.swift @@ -0,0 +1,183 @@ +import XCTest +import GRDB +@testable import WhoopStore + +/// `StoreReplication` is the process-wide default behind `WhoopStore(path:)`'s checkpointing +/// parameters. It exists because `wal_autocheckpoint` is per-connection and this app opens the same +/// database from two independent places (`Repository.ensureStore()` and `BLEManager.bootstrapStore()`), +/// so a single opener that does not select `.external` silently defeats it for every other opener — +/// and the only symptom is that the replicator's uploads never get smaller. +/// +/// These tests assert the *behaviour* — `PRAGMA wal_autocheckpoint` as observed on real pooled +/// connections — not that a stored enum echoes its argument back. +/// +/// Every test resets the global in `tearDown`. `WalCheckpointingTests.testDefaultLeaves…` asserts the +/// stock 1000-page autocheckpoint on a bare `WhoopStore(path:)`, and XCTest runs the whole bundle in +/// one process, so a leaked `.external` here would fail that test instead of this one. +final class StoreReplicationTests: XCTestCase { + + /// Reset on BOTH sides. `tearDown` alone is not enough: `openedStoreCount` is process-wide, and + /// the ~90 other test classes in this bundle open real stores between these tests, so a count + /// assertion that only cleaned up after itself passed in isolation and failed in the full run. + override func setUp() { + super.setUp() + StoreReplication.resetForTesting() + } + + override func tearDown() { + StoreReplication.resetForTesting() + super.tearDown() + } + + private func tempPath() -> String { + NSTemporaryDirectory() + "whoopstore-storerep-\(UUID().uuidString).sqlite" + } + + private func removeDB(_ path: String) { + for suffix in ["", "-wal", "-shm"] { + try? FileManager.default.removeItem(atPath: path + suffix) + } + } + + private func autocheckpoint(_ store: WhoopStore, reader: Bool) async throws -> Int { + let sql = "PRAGMA wal_autocheckpoint" + return reader + ? try await store.registryWriter.read { try Int.fetchOne($0, sql: sql) ?? -1 } + : try await store.registryWriter.write { try Int.fetchOne($0, sql: sql) ?? -1 } + } + + // MARK: - Defaults + + /// An unconfigured process is upstream's process. This is the property that lets this file live + /// in an upstream-shared package at all. + func testUnconfiguredDefaultsAreUpstreamBehaviour() { + StoreReplication.resetForTesting() + if case .external = StoreReplication.walCheckpointing { + XCTFail("default checkpointing must be .automatic") + } + XCTAssertEqual(StoreReplication.walBackstop, .standard) + XCTAssertFalse(StoreReplication.configuredAfterFirstOpen) + } + + // MARK: - The policy reaches a store that passes no arguments + + /// The whole point: an opener written with no knowledge of replication — `WhoopStore(path:)`, + /// which is literally what both real openers call — still gets `.external`. + func testConfiguredExternalReachesAnOpenerThatPassesNoArguments() async throws { + let path = tempPath() + defer { removeDB(path) } + StoreReplication.configure(walCheckpointing: .external) + + let store = try await WhoopStore(path: path) + + let writer = try await autocheckpoint(store, reader: false) + let reader = try await autocheckpoint(store, reader: true) + XCTAssertEqual(writer, 0, "policy must disable autocheckpoint on the writer connection") + XCTAssertEqual(reader, 0, "policy must disable autocheckpoint on reader connections too") + if case .automatic = store.walCheckpointing { + XCTFail("store should report the mode it actually opened with") + } + } + + /// Two independently-opened stores on the SAME file both get the policy. This is the multi-opener + /// case that the argument-at-each-call-site approach cannot guarantee: a second pool still + /// auto-checkpointing at ~4 MB restarts the WAL under the replicator and forces a full snapshot. + func testEverySeparateOpenerOfTheSameFileGetsThePolicy() async throws { + let path = tempPath() + defer { removeDB(path) } + StoreReplication.configure(walCheckpointing: .external) + + let first = try await WhoopStore(path: path) + let second = try await WhoopStore(path: path) + + for (name, store) in [("first", first), ("second", second)] { + let writer = try await autocheckpoint(store, reader: false) + let reader = try await autocheckpoint(store, reader: true) + XCTAssertEqual(writer, 0, "\(name) opener's writer must have autocheckpoint off") + XCTAssertEqual(reader, 0, "\(name) opener's readers must have autocheckpoint off") + } + XCTAssertEqual(StoreReplication.openedStoreCount, 2) + } + + /// The backstop that discharges `.external`'s obligation must be installed when the mode came + /// from the policy, not only when it was passed explicitly. Without this, a policy-configured + /// store would have autocheckpoint off and nothing at all bounding WAL growth. + func testPolicySelectedExternalStillInstallsTheWalBackstop() async throws { + let path = tempPath() + defer { removeDB(path) } + StoreReplication.configure(walCheckpointing: .external, walBackstop: .standard) + + let store = try await WhoopStore(path: path) + + XCTAssertNotNil(store.walBackstopMonitor, + "policy-selected .external must install a backstop, or nothing bounds the WAL") + } + + /// A caller that opts out of the backstop through the policy gets no monitor — the same escape + /// hatch the explicit argument has, and the same assertion it carries. + func testPolicyCanDisableTheBackstop() async throws { + let path = tempPath() + defer { removeDB(path) } + StoreReplication.configure(walCheckpointing: .external, walBackstop: .disabled) + + let store = try await WhoopStore(path: path) + + XCTAssertNil(store.walBackstopMonitor) + } + + // MARK: - Explicit arguments still win + + /// The policy is a default, not an override. Tests (and any future caller with a specific need) + /// keep full control. + func testExplicitArgumentOverridesThePolicy() async throws { + let path = tempPath() + defer { removeDB(path) } + StoreReplication.configure(walCheckpointing: .external) + + let store = try await WhoopStore(path: path, walCheckpointing: .automatic) + + let writer = try await autocheckpoint(store, reader: false) + let reader = try await autocheckpoint(store, reader: true) + XCTAssertEqual(writer, 1000) + XCTAssertEqual(reader, 1000) + } + + // MARK: - Late configuration is reported, not silent + + /// Configuring after a store is already open leaves that store's pool auto-checkpointing. That is + /// a genuine half-configured process, and the failure it produces (a replicator that snapshots + /// forever) is otherwise indistinguishable from the replicator being bad. It must be observable. + func testConfiguringAfterAStoreIsOpenIsFlagged() async throws { + let path = tempPath() + defer { removeDB(path) } + + let early = try await WhoopStore(path: path) + XCTAssertFalse(StoreReplication.configuredAfterFirstOpen) + + StoreReplication.configure(walCheckpointing: .external) + + XCTAssertTrue(StoreReplication.configuredAfterFirstOpen, + "a late configure must be visible; it leaves live pools on the old mode") + // And the claim is literally true of the already-open store: it kept SQLite's autocheckpoint. + let earlyPragma = try await autocheckpoint(early, reader: false) + XCTAssertEqual(earlyPragma, 1000) + + // Stores opened afterwards do get the new policy. + let path2 = tempPath() + defer { removeDB(path2) } + let late = try await WhoopStore(path: path2) + let latePragma = try await autocheckpoint(late, reader: false) + XCTAssertEqual(latePragma, 0) + } + + /// A failed open must not count: `configure` landing between a throwing open and a successful + /// retry is on time, not late. `quarantineIncompatibleDatabase` + the migrator make a throwing + /// open a real thing on this path (#222, #261). + func testAFailedOpenDoesNotCountAsAnOpenStore() async { + let dir = NSTemporaryDirectory() + "storerep-nodir-\(UUID().uuidString)/nested" + _ = try? await WhoopStore(path: dir + "/whoop.sqlite") + + XCTAssertEqual(StoreReplication.openedStoreCount, 0, + "an open that threw must not be counted, or a legitimate configure reads as late") + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/WalBackstopTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/WalBackstopTests.swift new file mode 100644 index 0000000000..4545a7114d --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/WalBackstopTests.swift @@ -0,0 +1,231 @@ +import XCTest +import GRDB +import WhoopProtocol +@testable import WhoopStore + +/// `WalCheckpointing.external` turns SQLite's autocheckpoint off, which leaves **nothing** bounding +/// WAL growth: the external replicator's own thresholds live inside its push loop, so they do not +/// fire when it is asleep, crashed, or never started. The backstop is what makes that safe. +/// +/// Two halves are tested separately on purpose: +/// +/// * `WalBackstopCore` is pure — threshold, in-flight suppression and retry backoff are asserted +/// with no database and no clock, so those rules cannot regress silently behind a slow test. +/// * The integration tests then prove the rule is actually wired into the write path: a real +/// `DatabasePool`, real commits, and a real `-wal` file that has to come back down. +final class WalBackstopTests: XCTestCase { + + private func tempPath() -> String { + NSTemporaryDirectory() + "whoopstore-backstop-\(UUID().uuidString).sqlite" + } + + private func removeDB(_ path: String) { + for suffix in ["", "-wal", "-shm"] { + try? FileManager.default.removeItem(atPath: path + suffix) + } + } + + private func walBytes(_ path: String) -> Int64 { + let attrs = try? FileManager.default.attributesOfItem(atPath: path + "-wal") + return (attrs?[.size] as? NSNumber)?.int64Value ?? 0 + } + + /// One transaction per call, mirroring NOOP's real commit shape (a `Collector` flush). + @discardableResult + private func writeTransactions(_ store: WhoopStore, count: Int, rowsEach: Int = 400, + startingAt ts0: Int = 0) async throws -> Int { + var ts = ts0 + for _ in 0.. String { + NSTemporaryDirectory() + "whoopstore-walcp-\(UUID().uuidString).sqlite" + } + + private func removeDB(_ path: String) { + for suffix in ["", "-wal", "-shm"] { + try? FileManager.default.removeItem(atPath: path + suffix) + } + } + + /// Reads `PRAGMA wal_autocheckpoint` on a WRITER connection. + private func autocheckpointOnWriter(_ store: WhoopStore) async throws -> Int { + try await store.registryWriter.write { db in + try Int.fetchOne(db, sql: "PRAGMA wal_autocheckpoint") ?? -1 + } + } + + /// Reads `PRAGMA wal_autocheckpoint` on a READER connection — a `DatabasePool` serves `.read` + /// from a separate connection than `.write`, which is exactly where the leak would hide. + private func autocheckpointOnReader(_ store: WhoopStore) async throws -> Int { + try await store.registryWriter.read { db in + try Int.fetchOne(db, sql: "PRAGMA wal_autocheckpoint") ?? -1 + } + } + + // MARK: - Default behaviour is unchanged + + /// The default must be indistinguishable from the pre-change store: SQLite's stock 1000-page + /// autocheckpoint, on both connection kinds. Upstream and Android depend on this. + func testDefaultLeavesAutocheckpointAtSQLiteDefault() async throws { + let path = tempPath() + defer { removeDB(path) } + + let store = try await WhoopStore(path: path) + let writer = try await autocheckpointOnWriter(store) + let reader = try await autocheckpointOnReader(store) + + XCTAssertEqual(writer, 1000, "default store must keep SQLite's stock autocheckpoint (writer)") + XCTAssertEqual(reader, 1000, "default store must keep SQLite's stock autocheckpoint (reader)") + } + + /// Passing `.automatic` explicitly is the same as passing nothing. + func testExplicitAutomaticMatchesDefault() async throws { + let path = tempPath() + defer { removeDB(path) } + + let store = try await WhoopStore(path: path, walCheckpointing: .automatic) + let value = try await autocheckpointOnWriter(store) + XCTAssertEqual(value, 1000) + } + + // MARK: - Opt-in disables it everywhere + + /// THE CORE ASSERTION: `.external` must reach every connection the pool opens, not just the + /// writer. A reader that still auto-checkpoints restarts the WAL and forces the replicator into + /// a full re-upload — the precise failure this option exists to prevent. + func testExternalDisablesAutocheckpointOnWriterAndReader() async throws { + let path = tempPath() + defer { removeDB(path) } + + let store = try await WhoopStore(path: path, walCheckpointing: .external) + let writer = try await autocheckpointOnWriter(store) + let reader = try await autocheckpointOnReader(store) + + XCTAssertEqual(writer, 0, "external checkpointing must disable autocheckpoint on the writer") + XCTAssertEqual(reader, 0, "external checkpointing must ALSO disable it on reader connections") + } + + /// Several concurrent readers force the pool to open additional connections; every one of them + /// must carry the pragma, since `prepareDatabase` — not a one-shot statement at open — is what + /// applies it. + func testExternalAppliesToEveryPooledReaderConnection() async throws { + let path = tempPath() + defer { removeDB(path) } + + let store = try await WhoopStore(path: path, walCheckpointing: .external) + // Seed a row so reads have something to do. + _ = try await store.insert(Streams(hr: [HRSample(ts: 1, bpm: 60)]), deviceId: "dev") + + let values = try await withThrowingTaskGroup(of: Int.self) { group -> [Int] in + for _ in 0..<8 { + group.addTask { + try await store.registryWriter.read { db in + try Int.fetchOne(db, sql: "PRAGMA wal_autocheckpoint") ?? -1 + } + } + } + var out: [Int] = [] + for try await v in group { out.append(v) } + return out + } + + XCTAssertEqual(values.count, 8) + XCTAssertTrue(values.allSatisfy { $0 == 0 }, + "every pooled reader connection must have autocheckpoint disabled, got \(values)") + } + + // MARK: - Behaviour, not just the pragma value + + /// The observable consequence, and the reason `.external` is dangerous without a checkpointer: + /// under `.automatic` the WAL plateaus near SQLite's ~4 MB (1000-page) threshold, because each + /// crossing checkpoints and rewinds the WAL to the start and the file is then REUSED in place. + /// Under `.external` nothing rewinds it, so it grows monotonically with the number of commits. + /// + /// The writes must therefore comfortably exceed 4 MB of WAL or neither store checkpoints and the + /// two are trivially identical. + func testExternalLetsWalGrowPastTheAutocheckpointThreshold() async throws { + let autoPath = tempPath() + let extPath = tempPath() + defer { removeDB(autoPath); removeDB(extPath) } + + let transactions = 250 + let rowsPerTransaction = 500 + + // NOOP's real commit shape: one transaction per Collector flush. + func writeManySmallTransactions(_ store: WhoopStore) async throws { + var ts = 0 + for _ in 0.. Int64 { + let attrs = try? FileManager.default.attributesOfItem(atPath: path + "-wal") + return (attrs?[.size] as? NSNumber)?.int64Value ?? 0 + } + + let autoStore = try await WhoopStore(path: autoPath, walCheckpointing: .automatic) + try await writeManySmallTransactions(autoStore) + + let extStore = try await WhoopStore(path: extPath, walCheckpointing: .external) + try await writeManySmallTransactions(extStore) + + let autoWal = walBytes(autoPath) + let extWal = walBytes(extPath) + + XCTAssertGreaterThan( + extWal, autoWal * 2, + "with autocheckpoint off the WAL must grow far past the self-checkpointing one " + + "(external=\(extWal) bytes, automatic=\(autoWal) bytes)") + // And the store still works: nothing about disabling checkpointing may change stored data. + let counts = try await extStore.storageStats_rowCountsForTest() + XCTAssertEqual(counts.hr, transactions * rowsPerTransaction, + "disabling autocheckpoint must not change what is stored") + } + + /// An explicit `checkpointWAL()` must still reclaim the WAL when autocheckpoint is off — this is + /// the safety valve an `.external` caller needs when its replicator is not running. + func testExplicitCheckpointStillWorksWithExternalMode() async throws { + let path = tempPath() + defer { removeDB(path) } + + let store = try await WhoopStore(path: path, walCheckpointing: .external) + var ts = 0 + for _ in 0..<40 { + var hr: [HRSample] = [] + for _ in 0..<200 { ts += 1; hr.append(HRSample(ts: ts, bpm: 60 + ts % 40)) } + _ = try await store.insert(Streams(hr: hr), deviceId: "dev") + } + + func walBytes() -> Int64 { + let attrs = try? FileManager.default.attributesOfItem(atPath: path + "-wal") + return (attrs?[.size] as? NSNumber)?.int64Value ?? 0 + } + let grown = walBytes() + XCTAssertGreaterThan(grown, 0, "WAL should have accumulated with autocheckpoint off") + + try await store.checkpointWAL() + XCTAssertLessThan(walBytes(), grown, "explicit checkpointWAL() must still truncate the WAL") + } +} diff --git a/README.md b/README.md index 5f48e66210..d5c5b611ab 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,27 @@ +> ### About this fork +> +> This is **Vishnu Kchitti's** working fork of [NOOP](https://github.com/ryanbr/noop), +> the offline WHOOP companion app by [@ryanbr](https://github.com/ryanbr) and contributors. +> The base app is their work; this fork is where my contributions to it are developed. +> +> **What I've contributed upstream:** 13 pull requests merged into `ryanbr/noop` +> ([see the merged PRs](https://github.com/ryanbr/noop/pulls?q=is%3Apr+author%3Avishk23+is%3Amerged)), +> concentrated in the sensor-protocol and analytics layers: +> +> - **WHOOP 5.0/MG raw sensor decoding** — 6-axis IMU offload buffer + activity features (#455), +> high-rate R22 deep buffers (#454), and persisting the v26 raw PPG waveform (#415) +> - **Sleep scoring** — motion-corroborated wake detection so elevated HR on a motionless +> wrist no longer scores WAKE (#465, #402) +> - **Analytics** — Recovery Index + Activity Balance terms in Charge (#417), an opt-in +> coarse workout-type classifier (#414) +> - **Import correctness** — Oura readiness/efficiency unit fixes across Swift + Kotlin +> +> This fork's `main` tracks upstream and carries my additional work in progress; see +> [branches](https://github.com/vishk23/noop/branches) for active experiments. +> The upstream README follows unchanged. + +--- +

NOOP

diff --git a/Rust/Cargo.lock b/Rust/Cargo.lock new file mode 100644 index 0000000000..b5b375bf90 --- /dev/null +++ b/Rust/Cargo.lock @@ -0,0 +1,995 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "askama" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf825125edd887a019d0a3a837dcc5499a68b0d034cc3eb594070c3e18addc" +dependencies = [ + "askama_macros", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c7065972a130eafa84215f21352ae15b4a7393da48c1f5e103904490736738" +dependencies = [ + "askama_parser", + "basic-toml", + "glob", + "memchr", + "proc-macro2", + "quote", + "rustc-hash", + "serde", + "serde_derive", + "syn 2.0.119", +] + +[[package]] +name = "askama_macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e23b1d2c4bd39a41971f6124cef4cc6fd0540913ecb90919b69ab3bbe44ae1a" +dependencies = [ + "askama_derive", +] + +[[package]] +name = "askama_parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7db09fde9143e7ac4513358fb32ee32847125b63b18ea715afd487956da715da" +dependencies = [ + "rustc-hash", + "serde", + "serde_derive", + "unicode-ident", + "winnow", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "fs-err" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "goblin" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b363a30c165f666402fe6a3024d3bec7ebc898f96a4a23bd1c99f8dbf3f4f47" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "liters" +version = "0.1.0" +source = "git+https://github.com/vishk23/liters-mobile.git?branch=main#6835f9f9f6fcfb2005005857981f4d53a522db93" +dependencies = [ + "libc", + "liters-storage", + "liters-wal", + "ltx", + "rand", + "rusqlite", + "thiserror", +] + +[[package]] +name = "liters-ffi" +version = "0.1.0" +source = "git+https://github.com/vishk23/liters-mobile.git?branch=main#6835f9f9f6fcfb2005005857981f4d53a522db93" +dependencies = [ + "liters", + "thiserror", + "uniffi", +] + +[[package]] +name = "liters-storage" +version = "0.1.0" +source = "git+https://github.com/vishk23/liters-mobile.git?branch=main#6835f9f9f6fcfb2005005857981f4d53a522db93" +dependencies = [ + "libc", + "ltx", + "thiserror", +] + +[[package]] +name = "liters-wal" +version = "0.1.0" +source = "git+https://github.com/vishk23/liters-mobile.git?branch=main#6835f9f9f6fcfb2005005857981f4d53a522db93" +dependencies = [ + "thiserror", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "ltx" +version = "0.1.0" +source = "git+https://github.com/vishk23/liters-mobile.git?branch=main#6835f9f9f6fcfb2005005857981f4d53a522db93" +dependencies = [ + "crc", + "lz4_flex", + "thiserror", + "twox-hash", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "noop-liters" +version = "0.1.0" +dependencies = [ + "liters-ffi", + "uniffi", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scroll" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uniffi" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a782a48d72cfd7a2d65cfc7c691dbf5375c43104b3c195f7eccc716dcc3540c8" +dependencies = [ + "anyhow", + "camino", + "cargo_metadata", + "clap", + "uniffi_bindgen", + "uniffi_core", + "uniffi_macros", + "uniffi_pipeline", +] + +[[package]] +name = "uniffi_bindgen" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533b0312c73e3b54eb78a4b257ceae390962dd4767995778309a74644643f9ac" +dependencies = [ + "anyhow", + "askama", + "camino", + "cargo_metadata", + "fs-err", + "glob", + "goblin", + "heck", + "indexmap", + "once_cell", + "serde", + "tempfile", + "textwrap", + "toml", + "uniffi_internal_macros", + "uniffi_meta", + "uniffi_pipeline", + "uniffi_udl", +] + +[[package]] +name = "uniffi_core" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e32e261c5b0dfaba6488f536e71957dddd6b1a498ac7eb791bee56b60a086be" +dependencies = [ + "anyhow", + "bytes", + "once_cell", + "static_assertions", +] + +[[package]] +name = "uniffi_internal_macros" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84ae78069a5e6772ef694fd5bdb628532c88d2c2f0e7142bf6a384636eadb1af" +dependencies = [ + "anyhow", + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "uniffi_macros" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330be6770532e86320df31f54c70bb0be67588594e8e77fa56e9083a3fed5d0d" +dependencies = [ + "camino", + "fs-err", + "once_cell", + "proc-macro2", + "quote", + "serde", + "syn 2.0.119", + "toml", + "uniffi_meta", +] + +[[package]] +name = "uniffi_meta" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78de021f5547e56ab16c665a49d67d4fd3d31e77422f7739a2e9359d328cd9e7" +dependencies = [ + "anyhow", + "siphasher", + "uniffi_internal_macros", + "uniffi_pipeline", +] + +[[package]] +name = "uniffi_pipeline" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f8201bb1907ed8a42d80e11cbc25c8a033e7a31c3cff1d911f56eedb81d4948" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "tempfile", + "uniffi_internal_macros", +] + +[[package]] +name = "uniffi_udl" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e57996bc58009cc29bf04845d627ae313c2547b87171c1c349d6c51a1656c0" +dependencies = [ + "anyhow", + "textwrap", + "uniffi_meta", + "weedle2", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "weedle2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998d2c24ec099a87daf9467808859f9d82b61f1d9c9701251aea037f514eae0e" +dependencies = [ + "nom", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Rust/Cargo.toml b/Rust/Cargo.toml new file mode 100644 index 0000000000..cbf064651a --- /dev/null +++ b/Rust/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "noop-liters" +version = "0.1.0" +edition = "2021" +publish = false +description = "NOOP's build surface for liters: pins the liters-ffi git dependency and builds it against the same SQLite GRDB links." + +[lib] +name = "noop_liters" +# `staticlib` is what the iOS xcframework wraps. `cdylib` exists so +# `uniffi-bindgen --library` has a host-native library to read the embedded +# interface metadata out of; `lib` so the bindgen bin can depend on this crate. +crate-type = ["lib", "staticlib", "cdylib"] + +[[bin]] +name = "uniffi-bindgen" +path = "src/bin/uniffi-bindgen.rs" + +[dependencies] +# liters — Litestream v0.5-compatible SQLite replication, by Kurt Mackey. +# Upstream is https://github.com/mrkurt/liters; this points at the standalone +# MIT-licensed derivative we develop in while upstream review is paused. +# +# `default-features = false` is LOAD-BEARING, not tidiness. It turns off +# liters' `bundled-sqlite`, so liters links the platform's libsqlite3 instead +# of compiling its own copy of the amalgamation into the staticlib. +# +# NOOP already links a SQLite: Packages/NoopLocalAccess and Packages/StrandImport +# both depend on GRDB.swift 6.29.3, which links Apple's system libsqlite3 +# through `.systemLibrary(name: "CSQLite")`. A bundled liters would put a +# SECOND SQLite in the same process, and the two do not share the +# process-global `unixInodeInfo` table SQLite uses to work around POSIX's +# "close any descriptor, lose all locks" rule — so either library can silently +# drop the other's advisory locks. For liters that is a correctness bug rather +# than a hazard: its guarantee that no foreign checkpointer restarts the WAL +# under the writer *is* a long-running read lock, and losing it forces a +# full-database snapshot upload on the next push. +# +# Do not add `features = ["bundled-sqlite"]` here. If a future change needs +# bindings generated against the platform's own sqlite3.h, use +# `features = ["system-sqlite-bindgen"]` (keeping default-features = false). +liters-ffi = { git = "https://github.com/vishk23/liters-mobile.git", branch = "main", default-features = false } + +# Must stay on the same major/minor as liters-ffi's uniffi, or the generated +# bindings and the shipped scaffolding come from different versions. +uniffi = { version = "0.32", features = ["cli"] } + +[profile.release] +lto = true +strip = true diff --git a/Rust/README.md b/Rust/README.md new file mode 100644 index 0000000000..7c3bad9821 --- /dev/null +++ b/Rust/README.md @@ -0,0 +1,155 @@ +# Rust — the liters dependency + +This directory is NOOP's build surface for +[liters](https://github.com/vishk23/liters-mobile): Litestream v0.5-compatible +SQLite replication, embeddable in iOS and Android apps. liters is +**[Kurt Mackey's](https://github.com/mrkurt/liters)** library, MIT-licensed; +`liters-mobile` is the standalone derivative we develop in while upstream +review is paused. + +`noop-liters` contains no logic. It exists so NOOP can pin liters as a git +dependency and build it with the one feature choice NOOP cannot get wrong, then +package it as an xcframework plus generated Swift bindings. + +## The SQLite rule + +**liters must link the system libsqlite3, never a bundled copy.** + +`Packages/NoopLocalAccess` and `Packages/StrandImport` both depend on +GRDB.swift 6.29.3, which links Apple's system libsqlite3 through +`.systemLibrary(name: "CSQLite")`. If liters bundled its own SQLite there would +be two SQLite libraries in one process, and they do not share the +process-global `unixInodeInfo` table SQLite uses to work around POSIX's "close +any descriptor to a file, lose all your locks on it" rule. Either library can +then silently drop the other's advisory locks. + +For most SQLite users that is a latent hazard. For liters it is a direct +correctness bug: the writer's guarantee that no foreign checkpointer restarts +the WAL underneath it *is* a long-running read lock. Drop that lock and a +foreign checkpoint restarts the WAL, the resume frame is overwritten, and the +next push recovers the only way it can — by uploading a full snapshot of the +database, which is the exact upload liters was adopted to avoid. + +`Cargo.toml` enforces this with `default-features = false` on `liters-ffi`, and +`build-ios.sh` offers no way to opt back in. Verify it on the built archive: + +```sh +nm -g target/aarch64-apple-ios/release/libnoop_liters.a | grep -c ' T _sqlite3_' +# 0 — defines none; they stay undefined and resolve against the same +# libsqlite3 GRDB uses (iOS SDK usr/lib/libsqlite3.tbd exports them) +``` + +## Building + +```sh +rustup target add aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios \ + aarch64-apple-darwin x86_64-apple-darwin +./build-ios.sh +xcodegen generate +``` + +`build-ios.sh` takes about 25 minutes cold and produces four things: + +| output | tracked | why | +|---|---|---| +| `target/apple/Liters.xcframework` | no | 83 MB per slice, three slices | +| `Strand/CloudSync/Generated/liters_ffi.swift` | **yes** | source the app compiles; a reviewer should be able to read it | +| `Config/LitersLocal.xcconfig` | no | names slice directories under the untracked `target/` | +| `target/apple/swift/*` | no | the staging copy the two above are made from | + +There is a macOS slice as well as the two iOS ones because `StrandTests` is a +macOS unit-test bundle (`project.yml`), so `StrandTests/LitersRoundTripTests.swift` +can only run against a macOS build. + +**Skipping this script is not an error.** `Config/Liters.xcconfig` is tracked and +defines every liters build setting as empty, then optionally includes the local +file this script writes. With no local file, `project.yml`'s three appends +(`LIBRARY_SEARCH_PATHS`, `SWIFT_INCLUDE_PATHS`, `OTHER_LDFLAGS`) expand to +nothing, the `LITERS` compilation condition is unset, and the generated bindings +compile to nothing. A checkout that has never seen Rust builds exactly as it +does today. Deleting `Config/LitersLocal.xcconfig` is how you turn liters back +off. + +To move the pin to a newer liters commit: + +```sh +cargo update -p liters-ffi # then commit the Cargo.lock change +``` + +`Cargo.lock` is committed on purpose: it records the exact liters commit this +tree builds against, so a checkout is reproducible even though the dependency +tracks a branch. As of this commit the pin is +`liters-mobile@6835f9f9`, and the four commits between it and `liters-mobile` +`main` are documentation and test-harness changes only. + +### What the pin does NOT contain + +There are **two** liters repos, and only one of them is pinned here: + +| repo | role | +|---|---| +| `vishk23/liters-mobile` | public, MIT, **what `Cargo.toml` pins** | +| `vishk23/liters` | the working clone; carries `fix/replica-lock-deadlock` | + +`fix(replica): bound and cancel the replica-file lock instead of blocking +forever` — the F_SETLKW fix that replaces a kernel-blocking `fcntl(F_SETLKW)` +with a bounded `F_SETLK` poll so `CancelToken` can interrupt it — exists only in +`vishk23/liters`. It has **not** been merged to `liters-mobile` `main`, so it is +**not** in the archive this directory builds. Check before assuming otherwise: + +```sh +git -C log --all --oneline --grep="bound and cancel" # currently empty +``` + +Like the integrity-check gotcha above, it is a Replica-side fix and so does not +affect the phone, which is a Writer. Both should be landed on `liters-mobile` +before anything runs a Replica against a system-SQLite build. + +## Build time, and the one thing that would fix it + +Most of the wall clock is `uniffi_bindgen` — a code-generation CLI — being +compiled at `opt-level=3` **once per cross-compiled target**. It is dead weight +in every one of them: nothing on a phone runs a bindings generator. + +It cannot be turned off from this crate. `liters-ffi` declares +`uniffi = { version = "0.32", features = ["cli"] }` among its normal +dependencies, and cargo feature unification means a downstream crate cannot +remove a feature an upstream crate asked for. The fix belongs in `liters-mobile`: +move that bin behind a `cli` feature and depend on plain `uniffi` otherwise. +`build-ios.sh` already passes `--lib` for the cross targets, which at least skips +*linking* the useless binary. + +## Known gotcha: the replica's integrity check under system SQLite + +`liters`' own test suite passes 80/80 with its default bundled SQLite, and +**31 passed / 49 failed** under `cargo test -p liters --no-default-features`, the +configuration NOOP actually ships. Every one of the 49 fails with the same +`SQLITE_CANTOPEN` ("unable to open database file"), and all of them are on the +Replica restore path. + +The cause is a single line — `crates/liters/src/replica.rs:648`, the only +read-only open in the crate, reached only from `check_integrity` after a restore: + +```rust +rusqlite::Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) +``` + +Apple's `libsqlite3` (3.51.0) cannot run a statement on a **WAL-mode** database +opened `SQLITE_OPEN_READONLY` when the `-shm` sidecar is absent — and the restore +deletes `-wal`/`-shm` just before this call. Reduced to a 20-line C program +against `/usr/lib/libsqlite3.dylib`: + +``` +WAL, sidecars removed, READONLY open=0 prepare=14 unable to open database file +WAL, sidecars intact, READONLY open=0 prepare=0 OK +WAL, sidecars removed, READWRITE open=0 prepare=0 OK +rollback journal, READONLY open=0 prepare=0 OK +``` + +**This does not affect NOOP's phone.** The phone is a Writer, and `writer.rs` +contains no read-only open at all — the whole crate has exactly one, on the +Replica path. The Replica runs on the server, in a Rust process with no GRDB, +which therefore builds liters with bundled SQLite and never reaches this. It is +still a real bug for any embedder that restores on a system-SQLite platform, and +it should be fixed upstream (open the integrity check READWRITE, or materialise +the `-shm` first). diff --git a/Rust/build-ios.sh b/Rust/build-ios.sh new file mode 100755 index 0000000000..eea4022576 --- /dev/null +++ b/Rust/build-ios.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Builds Liters.xcframework + the Swift bindings NOOP compiles, from the pinned liters-ffi git +# dependency, and writes the xcconfig that links them. +# +# Run this once before building the app with the page-replication trial. Afterwards a plain +# `xcodegen generate && xcodebuild …` picks the archive up with no extra flags: the last step here +# writes Config/LitersLocal.xcconfig, which the tracked Config/Liters.xcconfig optionally includes. +# Without that file every liters build setting expands to empty and the app builds exactly as it +# does today, so running this script is opt-in and skipping it is not an error. +# +# Prereqs: +# rustup target add aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios \ +# aarch64-apple-darwin x86_64-apple-darwin +# Xcode command line tools +# +# There is deliberately no bundled/system switch here. NOOP links GRDB, which links Apple's system +# libsqlite3, so liters MUST link that same libsqlite3 — `Cargo.toml` pins `liters-ffi` with +# `default-features = false` to guarantee it, and this script has no way to opt back in. See +# Cargo.toml for why two SQLite copies in one process is a correctness bug rather than a size +# problem, and the SANITY CHECK below, which fails the build if it ever stops holding. +set -euo pipefail +cd "$(dirname "$0")" +ROOT=$(cd .. && pwd) + +OUT=target/apple +BINDINGS=$OUT/swift +DEVICE_TARGET=aarch64-apple-ios +SIM_TARGETS=(aarch64-apple-ios-sim x86_64-apple-ios) +# macOS matters for more than completeness: StrandTests is a macOS unit-test bundle hosted in the +# macOS app (project.yml), so the round-trip tests in StrandTests/LitersRoundTripTests.swift can only +# run against a macOS slice. Both arches, because the macOS app builds ONLY_ACTIVE_ARCH=NO in Release. +MAC_TARGETS=(aarch64-apple-darwin x86_64-apple-darwin) + +# `--lib` on the cross-compiled targets skips the `uniffi-bindgen` BIN, which is a host developer +# tool and has no business being linked for a phone. (It does not avoid *compiling* the +# `uniffi_bindgen` crate: liters-ffi declares `uniffi = { features = ["cli"] }` among its normal +# dependencies, and cargo feature unification means a downstream crate cannot turn that off. Moving +# liters-ffi's own bindgen bin behind a `cli` feature would cut several minutes per target off this +# script; that has to be fixed in liters-mobile, not here.) +for t in "$DEVICE_TARGET" "${SIM_TARGETS[@]}" "${MAC_TARGETS[@]}"; do + echo "==> building $t" + cargo build -p noop-liters --release --target "$t" --lib +done + +# Generate Swift bindings from the host library's embedded metadata, built from the same sources and +# features as the device libraries so the bindings cannot drift from the shipped staticlib. +echo "==> generating Swift bindings" +cargo build -p noop-liters --release +rm -rf "$BINDINGS" && mkdir -p "$BINDINGS" +cargo run -p noop-liters --bin uniffi-bindgen -- generate \ + --library target/release/libnoop_liters.dylib \ + --language swift --out-dir "$BINDINGS" + +# Headers directory for the xcframework: the C header + module map. +HEADERS=$OUT/headers +rm -rf "$HEADERS" && mkdir -p "$HEADERS" +cp "$BINDINGS"/*.h "$HEADERS"/ +# uniffi emits a .modulemap; xcodebuild wants module.modulemap +cat "$BINDINGS"/*.modulemap > "$HEADERS"/module.modulemap + +# Fat simulator + fat macOS archives. `lipo` because an xcframework slice holds one archive per +# platform, not one per arch. +mkdir -p "$OUT/sim" "$OUT/mac" +lipo -create \ + $(for t in "${SIM_TARGETS[@]}"; do echo "target/$t/release/libnoop_liters.a"; done) \ + -output "$OUT/sim/libnoop_liters.a" +lipo -create \ + $(for t in "${MAC_TARGETS[@]}"; do echo "target/$t/release/libnoop_liters.a"; done) \ + -output "$OUT/mac/libnoop_liters.a" + +rm -rf "$OUT/Liters.xcframework" +xcodebuild -create-xcframework \ + -library "target/$DEVICE_TARGET/release/libnoop_liters.a" -headers "$HEADERS" \ + -library "$OUT/sim/libnoop_liters.a" -headers "$HEADERS" \ + -library "$OUT/mac/libnoop_liters.a" -headers "$HEADERS" \ + -output "$OUT/Liters.xcframework" + +# --------------------------------------------------------------------------- +# SANITY CHECK — a hard gate, not advice. +# +# The device archive must DEFINE no sqlite3_* symbols and leave them undefined, so they resolve +# against the libsqlite3 GRDB links. A non-zero count means `bundled-sqlite` came back on and a second +# SQLite is about to be linked into the app, which silently breaks the advisory locking liters' +# correctness rests on. That is worth failing a build over, so it does. +# --------------------------------------------------------------------------- +SYMS=$(nm -g "target/$DEVICE_TARGET/release/libnoop_liters.a" 2>/dev/null || true) +DEFINED=$(printf '%s\n' "$SYMS" | grep -c ' T _sqlite3_' || true) +UNDEFINED=$(printf '%s\n' "$SYMS" | grep -c ' U _sqlite3_' || true) +if [ "$DEFINED" -ne 0 ]; then + echo "FATAL: the device archive DEFINES $DEFINED sqlite3_* symbols — liters bundled its own SQLite." >&2 + echo " Check that Cargo.toml still pins liters-ffi with default-features = false." >&2 + exit 1 +fi +if [ "$UNDEFINED" -eq 0 ]; then + echo "FATAL: the device archive references NO sqlite3_* symbols at all, which cannot be right." >&2 + exit 1 +fi +echo "==> sqlite linkage OK: 0 defined, $UNDEFINED undefined (resolve from the host's libsqlite3)" + +# --------------------------------------------------------------------------- +# Install the Swift bindings into the app's source tree. +# +# Wrapped in `#if LITERS` because the file is TRACKED (it is source the app compiles, and a reviewer +# should be able to read it) while the archive it needs is NOT (83 MB per slice). A checkout that has +# never run this script must still build, so without the LITERS condition — set below, in the +# untracked xcconfig — the file has to compile to nothing rather than fail on `import liters_ffiFFI`. +# --------------------------------------------------------------------------- +GEN=$ROOT/Strand/CloudSync/Generated +mkdir -p "$GEN" +{ + echo "// GENERATED by Rust/build-ios.sh from the pinned liters-ffi. Do not edit by hand." + echo "// The #if is added by that script, not by uniffi: see the script for why it is here." + echo "#if LITERS" + cat "$BINDINGS/liters_ffi.swift" + # The leading newline is load-bearing. uniffi's output ends `// swiftlint:enable all` with NO + # trailing newline, so a plain `echo "#endif"` lands on the same line as that comment and is + # swallowed by it — producing a file whose `#if LITERS` is never closed. The compiler's complaint + # ("expected #else or #endif at end of conditional compilation block", pointing at EOF) does not + # mention the comment, so this is worth a comment rather than a rediscovery. + printf '\n#endif\n' +} > "$GEN/liters_ffi.swift" +echo "==> installed bindings: $GEN/liters_ffi.swift" + +# --------------------------------------------------------------------------- +# Write the linkage xcconfig FROM the layout xcodebuild just produced, so the slice directory names +# cannot drift from the ones actually in the xcframework. +# --------------------------------------------------------------------------- +slice_for() { # $1 = SupportedPlatform, $2 = SupportedPlatformVariant ("" for none) + python3 - "$OUT/Liters.xcframework/Info.plist" "$1" "$2" <<'PY' +import plistlib, sys +plist, platform, variant = sys.argv[1], sys.argv[2], sys.argv[3] +with open(plist, 'rb') as f: + libs = plistlib.load(f)['AvailableLibraries'] +for lib in libs: + if lib.get('SupportedPlatform') == platform and lib.get('SupportedPlatformVariant', '') == variant: + print(lib['LibraryIdentifier']); break +else: + sys.exit("no slice for %s/%s" % (platform, variant or '-')) +PY +} +IOS_SLICE=$(slice_for ios "") +SIM_SLICE=$(slice_for ios simulator) +MAC_SLICE=$(slice_for macos "") + +XCCONFIG=$ROOT/Config/LitersLocal.xcconfig +cat > "$XCCONFIG" <= 12, Whoop5Ecg.plausibleFilteredFrame(frame) else { return } + // CENSUS FIRST, unconditionally — before the heuristic gets a vote. Every frame that reaches here + // is one the run could not classify, and the ones the triage REJECTS are the ones whose bytes + // would otherwise be gone: a 2-bytes-per-sample length assumption discarded every 3-byte frame in + // silence, and the report then truthfully said no frame passed. The census is what makes the next + // wrong assumption visible instead of invisible. (The caller has already CRC-gated the frame.) + ecgProbeCensus.record(frame: frame) + guard frame.count >= 12 else { return } + // Which sample widths the length agreed with — recorded rather than re-derived, because a + // candidate under width 3 is a different finding from a candidate under width 2 and the report + // must not flatten them together. + let widths = Whoop5Ecg.filteredBytesPerSampleCandidates(frame: frame) + guard !widths.isEmpty else { return } ecgProbePacketsSeen += 1 + // The header is read directly rather than through `decodeFilteredFrame`, which decodes SAMPLES and + // therefore still assumes 2 bytes each: a width-3 candidate would fail it and lose its line while + // its status header parses perfectly. Nothing on this line comes from the sample blob. guard ecgProbeCandidates.count < BLEManager.ecgProbeMaxCandidates, - let packet = Whoop5Ecg.decodeFilteredFrame(frame) else { return } - let header = packet.header + let payload = Whoop5Ecg.innerPayload(frame), + let header = EcgStatusHeader(payload: payload) else { return } // The classifier byte is logged as a NUMBER, never as its token name: a strap log is a shareable // artefact, and no line in it should read like a clinical finding. The decoder maps the value // offline; the raw byte is lossless. - let line = String(format: "type=0x%02x len=%d samples=%d quality=%d leadsOn=%d hr=%d hrv=%d " - + "classifierRaw=%d statusRaw=%d (unvalidated instrumentation, not a diagnosis)", - Int(frame[8]), frame.count, Int(header.numberOfECGSamples), - Int(header.signalQualityRaw), header.heartKeyLeadsAreOn ? 1 : 0, - Int(header.heartKeyHR), Int(header.heartKeyHRV), - Int(header.heartKeyArrhythmiaCheckResultRaw), - Int(header.heartKeyArrhythmiaCheckStatusRaw)) + let shape = String(format: "type=0x%02x len=%d samples=%d", + Int(frame[8]), frame.count, Int(header.numberOfECGSamples)) + let status = String(format: "quality=%d leadsOn=%d hr=%d hrv=%d classifierRaw=%d statusRaw=%d", + Int(header.signalQualityRaw), header.heartKeyLeadsAreOn ? 1 : 0, + Int(header.heartKeyHR), Int(header.heartKeyHRV), + Int(header.heartKeyArrhythmiaCheckResultRaw), + Int(header.heartKeyArrhythmiaCheckStatusRaw)) + let line = "\(shape) widths=\(widths.map(String.init).joined(separator: ",")) \(status) " + + "(unvalidated instrumentation, not a diagnosis)" ecgProbeCandidates.append(line) log("ECG probe: candidate packet \(line)") // Dump the raw bytes of the FIRST few candidates so a plain strap-log export is enough to redo @@ -3407,6 +3526,43 @@ public final class BLEManager: NSObject, ObservableObject { clearRebootState() // clears the "Reconnecting…" pill → back to "Active · Live" } + /// Arm the stalled-handshake watchdog for the connection that just came up. + /// + /// Called from BOTH paths that can put the app on a live link: `didConnect` and the `willRestoreState` + /// branch that adopts an already-`.connected` peripheral. The restore branch is the one that actually + /// bit: it never reaches `didWriteValueFor`, which is the only caller of `startKeepAlive()`, so the + /// 120 s liveness watchdog that would have bounced the dead link is never armed and the app retries the + /// same corpse forever ("Backfill: deferred — connect handshake not done yet", 40+ minutes). + /// + /// One-shot, generation-guarded (the same `connectGeneration` idiom the #711 reconnect-guide timer uses), + /// so a timer left in flight by an earlier connection can never bounce a healthy later one. + private func armStalledHandshakeWatchdog() { + let gen = connectGeneration + DispatchQueue.main.asyncAfter(deadline: .now() + StalledHandshakePolicy.fuseSeconds) { [weak self] in + guard let self, self.connectGeneration == gen else { return } + guard StalledHandshakePolicy.shouldRecover( + connected: self.state.connected, + handshakeDone: self.connectHandshakeDone, + sawData: self.sawDataThisConnection, + intentionalDisconnect: self.intentionalDisconnect, + autoReconnectPaused: self.autoReconnectPausedForBondLoop, + bondRefused: self.state.pairingHint != nil) else { return } + // Log the recovery (the "log successes" forensics rule): without this line a strap log shows a + // reconnect with no cause, which is exactly how the original 40-minute spin stayed invisible. + self.log("Handshake stalled: connected for \(Int(StalledHandshakePolicy.fuseSeconds))s with no " + + "handshake and no data — the link is up but dead (a restored peripheral iOS still " + + "believes in). Cancelling it so the rescan can find the strap again.") + if TestCentre.active(.connection) { + self.state.append(log: "handshakeStalled gen=\(gen) fuse=\(Int(StalledHandshakePolicy.fuseSeconds))s " + + "(cancelling + rescanning)", domain: .connection) + } + // Recovery = the SAME lever the liveness watchdog pulls: cancel and let didDisconnectPeripheral's + // existing 3 s rescan re-find the strap. No new reconnect path, and it still honours the + // intentional-disconnect / bond-loop-pause gates that live there. + if let p = self.peripheral { self.central.cancelPeripheralConnection(p) } + } + } + private func startKeepAlive() { keepAliveTimer?.cancel() let s = BLEManager.keepAliveIntervalSeconds @@ -3667,20 +3823,7 @@ public final class BLEManager: NSObject, ObservableObject { for c in chars where !c.isNotifying || restoreNeedsResubscribe { requestNotify(c, on: p, reason: reason) } - // #490: a 5/MG's 0x2A19 battery READ is refused pre-bond (it fires in didDiscoverCharacteristicsFor, - // before the link is encrypted) and is never retried, so `batteryPct` stays nil from connect — and - // the 5/MG sends NO unsolicited battery notification (so the notify subscription above never fills - // it on its own). Re-issue the read here: every caller is post-bond (CLIENT_HELLO-ack, post-bond, - // keep-alive), so the link is encrypted and the read succeeds; the keep-alive caller also RE-polls - // it (~every 30 s) so the reading stays current as the strap drains and BatteryEstimator gets its - // discharge samples on any screen. This is the iOS parity for Android's post-handshake read + - // ~60 s keep-alive poll (WhoopBleClient BATTERY_ON_CONNECT_DELAY_MS / refreshBattery). 4.0 is - // EXCLUDED — its 0x2A19 is a stub constant 100 (real value = GET_BATTERY_LEVEL; re-reading the stub - // would revert the true reading, #77). The 600 s same-% throttle in LiveState guards the estimator. - if let b = batteryCharacteristic, b.properties.contains(.read), - selectedModel.deviceFamily != .whoop4 { - p.readValue(for: b) - } + retryBatteryReadIfDue(on: p, reason: reason) // #520 DIS identity read — same post-bond reasoning as the #490 battery read above: on a 5/MG the // link must be encrypted first. Gated to 5/MG (a 4.0 issues NO new reads, exactly like the battery // re-read excludes it) and fired ONCE per connection: serial + hardware revision are immutable, so @@ -3698,6 +3841,29 @@ public final class BLEManager: NSObject, ObservableObject { } } + /// Re-issue the standard Battery Level (0x2A19) READ that a pre-bond link refused. + /// + /// This is the missing half of `enableLiveNotifications`. That function's own call site claims it + /// recovers the "standard HR/battery that failed pre-bond", but it only ever re-ran `setNotifyValue` — + /// never `readValue`. So on a WHOOP 5/MG the one-shot read fired at discovery (unencrypted link) was + /// rejected, dropped in `didUpdateValueFor`'s error branch, and never retried: HR came back because the + /// strap PUSHES it, battery never did. Re-reading here means every moment that already re-subscribes + /// notifies — post-bond 5/MG, post-bond 4.0, keep-alive, manual refresh, realtime start — is also a + /// moment the battery reading can self-heal. `StrapBatteryReadPolicy` owns the family gate (#77: never + /// read a WHOOP 4.0's stub-100 characteristic) and the throttle (this runs on every HISTORY_END ack + /// during an offload); both are pinned by tests. + private func retryBatteryReadIfDue(on p: CBPeripheral, reason: String) { + guard let b = batteryCharacteristic else { return } + let now = Date().timeIntervalSince1970 + guard StrapBatteryReadPolicy.shouldRead(family: selectedModel.deviceFamily, + canRead: b.properties.contains(.read), + lastReadAt: lastBatteryReadAt, + now: now) else { return } + lastBatteryReadAt = now + p.readValue(for: b) + log("Battery Level read re-issued (\(reason))") + } + /// Resolve + log the 5/MG hardware variant once a DIS string lands (#520). Both characteristics /// arrive as SEPARATE async callbacks, so this runs on each and re-resolves — the contradiction rule /// in `Whoop5Variant` needs both before it can disagree. Diagnostic only: nothing gates on it yet. @@ -4208,6 +4374,11 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { // only — BLEManager stays decoupled from the store and the connect flow below is unchanged. connectedPeripheralUUID = peripheral.identifier.uuidString state.connected = true + // …but the link has proven NOTHING yet: this fires before discoverPrimaryServices below, so no + // characteristic exists and no byte has moved. The Devices pill gates "Active · Live" on this so it + // can't claim live data off a raw CBPeripheral handle. Set true by the first notification, or by the + // completed connect handshake, whichever lands first. + state.linkProven = false // A connect succeeded → clear the stale-bond re-pair guide UNLESS we are in a known bond-loop // (#617). In that loop the strap "connects" every ~3 s before timing out again, so clearing here // wiped the guide on EVERY cycle: it flashed for ~1 s and vanished, so the user could never read it @@ -4230,6 +4401,9 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { state.reconnectGuide = nil } lastDataAt = Date() + sawDataThisConnection = false // this link has not proven itself yet — see armStalledHandshakeWatchdog + lastBatteryReadAt = nil // a fresh link re-reads 0x2A19 immediately (5/MG; #77 spares the 4.0) + armStalledHandshakeWatchdog() log("Connected — discovering services") // #1020: stamped OUTSIDE the gate below. Test Centre is usually switched on AFTER something looks // wrong, so a stamp taken only when the mode happened to be on at connect time would either be @@ -4328,6 +4502,12 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { } bondedAt = nil // cleared after the bond-loop detector above read it (#617) state.connected = false + // Back to the inert default: the flag only ever means something while BLEManager owns a live link, + // and a stale `false` left here would make the NEXT source (Oura/Huami/FTMS/standard-HR, none of + // which touch it) read as un-live. The next connect/restore clears it again. + state.linkProven = true + sawDataThisConnection = false + lastBatteryReadAt = nil // the next link re-reads 0x2A19 immediately (5/MG; the 4.0 never reads it) state.encryptedBond = false // cleared with didBond; next session must re-prove the bond (#69) state.charging = nil // a stale charging flag must not outlive the link state.batteryMv = nil // #592: a stale pack voltage must not outlive the link @@ -4347,6 +4527,7 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { ecgProbeSteps = [] ecgProbeCandidates = [] ecgProbePacketsSeen = 0 + ecgProbeCensus = Whoop5EcgProbe.FrameCensus() // The DIS strings belong to the link that just dropped; a stale variant must not keep an MG-only // capability unlocked for whatever connects next. state.whoop5Variant = nil @@ -4554,6 +4735,21 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { Task { @MainActor in await bootstrapStore() } if p.state == .connected { state.connected = true + // The restored link is UNPROVEN despite the seeded bond flags above: those record that this + // strap was bonded in a PREVIOUS process, which says nothing about whether it is alive now. This + // is the exact case that made the pill lie — CoreBluetooth says .connected, the bond flags say + // bonded, and the strap is dead. + state.linkProven = false + // A restored peripheral is the ONE path onto a live link that never runs didConnect, so every + // per-connection reset and every watchdog here has to be done by hand — that omission is exactly + // what let a dead restored handle spin for 40+ minutes. Bump the generation first: this IS a new + // connection for the watchdog's staleness guard, and without the bump a restore that follows a + // real connect would reuse its generation. + connectGeneration &+= 1 + sawDataThisConnection = false + lastBatteryReadAt = nil + lastDataAt = Date() // start the liveness clock at the restore, not at BLEManager init + armStalledHandshakeWatchdog() // #613: the inherited notify subscriptions come back reported-active but dead. Force one real // off→on re-subscribe this session (see `requestNotify`) so live HR/R-R resume AND // `didUpdateNotificationStateFor` fires → `cmdNotifyConfirmedActive` → `connectSettled` → the @@ -4564,6 +4760,7 @@ extension BLEManager: @preconcurrency CBCentralManagerDelegate { } else { state.connected = false log("Restored DISCONNECTED peripheral \(p.identifier) — reconnect on poweredOn") + // This branch goes through central.connect -> didConnect, which arms the watchdog. if central.state == .poweredOn { connectRestored(p, reason: "willRestoreState") } else { @@ -4840,6 +5037,7 @@ extension BLEManager: @preconcurrency CBPeripheralDelegate { if !whoop5SessionStarted { whoop5SessionStarted = true connectHandshakeDone = true // unblocks beginBackfill()'s guard + state.linkProven = true // the strap acked CLIENT_HELLO — this link is genuinely alive log("WHOOP 5/MG: connect handshake done — backfill unblocked") noteRebootReconnectIfNeeded() // Re-apply the Broadcast-HR device-config flag if the user opted in (#181). @@ -4889,6 +5087,7 @@ extension BLEManager: @preconcurrency CBPeripheralDelegate { // type-47 fine because it runs the sequence once on a stable connection; the app stormed it. guard !connectHandshakeDone else { return } connectHandshakeDone = true + state.linkProven = true // the confirmed-write bond acked — this link is genuinely alive noteRebootReconnectIfNeeded() backfillStarted = true @@ -5115,6 +5314,13 @@ extension BLEManager: @preconcurrency CBPeripheralDelegate { guard let data = characteristic.value else { return } let bytes = [UInt8](data) lastDataAt = Date() // feed the liveness watchdog on every notification + // The strap has spoken on this link — it is not a zombie, so the stalled-handshake watchdog must + // stand down (the 120s liveness fuse owns a link that goes quiet LATER). This covers every source + // that lands here: a 5/MG's standard 0x2A37 HR, a WHOOP4 puffin/custom frame, a battery read. + if !sawDataThisConnection { + sawDataThisConnection = true + state.linkProven = true // …and the Devices pill may honestly say "Live" now (#221 sibling) + } switch characteristic.uuid { case BLEManager.heartRateChar: diff --git a/Strand/BLE/BackfillProgress.swift b/Strand/BLE/BackfillProgress.swift new file mode 100644 index 0000000000..6250ef997f --- /dev/null +++ b/Strand/BLE/BackfillProgress.swift @@ -0,0 +1,70 @@ +import Foundation + +/// What to tell the user about strap-history recovery, as a pure decision. +/// +/// This exists because the recovery was INVISIBLE. The app already showed "Syncing strap history…" while a +/// single offload session ran, but nothing said how far behind the strap was, whether the gap was closing, +/// or whether a quiet app meant "done" or "waiting 15 minutes for the next 60-second session". A user +/// sitting on an 18-hour backlog spent hours unable to tell whether his data was lost or merely pending — +/// and the honest answer ("~15h behind, recovering") was already computable from numbers BLEManager tracks. +/// +/// Both inputs are already on `LiveState`: `strapRange.newestUnix` (the newest record the strap says it +/// holds, from GET_DATA_RANGE) and `dataFrontierUnix` (the newest record we've actually persisted, read +/// once per offload session for the auto-continue decision). The gap between them IS the backlog. +enum BackfillProgress: Equatable { + /// Nothing honest to say — no link, or the strap hasn't answered a range yet. + case unknown + /// Behind, and actively draining right now. + case recovering(behindSeconds: Int, frontierUnix: Int) + /// Behind, but not draining this instant (between sessions / waiting out the periodic floor). + case pending(behindSeconds: Int, frontierUnix: Int) + /// The frontier is within `behindGapSeconds` of the strap's newest record — as caught up as the strap + /// can tell us. Deliberately the SAME threshold `BackfillContinuation` stops draining at, so the + /// readout can never say "behind" about a gap the drain has already decided is closed. + case upToDate + + /// `nowUnix` is unused for the decision itself (the gap is strap-newest vs our frontier, both strap-side + /// timestamps) but is taken so a caller can't be tempted to reach for a clock inside a pure function. + static func resolve(connected: Bool, + backfilling: Bool, + frontierUnix: Int?, + strapNewestUnix: Int?, + behindGapSeconds: Int = BackfillContinuation.defaultBehindGapSeconds) -> BackfillProgress { + guard connected, let frontier = frontierUnix, let newest = strapNewestUnix else { return .unknown } + let behind = newest - frontier + // A frontier AHEAD of the strap's reported newest is a stale/wrong-epoch range answer (#451) or a + // future-dated strap clock (#928), not negative backlog. Claim nothing rather than a nonsense figure. + guard behind > behindGapSeconds else { return behind >= -behindGapSeconds ? .upToDate : .unknown } + return backfilling + ? .recovering(behindSeconds: behind, frontierUnix: frontier) + : .pending(behindSeconds: behind, frontierUnix: frontier) + } + + /// How far behind, in whole hours/minutes — nil when there's nothing to report. + var behindSeconds: Int? { + switch self { + case let .recovering(b, _), let .pending(b, _): return b + case .unknown, .upToDate: return nil + } + } + + /// The strap-side instant our data currently reaches — the "we have your data up to HERE" anchor. + var frontierUnix: Int? { + switch self { + case let .recovering(_, f), let .pending(_, f): return f + case .unknown, .upToDate: return nil + } + } + + /// Compact "~15h behind" / "~42m behind". Whole hours once past an hour, minutes below it — the same + /// shape the battery runtime estimate uses, so the two readouts read as one family. + /// + /// A plain (unlocalized) string, deliberately: this mirrors `DevicePillState.label`, whose call site + /// wraps it as `StatePill(LocalizedStringKey(label))`. Keeping the resolver free of `String(localized:)` + /// is what lets the tests assert on the decision rather than on the user's current locale. + var behindLabel: String? { + guard let s = behindSeconds else { return nil } + if s >= 3600 { return "~\(s / 3600)h behind" } + return "~\(max(1, s / 60))m behind" + } +} diff --git a/Strand/BLE/LiveState.swift b/Strand/BLE/LiveState.swift index d47700bac1..8f1c470377 100644 --- a/Strand/BLE/LiveState.swift +++ b/Strand/BLE/LiveState.swift @@ -36,6 +36,20 @@ public final class LiveState: ObservableObject { /// per-connection guards (`connectHandshakeDone`, the cmd-notify-confirmed flag) reset on disconnect, /// so the next connection can bump this again. @Published public var connectSettled: Int = 0 + /// Has the CURRENT link proven itself — i.e. has the strap actually spoken (any notification), or has + /// the connect handshake completed? `connected` alone cannot answer that: BLEManager publishes it the + /// instant CoreBluetooth's `didConnect` fires, BEFORE service discovery, so it is true while no + /// characteristic exists and no byte has moved. Combined with iOS state restoration handing back a + /// peripheral that is `.connected` but dead, that made the Devices card advertise "Active · Live" for a + /// strap that was a corpse — the user read "Live" all evening on a band that had flat-lined. + /// + /// DEFAULTS TRUE, and only BLEManager ever clears it. That is deliberate: `connected` is also driven by + /// four non-WHOOP sources (Oura, Huami, FTMS, generic standard-HR), and each of those sets it true only + /// once it is genuinely subscribed and receiving. They never touch this flag, so they keep their exact + /// current behaviour — an inert `true` — instead of being regressed into looking offline. BLEManager + /// clears it at connect/restore, sets it on first data or a completed handshake, and restores the inert + /// `true` on disconnect so a stale `false` can't outlive the link and poison the next source. + @Published public var linkProven: Bool = true /// True ONLY when a non-WHOOP live source (currently the Oura ring) is actively streaming live HR. /// This is the green "STREAMING" signal for sources that have no WHOOP-style encrypted bond: it is /// DELIBERATELY separate from `bonded`, which carries WHOOP encrypted-bond + buzz semantics (it gates @@ -64,11 +78,23 @@ public final class LiveState: ObservableObject { /// GET_EXTENDED_BATTERY_INFO response (#592). Shown on the Devices card as a "x.xx V" readout beside the /// percent; nil until the first battery event lands. Twin of the Android LiveState.batteryMv. @Published public var batteryMv: Int? = nil - /// Charging flag from the strap's BATTERY_LEVEL events — wire observation: u8 bit0 in the + /// RAW charging flag from the strap's BATTERY_LEVEL events — wire observation: u8 bit0 in the /// event payload (4.0 @26 / 5.0 @30), pushed ~every 8 min on captured links. nil until the /// first event of a session; cleared on disconnect so a stale flag can't outlive the link. /// Flag ONLY — the battery % keeps its family-specific source (#77). + /// + /// This is the WIRE value and stays that way (it's what gets persisted). Do not render it directly: + /// the 5.0 offset is unverified and has been observed reporting 0 on a strap that was demonstrably + /// charging. `chargingEffective` is the value to display — see `StrapChargeInference`. @Published public var charging: Bool? = nil + /// The charging state worth showing: the raw flag, rescued by a measurably rising SoC when the flag + /// stays silent or wrong. Computed (never stored) over the already-`@Published` `charging` + + /// `batterySamples`, exactly like `batteryEstimate` — so any view reading it re-renders when either + /// witness changes, and there's no third piece of state to keep in sync. + public var chargingEffective: Bool? { + StrapChargeInference.resolve(flag: charging, samples: batterySamples, + nowUnix: Int(Date().timeIntervalSince1970)) + } /// The Oura ring's current wear/charge state (nil for non-Oura straps or before any evidence this /// session). Driven by OuraLiveSource from the live-HR push + the ring's STATE charger strings: a live @@ -204,6 +230,30 @@ public final class LiveState: ObservableObject { /// window can't outlive the link. public func clearStrapRange() { strapRange = nil } + /// The newest record WE'VE actually persisted (max HR ts) — our data frontier. Together with + /// `strapRange.newestUnix` (the newest the STRAP holds) this is the backlog: the gap between what the + /// strap banked and what we've recovered. BLEManager publishes it from the one place that already reads + /// it, the per-session auto-continue decision, so the readout costs no extra store queries. + /// + /// nil until the first offload session ends, and cleared on disconnect — a stale frontier would make the + /// progress readout quote a gap that no longer exists. + @Published public private(set) var dataFrontierUnix: Int? + + /// Publish the frontier read during an offload session's continuation decision. nil is a legitimate + /// answer (nothing persisted yet) and is stored as such. + public func setDataFrontier(_ ts: Int?) { dataFrontierUnix = ts } + + /// What to tell the user about history recovery right now. Computed over the already-`@Published` + /// `connected` / `backfilling` / `dataFrontierUnix` / `strapRange`, so a view reading it re-renders when + /// any input moves — same pattern as `batteryEstimate`. Internal (not public) because `BackfillProgress` + /// is: it's an app-module display type, and its threshold defaults come from `BackfillContinuation`, + /// which is internal too. Every consumer (the Today card, the tests via @testable) is in-module. + var backfillProgress: BackfillProgress { + BackfillProgress.resolve(connected: connected, backfilling: backfilling, + frontierUnix: dataFrontierUnix, + strapNewestUnix: strapRange.map { $0.newestUnix > 0 ? $0.newestUnix : nil } ?? nil) + } + @Published public var lastFrameType: String? = nil @Published public var lastEvent: String? = nil /// #987: unix of the most recent strap frame FrameRouter routed. Deliberately NOT @Published - the @@ -256,6 +306,12 @@ public final class LiveState: ObservableObject { /// explicitly not a medical measurement. @Published public var ecgProbe: String? = nil + /// A rendered snapshot of the raw-history reject archive (`RawHistoryArchive.summaryText`), shown + + /// copied from the Devices dialog. Purely local: it reads an on-disk file and sends the strap + /// nothing. The affordance for checking, straight after an experiment, whether the record you were + /// hunting is actually in the archive rather than already evicted. Cleared on dialog dismiss. + @Published public var rawHistoryArchiveReport: String? = nil + /// The 5-generation hardware variant resolved from the strap's Device Information Service /// (`Whoop5Variant.label`: "MG" / "5.0" / "—"), nil before any DIS string has landed. Published so an /// MG-only capability can gate on POSITIVELY identified hardware instead of guessing from a model @@ -530,6 +586,7 @@ public final class LiveState: ObservableObject { recentHrSamples.removeAll() // Sleep readout buffers must not outlive the link (Group E) recentGravitySamples.removeAll() clearStrapRange() // a stale clock-drift window must not outlive the link either + dataFrontierUnix = nil // …nor a stale frontier, which would quote a gap that's moved on lastFrameAtUnix = nil // #987: a stale "last frame" freshness must not outlive it either ouraWearState = nil // a stale worn/charging badge must not outlive the link either // Perf: flush the durable log tail on disconnect (mirroring is batched in `append`), so a completed diff --git a/Strand/BLE/PuffinDeepBufferLog.swift b/Strand/BLE/PuffinDeepBufferLog.swift index 56b0dff2e3..8dea1ceb3e 100644 --- a/Strand/BLE/PuffinDeepBufferLog.swift +++ b/Strand/BLE/PuffinDeepBufferLog.swift @@ -61,7 +61,17 @@ final class PuffinDeepBufferLog { /// The 2140-B buffers are ~4.3 KB of hex each and arrive in bursts, so a bigger cap than the /// EVENT log: 60 MB live keeps roughly a few hours of accumulated high-rate bursts, ample to /// reverse the layout, and rotation bounds total disk at ~120 MB. - private static let softCapBytes = 60 * 1024 * 1024 + /// + /// Overridable via the `PuffinDeepBufferSoftCapMB` default because the 60 MB figure is sized for + /// "a few hours of bursts", which is the STEADY-STATE case. Recovering a dead-strap backlog is the + /// other case: the strap banks one 1244-B + one 2140-B buffer per SECOND of history, so a 16 h + /// backlog is ~190 MB raw / ~380 MB of hex — it would rotate away most of itself before the drain + /// ends, and the ack-trim frees those buffers from the strap permanently as it goes. A one-shot + /// raise is the only way to keep a full backlog intact. 0 or unset keeps the 60 MB default. + private static var softCapBytes: Int { + let mb = UserDefaults.standard.integer(forKey: "PuffinDeepBufferSoftCapMB") + return (mb > 0 ? mb : 60) * 1024 * 1024 + } /// WHOOP 5/MG inner-record type byte for the R22 deep packets (type 47 / 0x2F), at offset 8 (the /// same position `BLEManager.isOffloadFrame` / `PuffinEventLog` index). The 1 Hz historical rollup @@ -71,6 +81,25 @@ final class PuffinDeepBufferLog { /// Skip the ~124-B ≈1 Hz record; keep the 1244-/2140-B high-rate buffers. private static let minBufferBytes = 1000 + /// Lowercase hex, table-driven. Runs on the MainActor in the frame-delivery path, so it avoids the + /// obvious `frame.map { String(format: "%02x", $0) }.joined()`, which costs one Foundation formatter + /// call + one String allocation PER BYTE. A nibble lookup into a byte array is materially cheaper and + /// allocates once. The two properties that matter are pinned by tests rather than asserted here: + /// `testHexStringMatchesFormatterForEveryByteValue` (byte-for-byte parity with the encoding it + /// replaces, across the whole domain) and `testHexStringIsFastEnoughForTheAckPath` (sub-millisecond + /// per buffer). NOTE: this path only runs when the capture toggle (`PuffinFrameRecorder.enabledKey`) + /// is ON — it is off by default, so this is research-capture cost, not a cost every user pays. + private static let hexDigits: [UInt8] = Array("0123456789abcdef".utf8) + nonisolated static func hexString(_ bytes: [UInt8]) -> String { + var out = [UInt8]() + out.reserveCapacity(bytes.count * 2) + for b in bytes { + out.append(hexDigits[Int(b >> 4)]) + out.append(hexDigits[Int(b & 0x0F)]) + } + return String(decoding: out, as: UTF8.self) + } + /// Pure predicate: is `frame` a WHOOP 5/MG high-rate deep buffer? A reassembled frame's inner-record /// type byte sits at offset 8, so this needs `count > 8` before indexing. Extracted so the offset-8 /// magic number is unit-testable without a strap. @@ -139,14 +168,25 @@ final class PuffinDeepBufferLog { guard !disabled, Self.isDeepBuffer(frame), isEnabled else { return } let tsMs = Int(Date().timeIntervalSince1970 * 1000) let strapTs = Self.strapTs(frame).map { String($0) } ?? "null" - let hex = frame.map { String(format: "%02x", $0) }.joined() + let hex = Self.hexString(frame) // #423/#455: run the raw-IMU decoder on the 1244-B buffer so every captured IMU frame carries // its decoded activity summary (cadence/energy/jerk/gyro) inline beside the raw hex. This is the // first CALLER of `Whoop5RawImu.decode` outside its own tests — it exercises the decoder on real // device captures and makes each JSONL line self-checking (raw ↔ decode) with NO stored table, // migration, or downstream gate. Instrumentation only, per the derived-signal rule; the 2140-B // optical buffer stays raw-only (its layout isn't decoded yet). - let imu = Self.decodedImuField(frame) + // LIVE frames only — a hygiene measure, and deliberately carrying NO measured throughput claim. + // An earlier version of this comment derived a link-rate model ("the link is a metronome") from + // the inter-frame gap asymmetry in a single 208 s window; that model did not survive + // re-measurement across more windows and has been removed rather than restated. Do not re-derive + // a link-rate or throughput conclusion from this file — it logs frames, it does not time them. + // + // The reason to skip during offload stands on its own, independent of any rate: it is per-frame + // work on the MainActor in the delivery path, it buys nothing there, and the raw `hex` on this + // very line is the decoder's own input — so the summary is recomputable offline from the archive + // at any time. Live frames arrive ~1/s, where the inline summary is free and makes each line + // self-checking. Capture captures; analysis is downstream. + let imu = isOffload ? "" : Self.decodedImuField(frame) let line = "{\"ts_ms\":\(tsMs),\"strap_ts\":\(strapTs),\"size\":\(frame.count)," + "\"offload\":\(isOffload),\"char\":\"\(char.uuidString.lowercased())\",\"hex\":\"\(hex)\"\(imu)}\n" do { diff --git a/Strand/BLE/PuffinExperiment.swift b/Strand/BLE/PuffinExperiment.swift index 02592f50c0..c2b9f3ada1 100644 --- a/Strand/BLE/PuffinExperiment.swift +++ b/Strand/BLE/PuffinExperiment.swift @@ -201,6 +201,22 @@ enum PuffinExperiment { static var motionAwareWakeEnabled: Bool { UserDefaults.standard.bool(forKey: motionAwareWakeKey) } + /// Opt-in "Bridge brief wakes" (default OFF): inside `SleepStager.detectSleep`, adjacent sleep runs + /// across a ≤ 45-min wake gap are assembled into ONE session with the wake staged inside it, so a + /// brief get-up (a bathroom trip smeared past the run-merge threshold by the centered rolling + /// stillness window) no longer splits the night into separate stored sessions, and sub-1h sleep + /// fragments between arousals are no longer dropped outright. Every existing guard (min duration, + /// the 16-h span cap — which the bridge also refuses to assemble past — HR confirmation, off-wrist, + /// daytime nap bars) still applies to the assembled run. Default OFF per the varying-input + /// validation discipline (#194/#345). NOTE: an HR-led rescue of restless-but-asleep nights was + /// prototyped behind this flag and WITHDRAWN after a real-data replay — see the history note above + /// `SleepStager.wakeBridgeMaxMin` before re-attempting it. Read at the scoring call site + /// (IntelligenceEngine) and threaded through `AnalyticsEngine.analyzeDay`. Mirrors the Android + /// `PuffinExperiment.KEY_SLEEP_WAKE_BRIDGE`. + static let wakeBridgeKey = "noopSleepWakeBridge" + + static var wakeBridgeEnabled: Bool { UserDefaults.standard.bool(forKey: wakeBridgeKey) } + /// Opt-in "WHOOP MG ECG (experimental)": unlock the gated, user-initiated ECG ("Labrador") probe that /// asks an MG strap to start its ECG subsystem. Default OFF, and OFF is not merely the default — with /// this key false the four ECG opcodes are dropped by the BLEManager allowlist, so no ECG byte can diff --git a/Strand/BLE/StalledHandshakePolicy.swift b/Strand/BLE/StalledHandshakePolicy.swift new file mode 100644 index 0000000000..1688344c92 --- /dev/null +++ b/Strand/BLE/StalledHandshakePolicy.swift @@ -0,0 +1,63 @@ +import Foundation + +/// Recovery decision for a link that CoreBluetooth calls connected but that never became usable. +/// +/// The failure this exists for: iOS state restoration hands back a peripheral whose `state` is already +/// `.connected`, so `willRestoreState` marks the link up, seeds the bond flags (a restored link WAS bonded +/// once) and re-discovers services — but the CLIENT_HELLO / bond write that follows is `.withResponse`, and +/// if the strap is wedged or dead it is never acked. `didWriteValueFor` therefore never fires, which is the +/// ONLY place that sets `connectHandshakeDone` **and the only place that calls `startKeepAlive()`** — so the +/// 120 s liveness watchdog inside `keepAliveFire` (the machinery that would have bounced exactly this link) +/// is never armed. The app then retries the same dead handle forever: an on-device strap log showed +/// "Backfill: deferred — connect handshake not done yet" repeating every ~1–3 min for 40+ minutes, straight +/// through the strap coming off its charger, and cost the user a whole night of biometrics. +/// +/// So the recovery is deliberately NOT a new reconnect mechanism: it cancels the peripheral connection and +/// lets `didDisconnectPeripheral`'s existing 3 s rescan (the same recovery the liveness watchdog uses at its +/// own fuse) bring the link back. This type only decides WHETHER to pull that trigger. Pure + no +/// CoreBluetooth, so the decision is unit-testable without a radio. +enum StalledHandshakePolicy { + + /// How long a link may sit connected-but-unproven before we cancel it. + /// + /// Chosen against the fuses this codebase already runs, not invented: + /// * a healthy handshake resolves in well under a second (the post-ack work defers the offload by 1.5 s), + /// so 90 s is ~60× the expected latency — it can never clip a strap that is merely slow; + /// * it sits ABOVE the 60 s reboot settle backstop, so a reboot-driven reconnect resolves itself first + /// (`noteRebootReconnectIfNeeded`) instead of being pre-empted by a bounce; + /// * it sits BELOW the 120 s `keepAliveFire` liveness fuse, so on any link that IS delivering data the + /// data-driven watchdog stays the one making the call — `shouldRecover`'s `sawData` guard defers to it + /// anyway, so the two can never fight over the same link. + static let fuseSeconds: TimeInterval = 90 + + /// Cancel-and-rescan this link? + /// + /// Every input is a reason to STAY OUT of the way, which is the point — this fires only on the narrow + /// case nothing else owns: + /// - `connected`: the link already dropped → the normal disconnect path owns the rescan. + /// - `handshakeDone`: the connect handshake completed → this is a healthy link, never touch it. + /// - `sawData`: ANY notification arrived since the link came up (0x2A37 HR, a puffin frame, a battery + /// read). The strap is talking to us and is useful even if the puffin handshake lags, and the 120 s + /// liveness fuse already covers a link that goes quiet later. Bouncing here would throw away a working + /// HR stream — the regression this guard exists to prevent. + /// - `intentionalDisconnect`: a user teardown is in flight. + /// - `autoReconnectPaused`: the #617/#747 bond-loop give-up deliberately STOPPED hammering this strap and + /// told the user to free it; a bounce here would restart exactly the loop that machinery just paused. + /// - `bondRefused`: the #78/#221 refusal state ("Connected · not paired") is a diagnosed, surfaced, + /// user-actionable condition with its own guidance on the Devices card. It is not a stall, and churning + /// the link would only hide the guidance behind reconnect noise. + static func shouldRecover(connected: Bool, + handshakeDone: Bool, + sawData: Bool, + intentionalDisconnect: Bool, + autoReconnectPaused: Bool, + bondRefused: Bool) -> Bool { + guard connected else { return false } + guard !handshakeDone else { return false } + guard !sawData else { return false } + guard !intentionalDisconnect else { return false } + guard !autoReconnectPaused else { return false } + guard !bondRefused else { return false } + return true + } +} diff --git a/Strand/BLE/StrapBatteryReadPolicy.swift b/Strand/BLE/StrapBatteryReadPolicy.swift new file mode 100644 index 0000000000..2504846c87 --- /dev/null +++ b/Strand/BLE/StrapBatteryReadPolicy.swift @@ -0,0 +1,49 @@ +import Foundation +import WhoopProtocol + +/// Whether to (re)issue a standard Battery Level (0x2A19) READ on the live link. +/// +/// The failure this exists for: the ONLY 0x2A19 read was a one-shot fired from +/// `didDiscoverCharacteristicsFor` — i.e. PRE-bond, on an unencrypted link. A WHOOP 5/MG rejects that read +/// ("Authentication is insufficient", the same refusal that gates its notify subscriptions), the failure +/// landed in `didUpdateValueFor`'s error branch, was logged, and was then DROPPED. Nothing ever retried it. +/// `enableLiveNotifications` — whose own comment claims it recovers the "standard HR/battery that failed +/// pre-bond" — only ever re-issued `setNotifyValue`, never `readValue`. That asymmetry is the whole bug: HR +/// recovers because it is notify-driven and the strap pushes it at 1 Hz, so the UI says "Live" while the +/// battery percent stays `nil` for the entire session. On a strap that then died on the user's wrist, the +/// app could not show a single battery reading all day. +/// +/// FAMILY-CORRECT BY CONSTRUCTION (#77): a WHOOP 4.0's 0x2A19 is a firmware STUB that always reports 100, +/// so reading it would flash a fake 100% over the real charge. The 4.0's true value comes only from the +/// proprietary `GET_BATTERY_LEVEL` command (COMMAND_RESPONSE, u16/10) that the keep-alive already sends +/// every ~60 s. Refusing the read for `.whoop4` here is what keeps that invariant intact, and it is pinned +/// by `StrapBatteryReadPolicyTests` so a future edit can't quietly re-open #77. +enum StrapBatteryReadPolicy { + + /// Floor between reads, matching the keep-alive tick (`BLEManager.keepAliveIntervalSeconds`). + /// + /// The throttle is load-bearing, not politeness: `didWriteValueFor` re-enters the 5/MG post-bond branch + /// on EVERY `.withResponse` ack — including every HISTORY_END ack during a multi-minute historical + /// offload — and that branch calls `enableLiveNotifications` unconditionally. The existing work there is + /// idempotent-cheap (`where !c.isNotifying` collapses to a no-op once subscribed); an unthrottled + /// `readValue` would NOT be, and would fire hundreds of ATT reads at a strap mid-sync. With the floor, + /// the steady-state cadence is one read per keep-alive tick, which is strictly lighter than the 4.0's + /// existing every-60 s `GET_BATTERY_LEVEL` poll. + static let minIntervalSeconds: TimeInterval = 30 + + /// - Parameters: + /// - family: the connected strap's family. `.whoop4` always refuses (see #77 above). + /// - canRead: the characteristic exists and advertises `.read`. + /// - lastReadAt: when we last issued a read this session; nil = never, which always reads (this is the + /// self-heal — the first post-bond call re-issues the read the pre-bond link refused). + /// - now: monotonic-ish seconds; injected so the throttle is testable without a clock. + static func shouldRead(family: DeviceFamily, + canRead: Bool, + lastReadAt: TimeInterval?, + now: TimeInterval) -> Bool { + guard family != .whoop4 else { return false } // #77: the 4.0's 0x2A19 is a stub-100 + guard canRead else { return false } + guard let last = lastReadAt else { return true } + return now - last >= minIntervalSeconds + } +} diff --git a/Strand/BLE/StrapChargeInference.swift b/Strand/BLE/StrapChargeInference.swift new file mode 100644 index 0000000000..23aa066e18 --- /dev/null +++ b/Strand/BLE/StrapChargeInference.swift @@ -0,0 +1,99 @@ +import Foundation +import StrandAnalytics + +/// The charging state the app is willing to STAND BEHIND — the strap's reported flag, corroborated (or +/// rescued) by the one thing that cannot lie: the state-of-charge going UP. +/// +/// Why this exists. The `battery_charging` flag is a single bit the decoder reads at a FIXED offset in the +/// BATTERY_LEVEL event (4.0 @26, 5.0 @30). For the 5.0 that offset is **unverified**: `decodeWhoop5Event`'s +/// own documentation states the deci-percent SoC was "confirmed by a clean monotonic discharge across a +/// real capture (49.9 → 47.7 %)" and makes no such claim for the charge bit — it is derived from the same +/// "+4 rule" as the other fields and has never been checked against a labelled charging session. Field +/// evidence says it is wrong or at least incomplete: a 5/MG sitting on its charging puck reported +/// `battery_charging = 0` while its SoC climbed. The `ch <= 1` guard fails closed on a wild byte, but it +/// cannot catch a byte that is plausibly 0 and simply means something else. +/// +/// So the app used to state "not charging" — and, worse, show a "~2 days left" DISCHARGE estimate — about a +/// strap that was visibly on the charger. That's an unverified bit being reported as fact. +/// +/// The fix is to stop treating the bit as the only witness. A rising SoC is direct, family-independent, +/// decoder-independent evidence of a charge, and this codebase ALREADY trusts exactly that inference: +/// `BatteryEstimator.chargeStepPct` is documented as "a SoC rise larger than this (percentage points) +/// between two consecutive readings marks a CHARGE", and the runtime estimate has always restarted its +/// discharge run on it. This reuses that same threshold rather than inventing a second notion of "charging". +/// +/// Strictly additive: a confirmed `true` is still `true`, and inference can only ever ADD a charge, never +/// deny one. If neither witness speaks, it answers `nil` (unknown) rather than asserting "not charging". +/// +/// ──────────────────────────────────────────────────────────────────────────────────────────────────── +/// KNOWN ISSUE — the premise is CONFIRMED, but this threshold probably never fires. NOT YET FIXED. +/// +/// Checked against the cloud mirror on 2026-07-26 (GitHub issues are disabled on this fork, so the +/// finding lives here and in docs/bugs/2026-07-15-…md §A6 rather than in a tracker). +/// +/// CONFIRMED — the bit really is wrong. One charging session with unambiguous ground truth +/// (CHARGING_ON ×10, BATTERY_PACK_CONNECTED ×3; 2026-07-15 11:07→13:34 MDT; SoC 4.7% → 100.0%) carries +/// `charging = false` on every one of its ~300 readings. Not `nil` — FALSE, i.e. the BATTERY_LEVEL path +/// decoded the bit and the strap asserted "not charging" while going from nearly flat to full. +/// +/// BUT — `isRising` requires > `chargeStepPct` (1.0 pp) between CONSECUTIVE readings, and that charge +/// stepped +0.3…+0.7 pp per reading at a ~30 s cadence. It never once cleared 1.0. The +/// `recentRiseWindowSeconds` comment below assumes the opposite ("~8 min apart"), which is what makes +/// 1.0 pp look like a low bar; at the real cadence it is not. The live path looks no better: a 5/MG's +/// live SoC is whole-percent 0x2A19, so a charge steps by exactly +1, and the strict `>` that exists to +/// survive that quantisation also excludes exactly 1.0. +/// +/// So `chargingEffective` likely degenerates to the raw flag and this type is inert for the very case it +/// was written for. It is not HARMFUL — the inference is additive, so behaviour equals pre-fix — but it +/// is not doing its job either. Do not read the passing tests as coverage: the synthetic samples in +/// `StrapChargeInferenceTests` step by more than 1.0 pp, which is exactly why they pass. +/// +/// LIKELY FIX: make it a RATE test (pp per unit time) instead of a fixed per-reading delta. A WHOOP 5 +/// charges at ~50 pp/h against a ~1.65 pp/h discharge — a ~30x separation that holds at any cadence, +/// where a fixed step does not. +/// +/// UNMEASURED, and needed before picking a constant: the `battery` table this came from is fed by the +/// decoded OFFLOAD stream, not the live path that actually populates `LiveState.batterySamples`. The +/// ~30 s cadence is therefore evidence about banked records; the LIVE cadence and quantisation are still +/// unconfirmed and want a real charging session to settle. +/// ──────────────────────────────────────────────────────────────────────────────────────────────────── +enum StrapChargeInference { + + /// A rise must be observed within this window to count as "charging NOW". The strap emits BATTERY_LEVEL + /// every ~8 min, so two consecutive readings are ~8 min apart; 20 min allows a missed event without + /// letting an hour-old charge keep claiming the strap is still on the puck. + static let recentRiseWindowSeconds = 20 * 60 + + /// Is the SoC series RISING fast enough to only be explicable as a charge? + /// + /// Uses `BatteryEstimator.chargeStepPct` (1.0 pp) between the two most recent readings, strictly + /// greater-than — which is what makes it robust against the quantisation seam: a 5/MG's live SoC comes + /// from 0x2A19 as a whole-percent u8, so consecutive readings can legitimately step by exactly 1.0 pp + /// without a charge. A real charge is nowhere near that subtle: a WHOOP 5 charges at roughly 50 pp/h + /// against a ~1.65 pp/h discharge, so a genuine charge moves several points per ~8-minute event. + static func isRising(samples: [(ts: Int, soc: Double)], + nowUnix: Int, + windowSeconds: Int = recentRiseWindowSeconds) -> Bool { + guard samples.count >= 2 else { return false } + let sorted = samples.sorted { $0.ts < $1.ts } + guard let last = sorted.last, let prev = sorted.dropLast().last else { return false } + guard nowUnix - last.ts <= windowSeconds else { return false } // stale: says nothing about NOW + guard last.ts - prev.ts <= windowSeconds else { return false } // a gap that wide isn't a slope + return (last.soc - prev.soc) > BatteryEstimator.chargeStepPct + } + + /// The charging state to display and to gate the runtime estimate on. + /// + /// - `true` — the strap said so, OR its charge is measurably climbing. + /// - `false` — the strap said not-charging AND the SoC is not climbing. Two witnesses agreeing. + /// - `nil` — no flag yet and no rise seen. Unknown, and the UI should say nothing rather than imply + /// "not charging" (which is exactly the false statement this whole type exists to stop). + static func resolve(flag: Bool?, + samples: [(ts: Int, soc: Double)], + nowUnix: Int, + windowSeconds: Int = recentRiseWindowSeconds) -> Bool? { + if flag == true { return true } + if isRising(samples: samples, nowUnix: nowUnix, windowSeconds: windowSeconds) { return true } + return flag // false (corroborated by a non-rising SoC) or nil (nothing to say) + } +} diff --git a/Strand/CloudSync/CloudEdit.swift b/Strand/CloudSync/CloudEdit.swift new file mode 100644 index 0000000000..fc8d19735c --- /dev/null +++ b/Strand/CloudSync/CloudEdit.swift @@ -0,0 +1,27 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation + +/// One row of the noop-cloud server's append-only edit journal. Field names match the server's +/// JSON keys exactly (camelCase), so no CodingKeys translation is needed. +struct CloudEdit: Decodable, Equatable { + let seq: Int + let editId: String + let kind: String + let payloadJSON: String + let beforeJSON: String? + let rationale: String? + let appliedAt: Int + let undoneBySeq: Int? + let ackedAt: Int? +} + +/// The `GET /edits` response envelope: the page of journal rows since the requested cursor, plus +/// the server's latest known seq (so the caller can detect it's fully caught up). +struct CloudEditsResponse: Decodable { + let edits: [CloudEdit] + let latestSeq: Int +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudEditApplier.swift b/Strand/CloudSync/CloudEditApplier.swift new file mode 100644 index 0000000000..fa5e91549c --- /dev/null +++ b/Strand/CloudSync/CloudEditApplier.swift @@ -0,0 +1,427 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import WhoopStore +import StrandAnalytics + +/// Outcome of applying one batch of `CloudEdit` journal rows to the local store. +/// +/// `appliedSeqs` collects EVERY processed seq — applied, skipped, AND needsAttention alike, never +/// just the successful ones: the sync cursor must advance past whatever this device has SEEN, not +/// just what it acted on, or an edit this device can never act on (an unknown forward-compat kind, a +/// permanently malformed payload) would wedge the cursor forever. `needsAttention` is a SUBSET of +/// processed edits that made no store change and need a human look (see `CloudEditApplier`'s per-kind +/// rules below); it is reported separately so the sync status can say so, but its seq is still acked. +struct CloudApplySummary: Equatable { + var applied = 0 + var skipped = 0 + var needsAttention = 0 + var touchedSleep = false + var touchedWorkouts = false + var touchedHr = false + var appliedSeqs: [Int] = [] +} + +/// Applies confirmed noop-cloud edit-journal rows to the on-device `WhoopStore` through the SAME +/// recompute-safe entry points a human on-device correction uses — `applySleepEdit`/ +/// `updateSleepStages` (which set `userEdited = 1`), never a raw UPDATE — so a later strap re-sync or +/// nightly recompute can't silently revert a cloud correction (`upsertSleepSessions`'s ON CONFLICT +/// preserves an edited row's bounds/stages exactly because that flag is set). Deletions pair a +/// physical DELETE with a `cloudTombstone` row (`CloudTombstoneStore`) so a later re-import/backfill +/// can't resurrect them either. Corrections/additions with no natural "my-whoop" identity (a +/// manually-added workout, a fixed workout's replacement copy) land under the synthetic +/// `"noop-cloud"` source, registered in the device registry once per batch, the first time it's +/// actually needed (mirrors `OuraSyncWriter.persist`'s step 0). +/// +/// Never throws: a malformed payload, an unknown (forward-compat) kind, or a store-level failure are +/// all folded into the summary (`skipped`/`needsAttention`) rather than propagated — one bad journal +/// row must never abort the whole batch. +enum CloudEditApplier { + /// The synthetic paired-device id every "no natural home" write (an add, a fix's replacement + /// copy) lands under — distinct from any real strap/import source id. + static let cloudDeviceId = "noop-cloud" + + static func apply(_ edits: [CloudEdit], store: WhoopStore) async -> CloudApplySummary { + var summary = CloudApplySummary() + var registeredCloudDevice = false + + for edit in edits.sorted(by: { $0.seq < $1.seq }) { + // Every processed row is acked, whatever happened to it — see the type doc above. + summary.appliedSeqs.append(edit.seq) + + // An undo marker, an edit some LATER row already undid, or the server-only baseline note + // all need no on-device action. + if edit.kind == "undo" || edit.undoneBySeq != nil || edit.kind == "set_baseline_note" { + summary.skipped += 1 + continue + } + + let payload = Data(edit.payloadJSON.utf8) + switch edit.kind { + case "adjust_sleep_bounds": + await applyAdjustSleepBounds(payload, store: store, summary: &summary) + case "edit_sleep_stages": + await applyEditSleepStages(payload, store: store, summary: &summary) + case "delete_workout": + await applyDeleteWorkout(payload, seq: edit.seq, store: store, summary: &summary) + case "fix_workout": + await applyFixWorkout(payload, beforeJSON: edit.beforeJSON, seq: edit.seq, store: store, + summary: &summary, registeredCloudDevice: ®isteredCloudDevice) + case "add_workout": + await applyAddWorkout(payload, store: store, summary: &summary, + registeredCloudDevice: ®isteredCloudDevice) + case "delete_hr_range": + await applyDeleteHrRange(payload, seq: edit.seq, store: store, summary: &summary) + case "delete_metric_point": + await applyDeleteMetricPoint(payload, seq: edit.seq, store: store, summary: &summary) + default: + summary.skipped += 1 // unknown kind: forward-compat, not an error + } + } + return summary + } + + // MARK: - adjust_sleep_bounds + + private struct AdjustSleepBoundsPayload: Decodable { + let deviceId: String + let startTs: Int + let newStartTs: Int? + let newEndTs: Int? + } + + private static func applyAdjustSleepBounds(_ data: Data, store: WhoopStore, + summary: inout CloudApplySummary) async { + guard let p = try? JSONDecoder().decode(AdjustSleepBoundsPayload.self, from: data) else { + summary.skipped += 1 + return + } + do { + guard let current = try await store.sleepSessions(deviceId: p.deviceId, from: p.startTs, + to: p.startTs, limit: 1).first else { + summary.needsAttention += 1 // session absent — nothing to adjust + return + } + // #940 belt-and-braces, mirrored from Repository.editSleepTimes: never persist a future- + // ending or inverted window, whatever the server payload says. A malicious/buggy server + // mirror or a stale queued edit must not be able to write a phantom night the display + // merge can't render — refuse (needsAttention) rather than clamp-and-apply silently. + guard let window = SleepEditGuard.clampedEditWindow( + start: p.newStartTs ?? current.effectiveStartTs, end: p.newEndTs ?? current.endTs, + now: Int(Date().timeIntervalSince1970)) else { + summary.needsAttention += 1 + return + } + _ = try await store.applySleepEdit( + deviceId: p.deviceId, detectedStartTs: p.startTs, + newStartTs: window.start, newEndTs: window.end) + summary.applied += 1 + summary.touchedSleep = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - edit_sleep_stages + + private struct SleepStagePayload: Decodable { + let start: Int + let end: Int + let stage: String + } + + private struct EditSleepStagesPayload: Decodable { + let deviceId: String + let startTs: Int + let stages: [SleepStagePayload] + } + + /// The noop-cloud server's stage vocabulary is "awake"/"light"/"deep"/"rem" (its `sleepStage` zod + /// enum, `noop-cloud/src/edits/kinds.ts`). The on-device stagesJSON legend uses "wake" for the + /// same concept — see `WearableSleepStageInterval`'s doc comment ("Normalized stage name written + /// into the stage JSON: 'deep' / 'light' / 'rem' / 'wake'") and `SleepStagerV2`'s own + /// `labels[i] == "awake" ? "wake" : labels[i]` conversion when it writes a detected session's + /// stages. "light"/"deep"/"rem" already match verbatim; only "awake" needs remapping so a + /// re-encoded edit reads identically to every other stage source (e.g. + /// `WearableImporter.efficiency`'s `stage != "wake"` check, `WhoopCsvExporter`'s + /// `("wake" == awake)` doc note). + private static func mapServerStage(_ stage: String) -> String { + stage == "awake" ? "wake" : stage + } + + /// Re-encode to the on-device `[{start,end,stage}]` shape (same JSON layout `OuraSyncWriter` + /// writes), mapping the stage vocabulary. nil for an empty list — a stage edit must carry at + /// least one segment; an empty array is treated as a malformed payload upstream. + private static func encodeStagesJSON(_ stages: [SleepStagePayload]) -> String? { + guard !stages.isEmpty else { return nil } + let segs = stages.map { + ["start": $0.start, "end": $0.end, "stage": mapServerStage($0.stage)] as [String: Any] + } + return (try? JSONSerialization.data(withJSONObject: segs)).flatMap { String(data: $0, encoding: .utf8) } + } + + private static func applyEditSleepStages(_ data: Data, store: WhoopStore, + summary: inout CloudApplySummary) async { + guard let p = try? JSONDecoder().decode(EditSleepStagesPayload.self, from: data), + let stagesJSON = encodeStagesJSON(p.stages) else { + summary.skipped += 1 + return + } + do { + guard let current = try await store.sleepSessions(deviceId: p.deviceId, from: p.startTs, + to: p.startTs, limit: 1).first else { + summary.needsAttention += 1 // session absent — nothing to restage + return + } + // updateSleepStages only touches userEdited=1 rows; promote first (CURRENT bounds, + // stages left untouched by this call) so an un-edited night can still receive a stage + // correction — see the brief's explicit two-step note. This promotion also calls + // applySleepEdit with a bounds pair, so it gets the same #940 guard as adjust_sleep_bounds + // above: a stored row with corrupt (inverted/future) bounds — e.g. a bad import that never + // went through the guard — must not be silently promoted into a userEdited phantom night. + if !current.userEdited { + guard let window = SleepEditGuard.clampedEditWindow( + start: current.effectiveStartTs, end: current.endTs, + now: Int(Date().timeIntervalSince1970)) else { + summary.needsAttention += 1 + return + } + _ = try await store.applySleepEdit(deviceId: p.deviceId, detectedStartTs: p.startTs, + newStartTs: window.start, newEndTs: window.end) + } + _ = try await store.updateSleepStages(deviceId: p.deviceId, detectedStartTs: p.startTs, + stagesJSON: stagesJSON) + // Stage-lock: a cloud stage edit is an AUTHORITATIVE correction, not a placeholder to be + // re-derived. Without this, `Repository.selfHealEditedStages` (run by every analyzeRecent) + // restages the night from raw and — since a deliberate correction always differs from the + // stager's output — silently reverts it (found live: 14 applied restages clobbered by the + // first recompute). The lock lives in the `cursors` table (no schema change); a backup + // restored without it self-heals, because the journal re-pull re-applies + re-locks. + try? await store.setCursor("stagelock:\(p.deviceId):\(p.startTs)", 1) + summary.applied += 1 + summary.touchedSleep = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - delete_workout + + private struct WorkoutKeyPayload: Decodable { + let deviceId: String + let startTs: Int + let sport: String + } + + private static func applyDeleteWorkout(_ data: Data, seq: Int, store: WhoopStore, + summary: inout CloudApplySummary) async { + guard let p = try? JSONDecoder().decode(WorkoutKeyPayload.self, from: data) else { + summary.skipped += 1 + return + } + do { + // Destructive step LAST: tombstone before the physical delete, so a mid-chain throw + // degrades to "the row is still visible but future re-imports/backfills are blocked" — + // never to "the delete happened but nothing recorded it, so a later re-import silently + // resurrects the workout the user just deleted." A crash between these two calls is the + // worst case this ordering must survive, and it does: re-processing the same journal row + // (addTombstone is idempotent by (kind, editSeq)) simply finishes the delete. + try await store.addTombstone(kind: "workout", deviceId: p.deviceId, startTs: p.startTs, endTs: nil, + sport: p.sport, day: nil, key: nil, editSeq: seq) + _ = try await store.deleteWorkouts(deviceId: p.deviceId, sport: p.sport, from: p.startTs, to: p.startTs) + summary.applied += 1 + summary.touchedWorkouts = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - fix_workout + + private struct FixWorkoutPatch: Decodable { + let sport: String? + let startTs: Int? + let endTs: Int? + let energyKcal: Double? + let distanceM: Double? + } + + private struct FixWorkoutPayload: Decodable { + let deviceId: String + let startTs: Int + let sport: String + let patch: FixWorkoutPatch + } + + /// Lenient snapshot of the server mirror's workout row (the `beforeJSON` the journal carries for + /// `fix_workout`). Every field optional: the mirror's exact row shape isn't this applier's + /// contract to enforce, and only `endTs` (when `patch` doesn't supply its own) is actually + /// load-bearing — everything else has a sane fallback or is itself optional on `WorkoutRow`. + private struct FixWorkoutBefore: Decodable { + let source: String? + let endTs: Int? + let energyKcal: Double? + let distanceM: Double? + } + + private static func applyFixWorkout(_ data: Data, beforeJSON: String?, seq: Int, store: WhoopStore, + summary: inout CloudApplySummary, + registeredCloudDevice: inout Bool) async { + guard let p = try? JSONDecoder().decode(FixWorkoutPayload.self, from: data) else { + summary.skipped += 1 + return + } + guard let beforeJSON, + let before = try? JSONDecoder().decode(FixWorkoutBefore.self, from: Data(beforeJSON.utf8)) else { + summary.needsAttention += 1 // no server snapshot to reconstruct the corrected copy from + return + } + let finalStartTs = p.patch.startTs ?? p.startTs + guard let finalEndTs = p.patch.endTs ?? before.endTs else { + summary.needsAttention += 1 // neither the patch nor the snapshot carries an end time + return + } + do { + // Same-batch race guard: a delete_workout for this exact key can sort BEFORE this row + // (lower seq) in the SAME batch and have already removed the original. Without this check + // the upsert below would resurrect it under noop-cloud, silently undoing a delete the user + // just made. + guard try await store.workoutExists(deviceId: p.deviceId, startTs: p.startTs, sport: p.sport) else { + let alreadyDeleted = try await store.workoutTombstones(deviceId: p.deviceId) + .contains { $0.startTs == p.startTs && $0.sport == p.sport } + if alreadyDeleted { + summary.skipped += 1 // an earlier/same-batch delete already won — nothing to fix + } else { + summary.needsAttention += 1 // gone with no tombstone — stale server mirror + } + return + } + + // Destructive step LAST: tombstone the original, write the corrected copy, THEN delete the + // original — so a mid-chain throw degrades to "an extra/duplicate row is visible", never to + // "the original is gone and no correction exists" (see the ordering rationale on + // applyDeleteWorkout above). The tombstone keys the ORIGINAL deviceId, so it can never + // block the noop-cloud upsert below (upsertWorkouts' resurrection guard only checks + // tombstones for the deviceId it is writing INTO). + try await store.addTombstone(kind: "workout", deviceId: p.deviceId, startTs: p.startTs, endTs: nil, + sport: p.sport, day: nil, key: nil, editSeq: seq) + + try await registerCloudDeviceIfNeeded(store: store, registeredCloudDevice: ®isteredCloudDevice) + let corrected = WorkoutRow( + startTs: finalStartTs, endTs: finalEndTs, sport: p.patch.sport ?? p.sport, + source: before.source ?? "manual", durationS: Double(finalEndTs - finalStartTs), + energyKcal: p.patch.energyKcal ?? before.energyKcal, avgHr: nil, maxHr: nil, strain: nil, + distanceM: p.patch.distanceM ?? before.distanceM, zonesJSON: nil, notes: nil) + _ = try await store.upsertWorkouts([corrected], deviceId: cloudDeviceId) + + _ = try await store.deleteWorkouts(deviceId: p.deviceId, sport: p.sport, from: p.startTs, to: p.startTs) + + summary.applied += 1 + summary.touchedWorkouts = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - add_workout + + private struct AddWorkoutPayload: Decodable { + let startTs: Int + let endTs: Int + let sport: String + let energyKcal: Double? + let distanceM: Double? + let notes: String? + } + + private static func applyAddWorkout(_ data: Data, store: WhoopStore, summary: inout CloudApplySummary, + registeredCloudDevice: inout Bool) async { + guard let p = try? JSONDecoder().decode(AddWorkoutPayload.self, from: data) else { + summary.skipped += 1 + return + } + do { + try await registerCloudDeviceIfNeeded(store: store, registeredCloudDevice: ®isteredCloudDevice) + let row = WorkoutRow(startTs: p.startTs, endTs: p.endTs, sport: p.sport, source: "manual", + durationS: Double(p.endTs - p.startTs), energyKcal: p.energyKcal, + avgHr: nil, maxHr: nil, strain: nil, distanceM: p.distanceM, + zonesJSON: nil, notes: p.notes) + _ = try await store.upsertWorkouts([row], deviceId: cloudDeviceId) + summary.applied += 1 + summary.touchedWorkouts = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - delete_hr_range + + private struct DeleteHrRangePayload: Decodable { + let deviceId: String + let fromTs: Int + let toTs: Int + } + + private static func applyDeleteHrRange(_ data: Data, seq: Int, store: WhoopStore, + summary: inout CloudApplySummary) async { + guard let p = try? JSONDecoder().decode(DeleteHrRangePayload.self, from: data) else { + summary.skipped += 1 + return + } + do { + // Destructive step LAST — see the ordering rationale on applyDeleteWorkout above. + try await store.addTombstone(kind: "hrRange", deviceId: p.deviceId, startTs: p.fromTs, endTs: p.toTs, + sport: nil, day: nil, key: nil, editSeq: seq) + _ = try await store.deleteHrRange(deviceId: p.deviceId, fromTs: p.fromTs, toTs: p.toTs) + summary.applied += 1 + summary.touchedHr = true + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - delete_metric_point + + private struct DeleteMetricPointPayload: Decodable { + let deviceId: String + let day: String + let key: String + } + + private static func applyDeleteMetricPoint(_ data: Data, seq: Int, store: WhoopStore, + summary: inout CloudApplySummary) async { + guard let p = try? JSONDecoder().decode(DeleteMetricPointPayload.self, from: data) else { + summary.skipped += 1 + return + } + do { + // Destructive step LAST — see the ordering rationale on applyDeleteWorkout above. + try await store.addTombstone(kind: "metricPoint", deviceId: p.deviceId, startTs: nil, endTs: nil, + sport: nil, day: p.day, key: p.key, editSeq: seq) + _ = try await store.deleteMetricPoint(deviceId: p.deviceId, day: p.day, key: p.key) + summary.applied += 1 + // No sleep/workouts/hr flag applies to a generic metric point. + } catch { + summary.needsAttention += 1 + } + } + + // MARK: - noop-cloud device registration + + /// Registers the synthetic `"noop-cloud"` paired-device row the first time this batch actually + /// writes under it — mirrors `OuraSyncWriter.persist`'s step 0. `DeviceRegistryStore.add` is an + /// upsert (`ON CONFLICT(id) DO UPDATE`), so re-registering across separate batches/runs is + /// harmless; `registeredCloudDevice` only avoids repeat writes WITHIN one `apply` call. + private static func registerCloudDeviceIfNeeded(store: WhoopStore, registeredCloudDevice: inout Bool) async throws { + guard !registeredCloudDevice else { return } + let now = Int(Date().timeIntervalSince1970) + let device = PairedDevice(id: cloudDeviceId, brand: cloudDeviceId, model: "Cloud edits", + sourceKind: .cloudImport, capabilities: [], status: .paired, + addedAt: now, lastSeenAt: now) + try DeviceRegistryStore(dbQueue: store.registryWriter).add(device) + registeredCloudDevice = true + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncAppDelegate.swift b/Strand/CloudSync/CloudSyncAppDelegate.swift new file mode 100644 index 0000000000..75ac65f1b6 --- /dev/null +++ b/Strand/CloudSync/CloudSyncAppDelegate.swift @@ -0,0 +1,120 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +#if os(iOS) +import Foundation +import UIKit + +/// Minimal `UIApplicationDelegate` shim (Cloud Sync v2) — `StrandiOSApp` is otherwise a pure SwiftUI +/// `App` with no AppDelegate at all (confirmed: no `@UIApplicationDelegateAdaptor` existed before this). +/// APNs device-token registration and silent (content-available) push delivery are UIKit AppDelegate +/// callbacks with no SwiftUI equivalent — `.onReceive`/`.task` can't observe them — so this is the +/// ONLY reason an AppDelegate exists in this app. CLOUD_SYNC + iOS gated: a default build never links +/// any of this push-notification plumbing at all. +/// +/// CODE-COMPLETE, CREDENTIALS LATER: `NOOP.entitlements` does NOT carry `aps-environment` in this +/// branch (deliberately — the integrator adds it when APNs credentials/signing are ready), so +/// `registerForRemoteNotifications()` will fail every launch until that lands. +/// `didFailToRegisterForRemoteNotificationsWithError` below is expected to fire on every run of this +/// build for exactly that reason. +@MainActor +final class CloudSyncAppDelegate: NSObject, UIApplicationDelegate { + /// Set by `StrandiOSApp.init()` (alongside the `registerForRemoteNotifications()` call) before any + /// delegate callback below can plausibly fire, so `didReceiveRemoteNotification` can run the same + /// gated background sync `CloudSyncBackgroundRefresh` uses — this class has no other way to reach + /// the shared `AppModel`/`Repository`/`IntelligenceEngine` (a `UIApplicationDelegateAdaptor` + /// instance is constructed by SwiftUI, not by `StrandiOSApp.init()`, so it can't be handed the + /// model at init time the way `CloudSyncBackgroundRefresh.register(model:)` is). + static var model: AppModel? + + /// APNs handed us a device token: hex-encode it (APNs' own wire format) and hand it to the + /// server's `/register-device` endpoint. Best-effort and silent either way — see + /// `CloudSyncClient.registerDevice`'s doc comment for why the server 404 case specifically is + /// EXPECTED (it ships in parallel with this client) rather than a real failure. + func application(_ application: UIApplication, + didRegisterForRemoteNotificationsWithDeviceToken deviceToken: Data) { + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { return } + let hexToken = deviceToken.map { String(format: "%02x", $0) }.joined() + // Breadcrumb (readable via the app container's preferences plist): which leg of + // token→POST last ran, so a silent registration failure is diagnosable without a debugger. + UserDefaults.standard.set("token-received \(Date())", forKey: Self.registrationBreadcrumbKey) + let client = CloudSyncClient(baseURL: url, token: token) + Task { + do { + try await client.registerDevice(token: hexToken) + UserDefaults.standard.set("registered \(Date())", forKey: Self.registrationBreadcrumbKey) + } catch CloudSyncError.badResponse(404, _) { + // Expected during rollout: the server's /register-device endpoint ships in parallel + // with this client. Benign — a fresh install, a token refresh (APNs rotates tokens + // occasionally), or simply the next relaunch after the server catches up will retry + // this same call, so there's nothing to recover here. + UserDefaults.standard.set("post-404 \(Date())", forKey: Self.registrationBreadcrumbKey) + } catch { + // Any other failure (network down, non-404 error): log-not-crash, matching every other + // CloudSync network call's posture — push registration is a nice-to-have, never + // something worth surfacing to the user or aborting launch over. + UserDefaults.standard.set("post-error: \(error) \(Date())", forKey: Self.registrationBreadcrumbKey) + } + } + } + + /// UserDefaults key for the registration breadcrumb above — diagnosis-only, never read by code. + static let registrationBreadcrumbKey = "cloudsync.push.lastRegistration" + + /// Request APNs registration HERE — the canonical didFinishLaunching site — not from + /// `StrandiOSApp.init()`: a request made during the SwiftUI `App` struct's init runs BEFORE the + /// app finishes launching, and UIKit silently ignores it (found live: no token callback and no + /// failure callback ever fired; the breadcrumb below never advanced past absent). + func application(_ application: UIApplication, + didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil) -> Bool { + if CloudSyncSettings.effectiveURL != nil, CloudSyncSettings.effectiveToken != nil { + UserDefaults.standard.set("requested \(Date())", forKey: Self.registrationBreadcrumbKey) + application.registerForRemoteNotifications() + } + return true + } + + /// Expected on EVERY launch of this branch (see the type's doc comment: no `aps-environment` + /// entitlement yet). Silent — there is no user-facing push-status UI to update, and the + /// BGAppRefreshTask + on-launch catch-up paths cover the same ground without APNs at all. + func application(_ application: UIApplication, didFailToRegisterForRemoteNotificationsWithError error: Error) { + UserDefaults.standard.set("os-fail: \(error.localizedDescription) \(Date())", + forKey: Self.registrationBreadcrumbKey) + } + + /// A silent (content-available) push arrived: run the SAME gated background sync the + /// `BGAppRefreshTask` handler uses (`CloudSyncBackgroundRefresh.handle`), then report the fetch + /// result iOS expects. A push is a stronger signal than the 4h background-refresh gate ("a + /// specific edit is now confirmed and waiting"), but it still goes through + /// `backgroundSyncIfDue`'s SAME `CloudSyncGate`/staleness gate rather than an unconditional sync — + /// a burst of pushes (or a redelivered one) must not be able to fire overlapping syncs any more + /// than two backgrounded refreshes could. `.newData`/`.noData` is a coarse signal (this handler has + /// no per-byte insight into what changed): `.newData` whenever a sync was actually ATTEMPTED, + /// `.noData` when a guard sent it home early (e.g. the 4h gate hasn't elapsed, or CloudSync isn't + /// configured on this device). + func application(_ application: UIApplication, didReceiveRemoteNotification userInfo: [AnyHashable: Any], + fetchCompletionHandler completionHandler: @escaping (UIBackgroundFetchResult) -> Void) { + guard let model = Self.model else { + completionHandler(.noData) + return + } + let cloudSync = CloudSyncModel() + let intelligence = model.intelligence + let repository = model.repo + cloudSync.postApplyRefresh = { + await intelligence.analyzeRecent() + await repository.refresh() + } + Task { + // Unconditional (CloudSyncGate-guarded) — see `syncFromPush`'s doc comment: the 4h + // staleness gate belongs to the OS-initiated BGAppRefresh path, not to an explicit + // server-side request for fresh data. + let ran = await cloudSync.syncFromPush(repo: repository) + completionHandler(ran ? .newData : .noData) + } + } +} +#endif // os(iOS) +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncBackgroundRefresh.swift b/Strand/CloudSync/CloudSyncBackgroundRefresh.swift new file mode 100644 index 0000000000..e85287bda1 --- /dev/null +++ b/Strand/CloudSync/CloudSyncBackgroundRefresh.swift @@ -0,0 +1,125 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +#if os(iOS) +import Foundation +import BackgroundTasks + +/// Registers and schedules the CLOUD_SYNC background-refresh `BGAppRefreshTask` (Cloud Sync v2) — the +/// iOS-only counterpart to `CloudSyncModel.autoSyncIfDue`'s on-launch catch-up, giving a backgrounded +/// app a chance to pick up confirmed edits (and re-upload changed data) without waiting for the next +/// foreground launch. Mirrors `ScheduledDebugExport`'s BGTask plumbing +/// (Strand/System/ScheduledDebugExport.swift) closely; the difference is this handler needs a real +/// `Repository`/`IntelligenceEngine` to run an actual sync (wired from the `AppModel` the app entry +/// point supplies), where that one only ever writes a local file. +/// +/// HONEST about platform limits, same posture as `ScheduledDebugExport`: iOS decides IF and WHEN a +/// submitted request actually runs — this is a best-effort extra chance to catch up sooner than the +/// guaranteed 20h on-launch catch-up, never a promise of timely background delivery. +@MainActor +enum CloudSyncBackgroundRefresh { + /// Must also appear in the iOS target's `BGTaskSchedulerPermittedIdentifiers` (project.yml) and be + /// registered at launch (`register(model:)`, called from `StrandiOSApp.init()`) for `schedule()` to + /// have any effect — see `ScheduledDebugExport.bgTaskIdentifier`'s doc comment for the identical + /// contract. + /// Resolved from the bundle, not hardcoded — same reason as `ScheduledDebugExport.bgTaskIdentifier`: + /// the permitted entries are namespaced under the bundle id, so a personal-deploy remap silently + /// stales a literal and takes this whole lane down with it. `BGTaskIdentifier` lives in + /// `Strand/System/` (NOT here) so that fix stays independent of this fork-only directory. + static let bgTaskIdentifier = BGTaskIdentifier.resolve( + suffix: "cloudsync.refresh", upstream: "com.noopapp.noop.cloudsync.refresh") + + /// Register the BGTask handler. MUST be called from the app's launch (before launch finishes), + /// same constraint as `ScheduledDebugExport.register()`. Safe to leave uncalled, or to have + /// `schedule()`'s submit fail silently: the sync this handler runs is exactly what `autoSyncIfDue` + /// already covers on the next foreground launch anyway — this is a strictly additive, best-effort + /// earlier chance, never a dependency for correctness. + static func register(model: AppModel) { + let registered = BGTaskScheduler.shared.register(forTaskWithIdentifier: bgTaskIdentifier, using: nil) { task in + // The launch handler runs on a queue the system chooses, NOT necessarily the main actor — + // hop explicitly before touching `CloudSyncModel` (`@MainActor`) or `AppModel`. + Task { @MainActor in + handle(task, model: model) + } + } + // Same contract as `ScheduledDebugExport.register()`: a false return means this identifier isn't + // permitted and the 4-hourly background sync will never be delivered. Never swallow it. + BGTaskIdentifier.report(bgTaskIdentifier, registered: registered) + } + + /// Submit a background-refresh request for *no earlier than* + /// `CloudSyncModel.backgroundSyncIntervalS` from now. Called when the app backgrounds + /// (`StrandiOSApp`'s `scenePhase` handler) — deliberately NOT at launch/register time, so a user + /// who keeps the app foregrounded for hours doesn't burn the window before ever backgrounding. + /// `try?` swallows the "identifier not permitted/registered" error so a build that hasn't wired + /// Info.plist yet still behaves (mirrors `ScheduledDebugExport.submitBackgroundRequest`). + static func schedule() { + let request = BGAppRefreshTaskRequest(identifier: bgTaskIdentifier) + request.earliestBeginDate = Date(timeIntervalSinceNow: CloudSyncModel.backgroundSyncIntervalS) + try? BGTaskScheduler.shared.submit(request) + } + + /// The delivered task's actual work: wire `CloudSyncModel` exactly like `RootTabView`'s on-launch + /// catch-up does (same `postApplyRefresh` closure, same `model.intelligence`/`model.repo`), run the + /// 4h-gated background sync, then unconditionally reschedule the next request and report + /// completion. Rescheduling happens FIRST, before the sync even starts, so the next window is + /// guaranteed to be requested whatever happens below — BGAppRefresh is single-shot, and nothing + /// else will ever submit the next one. + private static func handle(_ task: BGTask, model: AppModel) { + schedule() + + let cloudSync = CloudSyncModel() + let intelligence = model.intelligence + let repository = model.repo + cloudSync.postApplyRefresh = { + await intelligence.analyzeRecent() + await repository.refresh() + } + + // Exactly one `task.setTaskCompleted` call per delivered task, whichever finishes first: the + // sync itself, or iOS's expiration warning. `BGTaskCompletion` is a plain (non-actor-isolated) + // class precisely so the expiration handler below — invoked by the system on a thread of ITS + // choosing, not guaranteed to be the main actor — can call it directly with no `await`. + let completion = BGTaskCompletion(task) + let work = Task { + _ = await cloudSync.backgroundSyncIfDue(repo: repository) + completion.complete(success: true) + } + task.expirationHandler = { + // Cooperative cancellation: `backgroundSyncIfDue`'s network calls observe this and unwind + // via `URLError.cancelled`/`CancellationError`, which `performSync`'s `catch` already folds + // into a status line like any other failure — never crash-safe territory, just a sync that + // reports "cancelled" instead of a result. `Task.cancel()` is safe to call from any thread. + work.cancel() + completion.complete(success: false) + } + } +} + +/// Guarantees `BGTask.setTaskCompleted` is called exactly once for a given task, however many of the +/// (up to two) call sites race to finish first. Deliberately NOT `@MainActor` or an `actor`: iOS invokes +/// `BGTask.expirationHandler` on a thread of its own choosing (undocumented, not guaranteed to be the +/// main actor), and that closure is synchronous (`() -> Void`, no `await` available) — so the guard +/// itself has to be usable synchronously from an arbitrary thread. `NSLock` provides that directly; the +/// sync `Task`'s own completion (which runs on the main actor, inherited from `handle`) calls the SAME +/// synchronous, lock-guarded method with no isolation mismatch either way. +private final class BGTaskCompletion { + private let task: BGTask + private let lock = NSLock() + private var completed = false + + init(_ task: BGTask) { + self.task = task + } + + func complete(success: Bool) { + lock.lock() + defer { lock.unlock() } + guard !completed else { return } + completed = true + task.setTaskCompleted(success: success) + } +} +#endif // os(iOS) +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncClient.swift b/Strand/CloudSync/CloudSyncClient.swift new file mode 100644 index 0000000000..5fab409cee --- /dev/null +++ b/Strand/CloudSync/CloudSyncClient.swift @@ -0,0 +1,238 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation + +/// User-facing failure reasons for the cloud-sync lane, mapped to clear, non-crashing messages. +/// Mirrors OuraError's shape (LocalizedError + errorDescription). +enum CloudSyncError: LocalizedError, Equatable { + case badResponse(Int, String) + case network(String) + case decode + + var errorDescription: String? { + switch self { + case .badResponse(let code, let detail): + let extra = detail.isEmpty ? "" : " — \(detail)" + return "The cloud sync server returned an error (\(code))\(extra)." + case .network(let d): + return "Network problem talking to the cloud sync server: \(d)" + case .decode: + return "Couldn't read the cloud sync server's response." + } + } +} + +/// The noop-cloud edit-journal client. Injected `session` makes it URLProtocol-testable. +/// Networking lives here in the app target by design (mirrors Strand/Oura/OuraAPIClient.swift). +final class CloudSyncClient { + /// Internal rather than private so the liters push path can address the *same* server with the + /// *same* credential this client already holds. Deliberately not a second settings read: a + /// server URL that has been through `CloudSyncModel.saveSettings`' validation and a token that + /// came out of the Keychain are exactly what `/ingest` is using, and the two paths must never + /// be able to disagree about where "the server" is. + let baseURL: URL + let token: String + private let session: URLSession + + init(baseURL: URL, token: String, session: URLSession = .shared) { + self.baseURL = baseURL + self.token = token + self.session = session + } + + /// GET /edits?since=, Bearer-authed. Returns the page of rows plus the server's latest seq. + func fetchEdits(since: Int) async throws -> (edits: [CloudEdit], latestSeq: Int) { + var req = URLRequest(url: url(path: "edits", query: ["since": String(since)])) + req.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + let (data, status) = try await send(req) + guard (200..<300).contains(status) else { + throw CloudSyncError.badResponse(status, bodyPrefix(data)) + } + guard let decoded = try? JSONDecoder().decode(CloudEditsResponse.self, from: data) else { + throw CloudSyncError.decode + } + return (decoded.edits, decoded.latestSeq) + } + + /// POST /edits/ack {"seqs":[...]}, Bearer-authed. Returns the count the server actually acked. + func ack(seqs: [Int]) async throws -> Int { + var req = URLRequest(url: url(path: "edits/ack")) + req.httpMethod = "POST" + req.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + req.httpBody = try JSONEncoder().encode(AckRequest(seqs: seqs)) + let (data, status) = try await send(req) + guard (200..<300).contains(status) else { + throw CloudSyncError.badResponse(status, bodyPrefix(data)) + } + guard let decoded = try? JSONDecoder().decode(AckResponse.self, from: data) else { + throw CloudSyncError.decode + } + return decoded.acked + } + + // MARK: - Ingest (Phase 3.5: upload this device's own .noopbak) + + /// POST /ingest, Bearer-authed, with the file at `fileURL` streamed straight from disk as a raw + /// `application/octet-stream` body via `URLSession.upload(for:fromFile:)` — the whole-DB `.noopbak` + /// can be 100-300MB, so it must never be loaded into memory as `Data` the way `send(_:)`'s + /// `session.data(for:)` would. Parses `{ok,bytes,latestDay}`; only `bytes`/`latestDay` are modelled + /// (Decodable ignores the unmodelled `ok` key — it's redundant with the 2xx status check anyway). + func ingest(fileURL: URL) async throws -> (bytes: Int, latestDay: String?) { + var req = URLRequest(url: url(path: "ingest")) + req.httpMethod = "POST" + req.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + req.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") + // #tz-upload: the upload body is the whole SQLite DB (epoch-UTC timestamps only), so the server + // has no way to know which zone this phone is in right now. Ship the CURRENT IANA identifier as a + // header the server stores on the ingestLog row (and surfaces via data_freshness) — this answers + // "what zone is the phone in as of this upload" even before the per-day `phoneTimezone` table + // lands in a given mirror. + req.setValue(TimeZone.current.identifier, forHTTPHeaderField: "X-Phone-Timezone") + let (data, status) = try await sendUpload(req, fromFile: fileURL) + guard (200..<300).contains(status) else { + throw CloudSyncError.badResponse(status, bodyPrefix(data)) + } + guard let decoded = try? JSONDecoder().decode(IngestResponse.self, from: data) else { + throw CloudSyncError.decode + } + return (decoded.bytes, decoded.latestDay) + } + + // MARK: - Device registration (Cloud Sync v2: APNs silent-push wake) + + /// POST /register-device, Bearer-authed, `{"token":"","platform":"ios"}` — hands the server + /// this device's APNs device token so it can wake this device with a silent (content-available) + /// push the moment a NEW edit is confirmed elsewhere, instead of only ever reaching it on the next + /// background-refresh/launch poll. `token` is the hex-encoded APNs device token (the caller, + /// `CloudSyncAppDelegate`, does the hex-encoding — this method just ships whatever string it's + /// given). + /// + /// The server endpoint ships in parallel with this client (code-complete ahead of it), so a 404 + /// here is EXPECTED for a while — see `CloudSyncAppDelegate`'s call site for why that's treated as + /// benign rather than a real failure. Same uniform error contract as every other call on this + /// client (`CloudSyncError.badResponse` for any non-2xx status): no status-code special-casing + /// happens in here, that judgment call belongs to the caller. + func registerDevice(token: String) async throws { + var req = URLRequest(url: url(path: "register-device")) + req.httpMethod = "POST" + req.setValue("Bearer \(self.token)", forHTTPHeaderField: "Authorization") + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + req.httpBody = try JSONEncoder().encode(RegisterDeviceRequest(token: token, platform: "ios")) + let (data, status) = try await send(req) + guard (200..<300).contains(status) else { + throw CloudSyncError.badResponse(status, bodyPrefix(data)) + } + } + + // MARK: - Deep buffers (#423: WHOOP 5/MG high-rate offload archive) + + /// POST /deepbuf, Bearer-authed — one line-aligned, raw-DEFLATE-compressed byte range of + /// `PuffinDeepBufferLog`'s append-only JSONL. See `DeepBufferUploadPlan` for why the unit of upload + /// is `(generation, byteStart, byteEnd)` rather than a whole file. + /// + /// `compressed` is passed as `Data`, NOT streamed from a file the way `ingest` above must be: a + /// chunk is ~4 MB and is ALREADY in memory (it was just read and deflated), so spilling it to a + /// temp file purely to hand `upload(for:fromFile:)` a path would add two disk round-trips to buy + /// nothing. `ingest`'s whole-DB body is 100-300 MB, which is a different problem. + /// + /// The compression is announced in a CUSTOM header rather than `Content-Encoding: deflate` + /// deliberately: `Content-Encoding` invites any proxy in the path (and Express itself) to helpfully + /// decode the body, which would silently defeat the whole point of compressing on the phone and + /// leave the server parsing bytes it thinks are plaintext. A header nothing but this endpoint reads + /// cannot be helpfully misinterpreted by anything in between. + func uploadDeepBuffer(generation: String, byteStart: Int, byteEnd: Int, + compressed: Data) async throws -> DeepBufferUploadReceipt { + var req = URLRequest(url: url(path: "deepbuf")) + req.httpMethod = "POST" + req.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + req.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") + req.setValue("deflate-raw", forHTTPHeaderField: "X-Deepbuf-Compression") + req.setValue(generation, forHTTPHeaderField: "X-Deepbuf-Generation") + req.setValue(String(byteStart), forHTTPHeaderField: "X-Deepbuf-Byte-Start") + req.setValue(String(byteEnd), forHTTPHeaderField: "X-Deepbuf-Byte-End") + // Same #tz-upload convention as `ingest` above: every timestamp inside the payload is epoch-UTC, + // so the phone's current zone is only knowable if it says so. + req.setValue(TimeZone.current.identifier, forHTTPHeaderField: "X-Phone-Timezone") + let (data, status) = try await sendUpload(req, from: compressed) + guard (200..<300).contains(status) else { + throw CloudSyncError.badResponse(status, bodyPrefix(data)) + } + guard let decoded = try? JSONDecoder().decode(DeepBufferResponse.self, from: data) else { + throw CloudSyncError.decode + } + return DeepBufferUploadReceipt(storedBytes: decoded.storedBytes, lines: decoded.lines, + duplicate: decoded.duplicate ?? false) + } + + private func sendUpload(_ req: URLRequest, fromFile fileURL: URL) async throws -> (Data, Int) { + do { + let (data, resp) = try await session.upload(for: req, fromFile: fileURL) + return (data, (resp as? HTTPURLResponse)?.statusCode ?? 0) + } catch { + throw CloudSyncError.network(error.localizedDescription) + } + } + + private func sendUpload(_ req: URLRequest, from body: Data) async throws -> (Data, Int) { + do { + let (data, resp) = try await session.upload(for: req, from: body) + return (data, (resp as? HTTPURLResponse)?.statusCode ?? 0) + } catch { + throw CloudSyncError.network(error.localizedDescription) + } + } + + private func url(path: String, query: [String: String] = [:]) -> URL { + var comps = URLComponents(string: "\(baseURL.absoluteString)/\(path)")! + if !query.isEmpty { + comps.queryItems = query.map { URLQueryItem(name: $0.key, value: $0.value) } + } + return comps.url! + } + + private func send(_ req: URLRequest) async throws -> (Data, Int) { + do { + let (data, resp) = try await session.data(for: req) + return (data, (resp as? HTTPURLResponse)?.statusCode ?? 0) + } catch { + throw CloudSyncError.network(error.localizedDescription) + } + } + + /// Truncates the response body to a bounded prefix for error messages (never surface an + /// unbounded server body through an error description). + private func bodyPrefix(_ data: Data) -> String { + let s = String(data: data, encoding: .utf8) ?? "" + return s.count > 200 ? String(s.prefix(200)) : s + } +} + +private struct AckRequest: Encodable { + let seqs: [Int] +} + +private struct AckResponse: Decodable { + let acked: Int +} + +private struct RegisterDeviceRequest: Encodable { + let token: String + let platform: String +} + +private struct IngestResponse: Decodable { + let bytes: Int + let latestDay: String? +} + +/// `/deepbuf`'s reply. `duplicate` is optional because the server omits it on the normal path (it is +/// only present, and true, when the chunk was already in the manifest) — see `DeepBufferUploadReceipt`. +private struct DeepBufferResponse: Decodable { + let storedBytes: Int + let lines: Int + let duplicate: Bool? +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncCoordinator.swift b/Strand/CloudSync/CloudSyncCoordinator.swift new file mode 100644 index 0000000000..ba282e5715 --- /dev/null +++ b/Strand/CloudSync/CloudSyncCoordinator.swift @@ -0,0 +1,46 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import WhoopStore + +/// Abstracts the edit-journal fetch/ack calls so the coordinator is testable without a live network +/// (CloudSyncClient conforms below) — mirrors OuraSyncCoordinator's `OuraPageFetching` seam. +protocol CloudEditFetching { + func fetchEdits(since: Int) async throws -> (edits: [CloudEdit], latestSeq: Int) + func ack(seqs: [Int]) async throws -> Int +} +extension CloudSyncClient: CloudEditFetching {} + +/// Pulls confirmed noop-cloud edit-journal rows since this device's last cursor, applies them through +/// `CloudEditApplier`, and acknowledges what it processed. A pure coordination step — no state of its +/// own, so (like `CloudEditApplier`) it's a namespace of static functions rather than an instance. +enum CloudSyncCoordinator { + /// The `cursors` table name this lane's pull position is stored under. + static let cursorName = "cloud_edits" + + /// since = this device's stored cursor (0 if never synced) → fetch the page after it → apply + /// every row through `CloudEditApplier` → ack every seq the applier processed (applied, skipped, + /// AND needsAttention alike — see `CloudApplySummary.appliedSeqs`'s doc) → ONLY THEN advance the + /// cursor to the server's latest seq. + /// + /// The ack-then-cursor ordering is load-bearing: if `ack` throws (a transient network failure), + /// the cursor must stay at `since` so the NEXT pull re-fetches and re-applies the same batch — + /// safe only because `CloudEditApplier`'s writes are idempotent upserts/tombstones, never raw + /// appends. Advancing the cursor first would risk acknowledging-by-implication a batch the server + /// never actually heard back from us about, silently losing it forever. + /// + /// An empty page (nothing new since `since`) is a no-op: `appliedSeqs` is empty, so neither `ack` + /// nor the cursor write happens — there is nothing to acknowledge and the cursor is already correct. + static func pull(store: WhoopStore, client: any CloudEditFetching) async throws -> CloudApplySummary { + let since = try await store.cursor(cursorName) ?? 0 + let (edits, latestSeq) = try await client.fetchEdits(since: since) + let summary = await CloudEditApplier.apply(edits, store: store) + guard !summary.appliedSeqs.isEmpty else { return summary } + _ = try await client.ack(seqs: summary.appliedSeqs) + try await store.setCursor(cursorName, latestSeq) + return summary + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncModel.swift b/Strand/CloudSync/CloudSyncModel.swift new file mode 100644 index 0000000000..86d20a1996 --- /dev/null +++ b/Strand/CloudSync/CloudSyncModel.swift @@ -0,0 +1,464 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import WhoopStore +#if os(iOS) +import UIKit +#endif + +/// Process-wide reentrancy guard for the cloud-sync lane. `busy` on a `CloudSyncModel` INSTANCE only +/// guards within that one instance — but `autoSyncIfDue` runs on a throwaway `CloudSyncModel()` +/// created fresh in `RootView`/`RootTabView`'s launch `.task` (a new instance every launch), while +/// `DataSourcesView` holds its OWN long-lived `@StateObject` instance. Two different instances means +/// two different `busy` flags: a manual "Sync now" tap on the Data Sources card can't see that the +/// launch-time auto-sync (on its own throwaway instance) is already mid-upload, and vice versa — both +/// would happily run `CloudSyncUploader.upload` concurrently against the same on-disk store and the +/// same server. An `actor` serializes access to its own state automatically, making it the natural +/// process-wide singleton shape here — `shared` is the one gate for the whole process. +actor CloudSyncGate { + static let shared = CloudSyncGate() + private var inFlight = false + + /// Claims the gate if it's free. Returns false (and claims nothing) if a sync is already running. + func begin() -> Bool { + if inFlight { return false } + inFlight = true + return true + } + + /// Releases the gate. Must be called exactly once for every `begin()` that returned true. + func end() { + inFlight = false + } +} + +/// Drives the Data Sources "Cloud Sync" card: save the noop-cloud server URL + token → pull confirmed +/// edits → apply them locally → (iOS) re-export whatever changed into Apple Health. @MainActor so every +/// `@Published` mutation is main-thread; the network/apply/write-back work hops off-main. +/// +/// `repo: Repository` is taken as a call-time PARAMETER (not stored at construction), for the same +/// reason as `OuraConnectModel`: `Repository` only reaches `DataSourcesView` via `@EnvironmentObject`, +/// which SwiftUI populates after `init()` runs. +@MainActor +final class CloudSyncModel: ObservableObject { + @Published var busy = false + @Published var statusText: String? + /// True once both a server URL and a token are stored in the Keychain (`CloudSyncSettings`). + @Published var isConfigured = CloudSyncSettings.isConfigured + + /// Recompute-and-refresh hook the wiring point supplies (IntelligenceEngine.analyzeRecent + + /// Repository.refresh). Cloud edits rewrite stored sessions/workouts, but the daily rollups the + /// UI headlines read (`dailyMetric.totalSleepMin`, efficiency, Rest/Charge) are only rebuilt by + /// the analyze cycle — which a pull does NOT trigger, so applied edits left the Sleep/Today + /// numbers stale until the next strap-data analysis (found live: 14 restaged nights applied while + /// the headline kept the pre-edit total). `performSync` invokes this whenever the applied-edits + /// cursor is ahead of `recomputedCursorName`. + var postApplyRefresh: (() async -> Void)? + + /// Cursor (same `cursors` table as `CloudSyncCoordinator.cursorName`) recording the highest + /// journal seq the rollups were recomputed for. Cursor-vs-cursor — not `summary.applied > 0` — + /// so a build that gains this hook AFTER edits were already applied still heals on its first sync. + static let recomputedCursorName = "cloud_edits_recomputed" + + /// Validates `url` parses as an absolute URL (scheme + host) via `URLComponents` before saving. + /// `CloudSyncClient.url(path:query:)` force-unwraps its `URLComponents`-built request URL, so a + /// garbage base URL saved here would crash later at the first pull instead of failing this save + /// with a clear, recoverable message (flagged in the Task-3 review of `CloudSyncClient`). + func saveSettings(url: String, token: String) { + let trimmedURL = url.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmedToken = token.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedURL.isEmpty, !trimmedToken.isEmpty else { + statusText = "Enter both a server URL and a token." + return + } + guard let comps = URLComponents(string: trimmedURL), let scheme = comps.scheme, !scheme.isEmpty, + let host = comps.host, !host.isEmpty else { + statusText = "That doesn't look like a valid server URL (needs e.g. https://host)." + return + } + CloudSyncSettings.serverURL = trimmedURL + CloudSyncSettings.token = trimmedToken + isConfigured = CloudSyncSettings.isConfigured + statusText = "Saved." + } + + /// Pull confirmed edits since this device's cursor, apply them, and — on iOS — re-export whatever + /// changed into Apple Health. `repo.refresh()` runs unconditionally at the end (not just on + /// success): `CloudEditApplier` writes land in the local store BEFORE `ack`, so even a pull that + /// ultimately throws (e.g. the ack call itself failing) can leave real local changes the UI needs + /// to pick up. Uses the EFFECTIVE credentials (`CloudSyncSettings.effectiveURL`/`effectiveToken`) + /// so a bundle-only configuration (Phase 3.5, no Keychain save) works here too, not just in + /// `syncNow`. + func pullNow(repo: Repository) { + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { + statusText = "Add your noop-cloud server URL and token first." + return + } + setBusy(true); statusText = "Pulling edits…" + Task { + var line: String + if let store = await repo.storeHandle() { + let client = CloudSyncClient(baseURL: url, token: token) + do { + let summary = try await CloudSyncCoordinator.pull(store: store, client: client) + line = "Applied \(summary.applied) · skipped \(summary.skipped) · \(summary.needsAttention) need attention" + #if os(iOS) && canImport(HealthKit) + line += await reExportToAppleHealthIfNeeded(repo: repo, store: store, summary: summary) + #endif + } catch { + line = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + } + } else { + line = "No local store." + } + statusText = line + await repo.refresh() + setBusy(false) + } + } + + /// Pull + apply any confirmed edits, then upload this device's own `.noopbak` to the noop-cloud + /// server (skipped when nothing has changed since the last upload — see `performSync`'s doc + /// comment) — the two halves of "zero-touch" in one round trip: any edits confirmed elsewhere reach + /// this device, and this device's own (possibly just-corrected) data reaches the server. Stamps + /// `UserDefaults["cloudsync.lastAutoSync"]` on SUCCESS ONLY, so a failed attempt (server + /// unreachable, network down) never blocks the next 20h-gated retry. Used by the user-initiated + /// "Sync now" button; `autoSyncIfDue`'s on-launch catch-up runs the identical `performSync` body + /// under its own gated `Task` rather than calling this (see `runGatedSync`'s doc comment) — but a + /// manual tap still resets the SAME on-launch catch-up clock (`Self.lastAutoSyncKey`, written + /// inside `performSync`), so it doesn't immediately re-fire on the next launch. + func syncNow(repo: Repository) { + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { + statusText = "Add your noop-cloud server URL and token first." + return + } + Task { await runGatedSync(repo: repo, url: url, token: token) } + } + + /// Acquires the process-wide `CloudSyncGate`, runs `performSync`, and always releases it — + /// written out as one linear sequence rather than `defer` (which can't `await`). This is safe + /// specifically because `performSync` never throws and never returns early once it starts: every + /// internal failure is caught inside it and folded into its own status-line string (see its + /// body), so falling off the end of this function really does mean "released on every exit path." + /// Shared by `syncNow` and `autoSyncIfDue` — each spawns its own `Task` calling this — rather than + /// having `autoSyncIfDue` acquire the gate and then call `syncNow` (which would try to acquire it + /// again and immediately see it as already held by the very call chain it's part of, bailing with + /// "Sync already in progress" on every auto-sync). + private func runGatedSync(repo: Repository, url: URL, token: String) async { + guard await CloudSyncGate.shared.begin() else { + statusText = "Sync already in progress" + return + } + await performSync(repo: repo, url: url, token: token) + await CloudSyncGate.shared.end() + } + + /// The actual pull + upload work, gated by `runGatedSync` — never called directly. Byte-identical + /// to `syncNow`'s pre-gate Task body, plus persisting the outcome for the Data Sources card to + /// read back (Finding 2: `CloudSyncModel.persistLastStatus`, below). + /// + /// PULL-FIRST, UPLOAD-LAST (Cloud Sync v2 — reordered from the original upload-then-pull): pulling + /// and applying confirmed edits BEFORE this device uploads its own `.noopbak` means the upload + /// captures the just-corrected local state, so the server's mirror reflects THIS sync's edits + /// immediately — the original upload-first order sent the PRE-edit-application snapshot, so an + /// edit applied this sync only reached the server's mirror on the NEXT sync's upload, one full + /// cycle late. One consequence: a pull failure now aborts the whole sync before the upload ever + /// runs (previously the upload, which ran first, could still succeed even when the following pull + /// failed). That trade-off is intentional — an upload that ran anyway would still ship the stale + /// pre-edit state, exactly the problem this reorder fixes. + /// + /// SKIP-UNCHANGED UPLOAD: the upload only actually runs when `WhoopStore.contentToken()` (computed + /// AFTER pull/recompute, so a just-applied edit already counts) differs from the token saved after + /// the last successful upload FROM THIS DEVICE. A full re-export can be hundreds of MB; a device + /// that synced an hour ago with no new samples and no applied edits has nothing new to ship. + private func performSync(repo: Repository, url: URL, token: String) async { + setBusy(true); statusText = "Pulling edits…" + // Hoisted out of the `if let store` below (where it used to be built) so the deep-buffer drain + // at the end of this method can reach it: that drain reads a FILE, not the store, so it must + // still run on a device whose store handle is unavailable. + let client = CloudSyncClient(baseURL: url, token: token) + var line: String + var succeeded = false + if let store = await repo.storeHandle() { + do { + let summary = try await CloudSyncCoordinator.pull(store: store, client: client) + var parts = ["Applied \(summary.applied)", "skipped \(summary.skipped)"] + if summary.needsAttention > 0 { parts.append("\(summary.needsAttention) attention") } + line = parts.joined(separator: " · ") + statusText = line + + // Rebuild the daily rollups before the Apple Health re-export, so the vitals it + // exports (which read dailyMetric) reflect the just-applied edits too. + var didRecompute = false + if let refresh = postApplyRefresh { + let applied = (try? await store.cursor(CloudSyncCoordinator.cursorName)) ?? 0 + let recomputed = (try? await store.cursor(Self.recomputedCursorName)) ?? 0 + if applied > recomputed { + statusText = "Recomputing days touched by edits…" + await refresh() + try? await store.setCursor(Self.recomputedCursorName, applied) + didRecompute = true + parts.append("recomputed") + line = parts.joined(separator: " · ") + } + } + #if os(iOS) && canImport(HealthKit) + // force after a recompute: the vitals the write-back exports read dailyMetric, so a + // rollup rebuild needs a re-export even when THIS pull itself applied nothing. + line += await reExportToAppleHealthIfNeeded(repo: repo, store: store, summary: summary, + force: didRecompute) + #endif + + // Skip-unchanged upload (Cloud Sync v2): computed AFTER pull/recompute above, so an + // edit this device just applied already makes the token differ from the last upload's + // — the corrected state uploads THIS sync, matching the pull-first reorder's whole + // point (see this method's doc comment). `try?`: a token-computation failure must not + // abort an otherwise-successful sync — it just falls through to "upload anyway", the + // same safe default as a first-ever sync (no saved token to compare against). + let freshToken = try? await store.contentToken() + let lastUploadToken = UserDefaults.standard.string(forKey: Self.lastUploadTokenKey) + if let freshToken, freshToken == lastUploadToken { + parts.append("Up to date (nothing new to upload)") + line = parts.joined(separator: " · ") + } else { + statusText = "Uploading…" + let uploaded = try await CloudSyncUploader.upload(store: store, client: client) + let mb = Double(uploaded.bytes) / 1_048_576.0 + parts.append(String(format: "Uploaded %.1f MB", mb)) + line = parts.joined(separator: " · ") + if let freshToken { + UserDefaults.standard.set(freshToken, forKey: Self.lastUploadTokenKey) + } + } + succeeded = true + } catch { + line = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + } + } else { + line = "No local store." + } + + // Deep-buffer drain (#423) — deliberately OUTSIDE the do/catch above, and NOT gated on + // `succeeded`. The edit-journal lane and the deep-buffer archive fail independently and are not + // equally recoverable: a pull that throws (server down, bad token, a 500 on /edits) costs + // nothing permanent — the edits are still on the server and the next sync gets them. The deep + // buffers are the ONLY copy of packets the strap's ack-trim has already freed, and the log + // destroys its own oldest generation on rotation, so bytes not shipped are eventually gone for + // good. Letting an unrelated /edits failure skip the drain would trade unrecoverable data for + // nothing. Cheap when idle: two `stat`s and a watermark compare, no network at all. + statusText = "Shipping deep buffers…" + if let deep = await drainDeepBuffers(client: client) { line += " · \(deep)" } + + statusText = line + if succeeded { + UserDefaults.standard.set(Date().timeIntervalSince1970, forKey: Self.lastAutoSyncKey) + } + Self.persistLastStatus(line) + await repo.refresh() + setBusy(false) + } + + /// Ship whatever new bytes `PuffinDeepBufferLog` has appended since this device's last confirmed + /// chunk, returning a status-line SUFFIX (nil when there was nothing to do, so an idle drain adds + /// no noise to the card). + /// + /// UNGATED, unlike every other lane here — no 20h/4h staleness clock. The clock that matters is the + /// log's own: the strap banks ~40 KB/s of hex during an offload, so a generation fills the soft cap + /// in hours and the generation after that DELETES it. A staleness gate tuned for "has the user's + /// data changed" is the wrong instrument for "is the archive about to eat itself". Running every + /// sync costs two `stat` calls when there is nothing new (see `DeepBufferUploader.drain`), so + /// there is nothing to gate away. + private func drainDeepBuffers(client: CloudSyncClient) async -> String? { + let summary = await DeepBufferUploader.drain(client: client, + watermarks: DeepBufferUserDefaultsWatermarks()) + if let error = summary.error { return "Deep buffers: \(error)" } + guard !summary.isEmpty else { return nil } + let sentMB = Double(summary.sentBytes) / 1_048_576.0 + let rawMB = Double(summary.rawBytes) / 1_048_576.0 + var s = String(format: "Deep buffers %.1f→%.1f MB (%d chunks)", rawMB, sentMB, summary.chunks) + if summary.moreRemaining { s += " · more queued" } + return s + } + + /// UserDefaults key for the last successful `syncNow` (manual or automatic) — read by + /// `autoSyncIfDue`, written by `performSync` on success only. + private static let lastAutoSyncKey = "cloudsync.lastAutoSync" + + /// UserDefaults key for the `WhoopStore.contentToken()` value as of the last successful upload + /// FROM THIS DEVICE (Cloud Sync v2 skip-unchanged upload) — written by `performSync` only when an + /// upload actually ran and succeeded, never on a skip (a skip means the saved token is STILL + /// accurate) or a failure (a failed upload never shipped that token's content, so the NEXT sync + /// must still compare against whatever WAS last actually uploaded). Deliberately a `UserDefaults` + /// key, like `lastAutoSyncKey`/`lastStatusKey` above, not a `WhoopStore` cursor: it is per-device + /// sync bookkeeping about an UPLOAD this device performed, not data the store itself owns, and + /// `cursors` only stores `Int` values anyway (this token is a composite string). + private static let lastUploadTokenKey = "cloudsync.lastUploadToken" + + /// UserDefaults key for the outcome of the last sync attempt, success OR failure, persisted by + /// `performSync` on EVERY completion (Finding 2). Read directly by the Data Sources card so an + /// auto-sync that ran on a throwaway `CloudSyncModel` instance (see `CloudSyncGate`'s doc comment) + /// is still visible on the card's own instance, whose `statusText` never saw it happen. + private static let lastStatusKey = "cloudsync.lastStatus" + + /// The last sync's outcome, formatted for display (" · "), or nil if no sync + /// has ever completed on this device. A plain synchronous UserDefaults read — deliberately not + /// reactive/`@Published`: `performSync` always ends with a `repo.refresh()`, which already + /// re-renders the screens that observe `repo`, so the next render picks up a fresh value with no + /// extra observation machinery. + static var lastPersistedStatus: String? { + UserDefaults.standard.string(forKey: lastStatusKey) + } + + private static let shortTimeFormatter: DateFormatter = { + let f = DateFormatter() + f.dateStyle = .none + f.timeStyle = .short + return f + }() + + /// Persist `line` (the human-readable outcome `performSync` just computed) plus a short local + /// time, so a later launch's Data Sources card can show "Last sync: …" even when ITS OWN + /// `statusText` is nil — e.g. right after launch, before that screen's own `CloudSyncModel` + /// `@StateObject` has run a sync itself. + private static func persistLastStatus(_ line: String) { + UserDefaults.standard.set("\(line) · \(shortTimeFormatter.string(from: Date()))", forKey: lastStatusKey) + } + + /// Pure 20h gate, `(lastRun, now) -> Bool`, so the threshold is unit-testable without UserDefaults, + /// a store, or a network (see `CloudSyncModelAutoSyncGateTests`). `lastRun == 0` (never synced) is + /// always due — a fresh install syncs on its very first launch, matching "zero manual taps". + /// `nonisolated`: touches no actor-isolated state, so it's callable synchronously from a plain + /// (non-`@MainActor`, non-`async`) XCTest method — without this, it inherits the enclosing class's + /// `@MainActor` and a test would need `await MainActor.run { … }` just to call a pure function. + nonisolated static func isAutoSyncDue(lastRun: TimeInterval, now: TimeInterval, + intervalS: TimeInterval = 20 * 3600) -> Bool { + now - lastRun >= intervalS + } + + /// True when this process is running inside an XCTest test runner. Guards `autoSyncIfDue` against + /// the incident where the macOS TEST HOST — `StrandTests` runs inside the full `Staging.app` via + /// `TEST_HOST`, not a bare test bundle — executed `RootView`'s launch `.task` for real during a + /// test run. Bundle credentials were present, so `CloudSyncModel().autoSyncIfDue(...)` fired and + /// auto-uploaded the Mac's (empty) test database, replacing the production mirror. Two checks, + /// either sufficient on its own: `XCTestConfigurationFilePath` is set by Xcode/`xcodebuild test` + /// for the process actually running the tests; `NSClassFromString("XCTestCase") != nil` is + /// belt-and-suspenders for the TEST_HOST shape, where XCTest is merely linked into the process. + /// Only checked in `autoSyncIfDue` — a manual "Sync now" tap (`syncNow`) stays allowed under test, + /// same as in production, because a human (or an explicit UI-driven test) triggered it on purpose. + /// `nonisolated`, same reason as `isAutoSyncDue`: `CloudSyncModel` is `@MainActor`, and that + /// isolation applies to `static` members by default — without this a plain synchronous XCTest + /// method couldn't call it directly. + nonisolated static var isRunningUnderXCTest: Bool { + ProcessInfo.processInfo.environment["XCTestConfigurationFilePath"] != nil + || NSClassFromString("XCTestCase") != nil + } + + /// On-launch daily catch-up (Phase 3.5: zero-touch): silently no-ops when the lane isn't configured + /// — `CloudSyncSettings.isConfigured` already covers both a bundle-injected build and a manual + /// Keychain save — otherwise runs the same gated sync as `syncNow` when it's been >20h since the + /// last success. Call-time `repo:` parameter, same reason as every other method here (see the + /// `@StateObject` declaration's note). A plain (non-`async`) MainActor call, mirroring + /// `pullNow`/`syncNow`'s own shape: it spawns its OWN `Task` internally and returns immediately, so + /// a caller on the launch-critical path (`RootView`/`RootTabView`'s `.task`) is never blocked + /// waiting on the network. Deliberately does NOT call `syncNow(repo:)` — see `runGatedSync`'s doc + /// comment for why that would self-deadlock the gate. + func autoSyncIfDue(repo: Repository) { + guard !Self.isRunningUnderXCTest else { return } + guard CloudSyncSettings.isConfigured else { return } + let last = UserDefaults.standard.double(forKey: Self.lastAutoSyncKey) + guard Self.isAutoSyncDue(lastRun: last, now: Date().timeIntervalSince1970) else { return } + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { return } + Task { await runGatedSync(repo: repo, url: url, token: token) } + } + + #if os(iOS) + /// Staleness gate for the BACKGROUND refresh path (`backgroundSyncIfDue`, below) — SHORTER than + /// `isAutoSyncDue`'s 20h default because a background wake (BGAppRefreshTask or a silent push) is + /// opportunistic and comparatively cheap: iOS decides IF and WHEN it actually runs at all, and + /// skip-unchanged upload (see `performSync`'s doc comment) means most invocations that DO run ship + /// no bytes. There's no reason to make a user wait as long as the guaranteed on-launch catch-up does. + static let backgroundSyncIntervalS: TimeInterval = 4 * 3600 + + /// Background-refresh entry point — called from the `BGAppRefreshTask` handler + /// (`CloudSyncBackgroundRefresh`) and the silent-push handler (`CloudSyncAppDelegate`), both iOS + /// only. Unlike `autoSyncIfDue` (fire-and-forget: spawns its own `Task` and returns immediately so + /// it never blocks the launch-critical path), this AWAITS the sync to completion — both callers + /// need to know when it's done before they report back to iOS (`task.setTaskCompleted`, or the + /// push handler's `fetchCompletionHandler`). Returns `true` when a sync was actually ATTEMPTED + /// (whatever its outcome), `false` when a guard sent it home early (not configured, not due, bad + /// URL, or running under XCTest) — the push handler maps this straight to `.newData`/`.noData`. + /// + /// Reuses `runGatedSync`, the SAME `CloudSyncGate` every other entry point shares, so a background + /// wake that overlaps a foreground "Sync now" (or another background wake — a redelivered push, a + /// BGAppRefreshTask firing moments after a push already ran one) can't start a second sync; it just + /// sees the gate held and returns. Shares `Self.lastAutoSyncKey` with `autoSyncIfDue` too — a + /// successful background sync resets the SAME clock the 20h on-launch gate reads, so a launch right + /// after a successful background sync doesn't immediately re-fire (mirrors `syncNow`'s doc comment + /// on the identical interaction). + @discardableResult + func backgroundSyncIfDue(repo: Repository) async -> Bool { + guard !Self.isRunningUnderXCTest else { return false } + // Effective (Keychain OR bundle) credentials are the configuration check — `isConfigured` + // is Keychain-only and would keep a zero-touch bundle-credentialed device gated forever. + let last = UserDefaults.standard.double(forKey: Self.lastAutoSyncKey) + guard Self.isAutoSyncDue(lastRun: last, now: Date().timeIntervalSince1970, + intervalS: Self.backgroundSyncIntervalS) else { return false } + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { return false } + await runGatedSync(repo: repo, url: url, token: token) + return true + } + + /// Push-triggered sync (`request_sync` → APNs silent push → here): UNCONDITIONAL, no staleness + /// gate — the push already carries intent ("an agent wants fresh data NOW"; the server throttles + /// request_sync to one push per 120s), so routing it through `backgroundSyncIfDue`'s 4h gate + /// would silently ignore exactly the syncs the feature exists to perform. Overlap safety is + /// `runGatedSync`'s CloudSyncGate, same as every other entry point. + @discardableResult + func syncFromPush(repo: Repository) async -> Bool { + guard !Self.isRunningUnderXCTest else { return false } + guard let urlString = CloudSyncSettings.effectiveURL, let token = CloudSyncSettings.effectiveToken, + let url = URL(string: urlString) else { return false } + await runGatedSync(repo: repo, url: url, token: token) + return true + } + #endif + + #if os(iOS) && canImport(HealthKit) + /// Re-export into Apple Health when a pulled batch touched sleep/workouts/HR — shared by `pullNow` + /// and `syncNow` so both surface the same "Apple Health re-export failed" note. Only re-exports + /// what could plausibly have changed on Health's side. A fresh bridge starts at auth == .unknown + /// (it isn't the app's long-lived HealthKitBridge instance), so prime it from the existing grant — + /// `refreshAuthIfPreviouslyGranted` only READS share status, it never prompts — before attempting + /// the write-back. Returns a status-line SUFFIX: empty when nothing ran or it succeeded. + private func reExportToAppleHealthIfNeeded(repo: Repository, store: WhoopStore, + summary: CloudApplySummary, + force: Bool = false) async -> String { + guard force || summary.touchedSleep || summary.touchedWorkouts || summary.touchedHr else { return "" } + do { + let bridge = HealthKitBridge(repo: repo, appleDeviceId: "apple-health", noopDeviceId: "my-whoop") + bridge.refreshAuthIfPreviouslyGranted() + try await bridge.rerunWriteBack(whoopStore: store, days: 30) + return "" + } catch { + return " · Apple Health re-export failed: \(error.localizedDescription)" + } + } + #endif + + /// Busy also pins the screen awake on iOS, mirroring `OuraConnectModel.setBusy`: the pull + + /// Apple Health re-export is a foreground async flow, and an auto-lock mid-run suspends the app + /// and freezes it with no error. Always reset on exit. + private func setBusy(_ b: Bool) { + busy = b + #if os(iOS) + UIApplication.shared.isIdleTimerDisabled = b + #endif + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncSettings.swift b/Strand/CloudSync/CloudSyncSettings.swift new file mode 100644 index 0000000000..4eeaca163e --- /dev/null +++ b/Strand/CloudSync/CloudSyncSettings.swift @@ -0,0 +1,165 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import Security + +/// Keychain Services wrapper for the noop-cloud server URL + read-write token. One service, two +/// generic-password accounts, plain UTF-8 string values — neither ever lands in UserDefaults, a +/// plist, or on disk in the clear. Mirrors OuraTokenStore's get/set/delete shape; value encoding +/// (trim, empty-means-clear) mirrors AICoach's AIKeyStore. +enum CloudSyncSettings { + private static let service = "com.noop.cloudsync" + private static let serverURLAccount = "server-url" + private static let tokenAccount = "rw-token" + + private static func baseQuery(account: String) -> [String: Any] { + [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + } + + /// The noop-cloud server base URL (e.g. "https://vk-noop-cloud.fly.dev"), or nil if unset. + static var serverURL: String? { + get { load(account: serverURLAccount) } + set { save(newValue, account: serverURLAccount) } + } + + /// The read-write bearer token for the noop-cloud server, or nil if unset. + static var token: String? { + get { load(account: tokenAccount) } + set { save(newValue, account: tokenAccount) } + } + + /// True once both an EFFECTIVE server URL and token are available — a Keychain value, a + /// bundle-injected one, or a mix of both. Phase 3.5 (zero-touch): a bundle-only install counts as + /// configured with no Keychain write ever happening. + static var isConfigured: Bool { isConfigured(info: Bundle.main.infoDictionary ?? [:]) } + + /// Testable variant of `isConfigured`: identical Keychain-wins-over-bundle precedence, but takes + /// an explicit info dict instead of reading live `Bundle.main`. Keychain state never needed a seam + /// — a test can already set/clear it directly via `serverURL`/`token`/`clear()` — only the bundle + /// side was unswappable. Pass `info: [:]` to assert "Keychain-only" behaviour deterministically, + /// regardless of whatever CLOUDSYNC_URL/CLOUDSYNC_TOKEN this test PROCESS's real bundle happens to + /// carry: once a personal `OuraSecrets.xcconfig` carries real cloud-sync credentials, the live, + /// no-arg `isConfigured` reads true even with the Keychain cleared, which the original + /// bundle-unaware `CloudSyncSettingsTests` didn't anticipate (see that file's history). + /// + /// Also runs the URL through `isValidServerURL`, same as `effectiveURL` — this must stay in sync + /// with `effectiveURL`'s own validation (both derive the URL half from the same `effectiveValue` + /// call), or `isConfigured` could read true for a malformed bundle URL that `effectiveURL` itself + /// then reads as nil, enabling a "Sync now" button that immediately fails. + static func isConfigured(info: [String: Any]) -> Bool { + guard let url = effectiveValue(keychain: serverURL, infoKey: "CLOUDSYNC_URL", info: info), + isValidServerURL(url) else { return false } + return effectiveValue(keychain: token, infoKey: "CLOUDSYNC_TOKEN", info: info) != nil + } + + /// Remove both stored values. + static func clear() { + SecItemDelete(baseQuery(account: serverURLAccount) as CFDictionary) + SecItemDelete(baseQuery(account: tokenAccount) as CFDictionary) + } + + // MARK: - Bundle fallback (Phase 3.5: zero-touch) + // + // Mirrors Strand/Oura/OuraCredentials.swift's shape: a pure, dict-driven function that's directly + // unit-testable (`effectiveValue`), plus a thin `Bundle.main`-reading wrapper (`effectiveURL`/ + // `effectiveToken`) that can't be redirected in a test but has no logic of its own to get wrong. + + /// Read `key` from an Info-dictionary-shaped map, trimmed; blank/absent both read as nil so a + /// build without injected credentials cleanly falls through to "unconfigured" rather than saving a + /// whitespace server URL. + private static func bundleValue(_ key: String, from info: [String: Any]) -> String? { + guard let s = (info[key] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines), + !s.isEmpty else { return nil } + return s + } + + /// Keychain-wins-over-bundle precedence, as a pure function of its inputs — the seam that makes + /// the precedence rule unit-testable without a real Info.plist (see `CloudSyncSettingsTests`). + static func effectiveValue(keychain: String?, infoKey: String, info: [String: Any]) -> String? { + keychain ?? bundleValue(infoKey, from: info) + } + + /// The noop-cloud server URL a manual Save in the Data Sources card wrote to the Keychain, or the + /// build-injected `CLOUDSYNC_URL` (from the untracked `OuraSecrets.xcconfig` → project.yml → + /// Info.plist chain — see `OuraCredentials.swift`'s doc comment for the identical pattern) when no + /// Keychain override exists. nil disables the whole cloud-sync lane. Runs the merged value through + /// `isValidServerURL` before returning it — a Keychain value already passed that exact check at + /// `CloudSyncModel.saveSettings` time (so this is a no-op there), but a bundle-injected URL from + /// Info.plist is never validated anywhere else, so a malformed `CLOUDSYNC_URL` xcconfig value is + /// caught here instead of reaching `CloudSyncClient.url(path:query:)`'s force-unwrapped + /// `URLComponents`-built request URL and crashing at the first sync. + static var effectiveURL: String? { + guard let raw = effectiveValue(keychain: serverURL, infoKey: "CLOUDSYNC_URL", + info: Bundle.main.infoDictionary ?? [:]), + isValidServerURL(raw) else { return nil } + return raw + } + + /// The read-write token, same Keychain-wins-over-bundle precedence as `effectiveURL`. + static var effectiveToken: String? { + effectiveValue(keychain: token, infoKey: "CLOUDSYNC_TOKEN", info: Bundle.main.infoDictionary ?? [:]) + } + + /// True when `s` parses as an absolute URL with a scheme and host — the same shape check + /// `CloudSyncModel.saveSettings` runs before a manually-entered URL is ever written to the + /// Keychain. Internal (not `private`) so it's directly unit-testable — pure, no `Bundle`/Keychain + /// dependency at all. + static func isValidServerURL(_ s: String) -> Bool { + guard let comps = URLComponents(string: s), let scheme = comps.scheme, !scheme.isEmpty, + let host = comps.host, !host.isEmpty else { return false } + return true + } + + /// True when the lane is configured ENTIRELY from the bundle-injected build credentials, with no + /// Keychain override at all. Drives the Data Sources card: bundle-configured shows a one-line + /// "Configured from build" note instead of the URL/token fields; any Keychain value (even a + /// partial one) falls through to the manual fields so a user's own in-progress edit is never + /// silently hidden behind the collapsed summary. + static var isBundleConfigured: Bool { isBundleConfigured(info: Bundle.main.infoDictionary ?? [:]) } + + /// Testable variant of `isBundleConfigured` — same shape as `isConfigured(info:)` above and for + /// the same reason (see its doc comment): pass `info: [:]` to assert "no bundle creds" + /// deterministically regardless of this test process's real `Bundle.main`. + static func isBundleConfigured(info: [String: Any]) -> Bool { + guard serverURL == nil, token == nil else { return false } + return bundleValue("CLOUDSYNC_URL", from: info) != nil && bundleValue("CLOUDSYNC_TOKEN", from: info) != nil + } + + /// Store (or replace) `value` for `account`. Empty/whitespace input clears the item instead + /// (so an accidental blank Save can't leave a dead-but-present Keychain entry). + @discardableResult + private static func save(_ value: String?, account: String) -> Bool { + let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let query = baseQuery(account: account) + + // Delete any existing item first so we always insert a single, fresh value (or none). + SecItemDelete(query as CFDictionary) + guard !trimmed.isEmpty else { return true } + guard let data = trimmed.data(using: .utf8) else { return false } + + var attrs = query + attrs[kSecValueData as String] = data + attrs[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlock + return SecItemAdd(attrs as CFDictionary, nil) == errSecSuccess + } + + /// Load the stored value for `account`, or nil if none. + private static func load(account: String) -> String? { + var query = baseQuery(account: account) + query[kSecReturnData as String] = kCFBooleanTrue + query[kSecMatchLimit as String] = kSecMatchLimitOne + var item: CFTypeRef? + guard SecItemCopyMatching(query as CFDictionary, &item) == errSecSuccess, + let data = item as? Data, + let str = String(data: data, encoding: .utf8), + !str.isEmpty else { return nil } + return str + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/CloudSyncUploader.swift b/Strand/CloudSync/CloudSyncUploader.swift new file mode 100644 index 0000000000..bfb6ef1853 --- /dev/null +++ b/Strand/CloudSync/CloudSyncUploader.swift @@ -0,0 +1,453 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import WhoopStore + +/// Abstracts the raw-bytes ingest POST so `CloudSyncUploader` is testable without a live network — +/// mirrors `CloudSyncCoordinator`'s `CloudEditFetching` seam. `CloudSyncClient` conforms below. +protocol CloudIngesting { + func ingest(fileURL: URL) async throws -> (bytes: Int, latestDay: String?) + + /// Where a page-replication push would go, and with what credential — the same server and the + /// same token `ingest` uses. `nil` means "this ingester has no liters destination", which is + /// the correct answer for every test double and makes the liters branch inert for them without + /// each one having to opt out. + var litersDestination: (endpoint: String, token: String)? { get } +} + +extension CloudIngesting { + /// Default: no liters destination. Only `CloudSyncClient` overrides this. + var litersDestination: (endpoint: String, token: String)? { nil } +} + +extension CloudSyncClient: CloudIngesting { + var litersDestination: (endpoint: String, token: String)? { + #if LITERS + return (LitersReplicator.endpoint(base: baseURL), token) + #else + // The xcframework is not in this build, so there is nothing to push with. Reporting `nil` + // rather than an endpoint keeps the trial flag from selecting a path that cannot exist. + return nil + #endif + } +} + +/// The export half's user-facing failure. The network half of an upload throws `CloudSyncError` (same +/// typed error every other CloudSync network call throws); this covers only "never got as far as +/// having bytes to send". +enum CloudSyncUploadError: LocalizedError, Equatable { + case exportFailed(String) + /// The store has no real data yet — refused BEFORE export ever runs. Distinct from + /// `exportFailed`: nothing went wrong, there was simply nothing to upload. Guards against the + /// incident where the macOS TEST HOST (`StrandTests` running inside the full `Staging.app` via + /// `TEST_HOST`) executed the launch-time auto-sync `.task` with bundle credentials present, and + /// auto-uploaded the Mac's empty database, replacing the production mirror. This check protects + /// EVERY upload path — fresh installs, a never-paired Mac container, and any future race — not + /// just the test-host case (see `CloudSyncModel.isRunningUnderXCTest` for that separate guard). + case emptyStore + /// A liters push failed and the fallback to `/ingest` was deliberately WITHHELD to protect the + /// delta lineage. Not a dead end: the next sync retries the (small) delta push, which is the + /// whole point — an automatic `/ingest` here would reset the server-side lineage and force the + /// next push to a full ~390 MB snapshot, which cannot complete in a background window, which + /// falls back to `/ingest` again: the loop that kept 2026-08-03's syncs at snapshot size all + /// day. `/ingest` still runs after `litersRetryThreshold` consecutive failures (see + /// `ingestEscalationAllowed`), so a genuinely dead liters path degrades to the proven + /// whole-database upload instead of silent staleness. + case litersRetryPending(streak: Int, reason: String) + + var errorDescription: String? { + switch self { + case .exportFailed(let detail): + return "Couldn't prepare the backup to upload. \(detail)" + case .emptyStore: + return "Nothing to upload yet — the local database is empty." + case .litersRetryPending(let streak, let reason): + return "Delta sync didn't complete (attempt \(streak)) — will retry next sync. \(reason)" + } + } +} + +/// Produces this device's own checkpointed, integrity-verified `.noopbak` and POSTs it to the +/// noop-cloud server's `/ingest` endpoint — the "upload" half of Phase 3.5's zero-touch sync (the +/// "pull" half is `CloudSyncCoordinator`). A pure coordination step with no state of its own, like +/// `CloudSyncCoordinator`, so it's a namespace of static functions rather than an instance. +enum CloudSyncUploader { + // MARK: - Liters fallback policy (pure) + + /// What one liters push attempt amounted to, classified so `upload` can decide between the three + /// legitimate responses: report success, retry via liters next sync, or fall back to `/ingest`. + /// The classification exists because "didn't ship bytes" conflates four states with OPPOSITE + /// correct responses, and conflating them is what kept 2026-08-03's syncs at snapshot size: + /// a healthy "already in sync" was read as failure and answered with a 208 MB `/ingest`, which + /// reset the server-side lineage and forced the next push to a full snapshot. + enum LitersPushOutcome: Equatable { + /// Shipped bytes. The lineage advanced; the sync is delivered. + case pushed(bytes: Int) + /// Shipped nothing because there is nothing to ship: the lineage has content (`txid > 0`) + /// and the remote matches it exactly. Everything in the database is already on the server — + /// success, not failure. Falling through to `/ingest` from here is the single worst response + /// available: it re-uploads a couple hundred MB the server already has, then destroys the + /// lineage that made the no-op possible. + case inSync + /// Didn't ship and can't be trusted as delivered: a thrown push, a captured-but-unshipped + /// L0 (`uploaded == 0 && !synced`), or a vacuous sync on an empty lineage + /// (`txid == 0` — both sides agree on *nothing*, observed 2026-08-03 17:16 right after a + /// lineage reset). The correct response is to retry via liters next sync, NOT `/ingest`: + /// the retry is cheap (a delta, or one snapshot after a reset) and preserves the lineage, + /// while `/ingest` guarantees the next push is a full snapshot. + case retryable(String) + /// Liters cannot run by configuration: trial off, `.external` not in force, no destination, + /// or not compiled in. `/ingest` is not a fallback here — it is simply the path. + case unavailable(String) + } + + /// Pure classification of a push summary's scalar fields (kept scalar so this compiles and + /// tests without the LITERS xcframework). See `LitersPushOutcome` for what each case means. + static func classifyLitersPush(uploaded: UInt64, synced: Bool, + txid: UInt64, remoteTxid: UInt64, + bytesUploaded: UInt64) -> LitersPushOutcome { + if uploaded > 0 { return .pushed(bytes: Int(bytesUploaded)) } + if synced && txid > 0 && remoteTxid == txid { return .inSync } + if synced && txid == 0 { + return .retryable("empty lineage on both sides (txid=0) — the next push re-baselines") + } + return .retryable("captured but unshipped (uploaded=0 synced=\(synced) " + + "txid=\(txid) remoteTxid=\(remoteTxid))") + } + + /// How many consecutive `retryable` outcomes are tolerated before `/ingest` runs anyway. + /// Below the threshold, a failed delta push throws `litersRetryPending` and the sync retries + /// next cycle with the lineage intact. At or past it, data flow wins: a genuinely dead liters + /// path (sink secret lost in a redeploy, rotated token, FFI regression) degrades to the proven + /// whole-database upload rather than going silently stale. Three, not one, because the common + /// transient failures — a background window expiring mid-push, one 503 — must not cost the + /// lineage; and not ∞ because `/ingest` surviving as disaster recovery is deliberate + /// (liters has ~a week of production history; `/ingest` has months). + static let litersRetryThreshold = 3 + + /// Whether this sync should fall through to `/ingest` after a retryable liters outcome. + static func ingestEscalationAllowed(retryStreak: Int, + threshold: Int = litersRetryThreshold) -> Bool { + retryStreak >= threshold + } + + /// Consecutive retryable liters outcomes, persisted so the escalation decision survives app + /// relaunches (a background sync IS a fresh process often enough). Reset only by a liters + /// success (`pushed`/`inSync`) — deliberately NOT by a successful `/ingest`, so a persistently + /// dead liters path keeps flowing through `/ingest` without paying two throwaway syncs between + /// each delivery. + static let litersRetryStreakKey = "cloudsync.liters.retryStreak" + + static func litersRetryStreak() -> Int { + UserDefaults.standard.integer(forKey: litersRetryStreakKey) + } + + @discardableResult + static func bumpLitersRetryStreak() -> Int { + let next = litersRetryStreak() + 1 + UserDefaults.standard.set(next, forKey: litersRetryStreakKey) + return next + } + + static func resetLitersRetryStreak() { + UserDefaults.standard.removeObject(forKey: litersRetryStreakKey) + } + + /// Produces a `.noopbak` at `dest` from `store`, returning `DataBackup.BackupResult` so a real + /// export failure's message survives. Injectable so a test can supply canned bytes without + /// touching the app's real on-disk database: `WhoopStore.inMemory()` test stores have no backing + /// file at all (see `WhoopStore.inMemory()`'s doc comment — a `DatabaseQueue`, not a file-backed + /// `DatabasePool`), and the production default below is hardcoded to + /// `StorePaths.defaultDatabasePath()` regardless of which `WhoopStore` instance is passed in. That + /// fixed path is correct for production — there is only ever one real on-disk database, and + /// `FolderBackup.backupNow`/`DataBackup.runExport` resolve it the exact same way — but it makes the + /// default exporter untestable against a throwaway store, hence the seam. + typealias Exporter = (WhoopStore, URL) async -> DataBackup.BackupResult + + /// The real export: checkpoint `store`'s WAL (so the single `.sqlite` file is whole), then reuse + /// the SAME checkpointed, `PRAGMA quick_check`-verified export `BackupSync`/`FolderBackup` use — an + /// auto-uploaded snapshot is byte-identical to a manual "Export backup". + /// + /// ## …except when a page replicator owns the WAL, and that exception is the whole point + /// + /// This fallback runs on exactly the syncs where the liters push did *not*, so whatever it does + /// to the WAL happens between every pair of pushes. `checkpointWAL()` is + /// `wal_checkpoint(TRUNCATE)`, which restarts the WAL and destroys the replicator's resume + /// offset. Measured on VK's device 2026-07-28: this checkpoint at 06:03, then a push at 14:23 + /// reporting `snapshotReason: "wal truncated by another process"` and uploading 640 MB. Every + /// recorded push was a full upload — page replication reduced to a full upload with extra steps, + /// by the path it was replacing. + /// + /// A gentler checkpoint mode does **not** fix it: `FULL` leaves the `-wal` file at its full size + /// and still costs the next push a snapshot, because SQLite restarts a fully-backfilled WAL on + /// the next write. Both are measured in `LitersRoundTripTests`. + /// + /// So under `.external` the fallback takes no checkpoint at all. It stages a consistent full copy + /// through SQLite's Online Backup API — which reads *through* the WAL and never checkpoints — + /// and archives that. `/ingest` still ships a complete, fresh, `quick_check`-verified database; + /// the replicator's resume point is untouched. The cost is one full-size temp write, paid only on + /// the fallback path, and only on a device running the trial. + /// + /// If staging fails for any reason (disk, I/O), it degrades to the checkpointing path rather than + /// failing the sync: a sync that ships everything and costs one snapshot beats a sync that ships + /// nothing. + static let defaultExporter: Exporter = { store, dest in + guard case .external = store.walCheckpointing else { + return await DataBackup.writeBackup(checkpoint: { (try? await store.checkpointWAL()) != nil }, + to: dest) + } + // Alongside `dest` (Caches), which the caller already `defer`s a removal of — one directory, + // one cleanup story, and never Documents. + let staged = dest.deletingLastPathComponent() + .appendingPathComponent("cloudsync-staged-\(UUID().uuidString).sqlite") + defer { + let fm = FileManager.default + for suffix in ["", "-wal", "-shm"] { + try? fm.removeItem(atPath: staged.path + suffix) + } + } + do { + try await store.writeConsistentCopy(to: staged.path) + } catch { + NSLog("cloudsync: staging a consistent copy failed (%@) — falling back to a checkpointed " + + "export, which costs the replicator one snapshot", String(describing: error)) + return await DataBackup.writeBackup(checkpoint: { (try? await store.checkpointWAL()) != nil }, + to: dest) + } + return await DataBackup.writeBackup(stagedDatabaseAt: staged, to: dest) + } + + /// Export the live store to a disposable temp file in Caches (never Documents — nothing here is + /// meant to persist or be user-visible) and POST it to `/ingest`. The temp file is removed in + /// `defer`, whatever happens: success, an export failure, or a network failure. The DB can be + /// 100-300MB, so `CloudSyncClient.ingest` streams it from this file via + /// `URLSession.upload(for:fromFile:)` rather than loading it into memory. + /// + /// `telemetry` records one `SyncPushObservation` per successful upload. Today every one of them + /// is `snapshotted: true` with reason `full-ingest`, because `/ingest` *is* a full snapshot — + /// that is not a placeholder, it is the honest baseline the page-replication trial is measured + /// against. What the field actually buys before a replicator exists is the other half of each + /// record: the `-wal` size at push time and the interval between pushes. Those are the two + /// numbers that decide whether `WalCheckpointing.external` is safe on this device, and neither is + /// observable anywhere else. Injectable so a test writes to a temp directory rather than the + /// app's real Application Support. + static func upload(store: WhoopStore, client: any CloudIngesting, + exporter: Exporter = defaultExporter, + telemetry: SyncPushTelemetry? = SyncReplicationTrial.shared) + async throws -> (bytes: Int, latestDay: String?) { + // Refuse an empty/trivial store BEFORE touching export or the network at all — see + // `CloudSyncUploadError.emptyStore`'s doc comment for the incident this guards against. + // `dailyMetric` is written by every ingest path (BLE-derived recompute, WHOOP/Apple + // Health/Oura/Xiaomi imports), so a genuinely fresh/never-populated store has zero rows here. + guard try await store.hasAnyDailyMetrics() else { + throw CloudSyncUploadError.emptyStore + } + let cachesDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + let tempURL = cachesDir.appendingPathComponent("cloudsync-upload-\(UUID().uuidString).noopbak") + defer { try? FileManager.default.removeItem(at: tempURL) } + + // Sampled BEFORE the exporter runs, because `defaultExporter`'s first act is + // `checkpointWAL()` (a `wal_checkpoint(TRUNCATE)`), which leaves the `-wal` at ~0 bytes. + // Reading it afterwards would record that zero and measure nothing. What is wanted is how far + // the WAL had grown by the time a sync began — the quantity `WalCheckpointing.external` puts + // at risk. One `stat(2)`; nil only for an in-memory test store. + // + // Worth flagging for the replicator work that follows: that same `checkpointWAL()` is exactly + // what a page replicator must be the only caller of. Leaving it here while a replicator is + // also running would restart the WAL underneath it and force a full snapshot on every sync — + // i.e. the upload path would defeat the thing it is being replaced by. + let walBytes = store.walFileSizeBytes() ?? 0 + + // --- page replication, when the trial is on ------------------------------------------- + // + // Placed HERE, before the exporter, and that placement is the whole design. `exporter`'s + // first act is `checkpointWAL()` — a `wal_checkpoint(TRUNCATE)` — which is precisely what a + // page replicator must be the sole caller of (see the note above `walBytes`). Running the + // export and then pushing would restart the WAL underneath the writer and force + // `snapshotted: true` on every push: the old path would defeat the new one while both ran. + // + // Three conditions, all required, all cheap to check: + // * the trial flag is on (`UserDefaults`, default false — a shipped build is unaffected); + // * `.external` is actually in force, not merely requested. `isEnabled` and `isInForce` + // differ for exactly one launch after every flip, and pushing while SQLite still owns + // autocheckpoint is the configuration that snapshots on every sync; + // * the client has a liters destination (it does not when LITERS is not compiled in, and + // no test double has one). + // + // What happens when the push does NOT deliver is a POLICY, not a reflex (changed + // 2026-08-03). The old rule — any non-push falls through to `/ingest` — looked safe + // per-sync and was catastrophic as a loop: every `/ingest` resets the server-side liters + // lineage (see noop-cloud `ingest.ts` — correct and unavoidable once the mirror is + // replaced wholesale), which forces the next push to a full ~390 MB snapshot, which cannot + // complete in a background window, which fell through to `/ingest` again. Measured on VK's + // server: 27 consecutive full-size segments, and the one lineage that survived long enough + // produced deltas of 125 KB and 198 B. So: + // * a delivered push, or a verified "already in sync", returns — streak resets; + // * a retryable outcome throws `litersRetryPending` and the NEXT sync retries the cheap + // delta — the lineage is worth more than this one sync; + // * after `litersRetryThreshold` consecutive retryables, `/ingest` runs anyway — a + // genuinely dead liters path must degrade to the proven upload, not to silence; + // * `unavailable` (trial off / not compiled / no destination) goes straight to + // `/ingest`, which for those configurations is simply the path, as always. + switch await litersAttemptIfEnabled(client: client, walBytes: walBytes, + telemetry: telemetry) { + case .pushed(let bytes): + resetLitersRetryStreak() + return (bytes, nil) + case .inSync: + resetLitersRetryStreak() + return (0, nil) + case .retryable(let why): + let streak = bumpLitersRetryStreak() + guard ingestEscalationAllowed(retryStreak: streak) else { + NSLog("liters: retryable outcome (streak %d/%d) — withholding /ingest to protect " + + "the lineage; will retry the delta next sync. %@", + streak, litersRetryThreshold, why) + throw CloudSyncUploadError.litersRetryPending(streak: streak, reason: why) + } + NSLog("liters: %d consecutive retryable outcomes — escalating to /ingest " + + "(the lineage will reset and the next push re-baselines)", streak) + case .unavailable: + break + } + + switch await exporter(store, tempURL) { + case .exported: + let result = try await client.ingest(fileURL: tempURL) + // Recorded only on success, deliberately: a failed upload shipped nothing, and counting it + // would understate the byte cost per delivered sync. Never allowed to throw — `record` is + // best-effort by construction (see `SyncPushTelemetry.persist`). + if let telemetry { + telemetry.record(snapshotted: true, + snapshotReason: "full-ingest", + bytesUploaded: Int64(result.bytes), + walBytes: walBytes, + txid: 0) + // The trial has no UI. This line is how it is read — off a device console, or from the + // container's log — so it has to carry the aggregate and not just this one push. + NSLog("SyncPushTelemetry: %@ walAtPush=%lld backstopFirings=%d", + telemetry.oneLineSummary, walBytes, store.walBackstopFirings) + } + return result + case .failure(let message): + throw CloudSyncUploadError.exportFailed(message) + case .cancelled, .imported: + // The checkpointed export path (no picker, no import flow) never actually returns these — + // handled explicitly so the switch stays exhaustive without a silently-wrong `default`. + throw CloudSyncUploadError.exportFailed("The export step returned an unexpected result.") + } + } + + /// One page-replication push attempt, classified. The caller's response differs per case — + /// that difference IS the 2026-08-03 fix; see the switch in `upload` and + /// `LitersPushOutcome`'s per-case docs. The old shape (`nil` == "use /ingest" for every + /// non-push reason, failure and healthy no-op alike) is what turned one bad sync into a + /// permanent snapshot loop. + /// + /// `latestDay` is `nil` on this path by construction: `/liters` is a byte pipe into + /// `mirror.sqlite` and answers with liters' protocol, not with `/ingest`'s + /// `{ok,bytes,latestDay}`. `CloudSyncModel` uses only `bytes`. + /// `UserDefaults` key carrying the outcome of the LAST liters attempt — which branch was taken + /// and, on a failure, the error verbatim. + /// + /// Not decoration. Every non-push outcome below is otherwise reported only through `NSLog`, and + /// an `NSLog` on a phone that is not attached to Xcode goes nowhere a person can read: a trial + /// that is failing every push and a trial that was never entered produce byte-identical evidence + /// (no telemetry record, an empty bucket, and a perfectly healthy `/ingest`). That ambiguity cost + /// a full debugging session — the server said `applies: 0`, the phone said the flag was on, and + /// nothing anywhere named the actual failure. Mirrors + /// `CloudSyncAppDelegate.registrationBreadcrumbKey`, which exists for the same reason. + static let litersBreadcrumbKey = "cloudsync.liters.lastOutcome" + + /// Records one liters outcome. Best-effort and never able to fail a sync, like every other + /// breadcrumb on this lane. + private static func noteLiters(_ outcome: String) { + UserDefaults.standard.set("\(outcome) \(Date())", forKey: litersBreadcrumbKey) + } + + private static func litersAttemptIfEnabled( + client: any CloudIngesting, walBytes: Int64, telemetry: SyncPushTelemetry? + ) async -> LitersPushOutcome { + #if LITERS + guard SyncReplicationTrial.isEnabled else { return .unavailable("trial off") } + // `isEnabled` is the intent; `isInForce` is the reality. Pushing while SQLite still owns + // wal_autocheckpoint means a foreign checkpoint can restart the WAL between pushes, and + // every push then ships the whole database — measurably, not theoretically. + guard SyncReplicationTrial.isInForce else { + NSLog("liters: trial is on but WAL checkpointing is still .automatic " + + "(restart pending) — using /ingest for this sync") + noteLiters("skipped: not-in-force (restart pending)") + return .unavailable("not-in-force (restart pending)") + } + guard let dest = client.litersDestination else { + noteLiters("skipped: no liters destination") + return .unavailable("no liters destination") + } + + do { + // The one real on-disk database, resolved exactly the way `DataBackup`/`FolderBackup` + // resolve it. Throwing is folded into the same catch: a store path we cannot even name + // is a reason to use `/ingest`, not a reason to fail the sync. + let dbPath = try StorePaths.defaultDatabasePath() + // Synchronous and blocking by liters' contract; kept off the cooperative pool's + // forward progress by running it on a detached background task. + let summary = try await Task.detached(priority: .utility) { + try LitersReplicator.shared.push(databasePath: dbPath, + endpoint: dest.endpoint, + token: dest.token) + }.value + + // `uploaded == 0` is FOUR different states, not one — see `classifyLitersPush`. The + // stamp-the-token trap the old comment warned about is handled by the classifier: + // only `txid > 0 && remoteTxid == txid` counts as "in sync" (content verifiably on the + // server), so the vacuous post-reset sync (`txid == 0`, observed 2026-08-03 17:16) is + // retryable, never success — `lastUploadToken` is not stamped for it. + let outcome = classifyLitersPush(uploaded: summary.uploaded, synced: summary.synced, + txid: summary.txid, remoteTxid: summary.remoteTxid, + bytesUploaded: summary.bytesUploaded) + switch outcome { + case .pushed: + telemetry?.record(snapshotted: summary.snapshotted, + snapshotReason: summary.snapshotReason, + bytesUploaded: Int64(summary.bytesUploaded), + walBytes: walBytes, + txid: summary.txid) + NSLog("liters: pushed txid=%llu bytes=%llu snapshotted=%d reason=%@ walAtPush=%lld", + summary.txid, summary.bytesUploaded, summary.snapshotted ? 1 : 0, + summary.snapshotReason ?? "-", walBytes) + noteLiters("pushed: files=\(summary.uploaded) bytes=\(summary.bytesUploaded) " + + "txid=\(summary.txid) snapshotted=\(summary.snapshotted)") + case .inSync: + NSLog("liters: in sync at txid=%llu — nothing to ship", summary.txid) + noteLiters("in-sync: txid=\(summary.txid)") + case .retryable(let why): + NSLog("liters: push shipped no files (txid=%llu synced=%d remoteTxid=%llu) — %@", + summary.txid, summary.synced ? 1 : 0, summary.remoteTxid, why) + noteLiters("retryable: \(why)") + case .unavailable: + break // classifyLitersPush never returns this + } + return outcome + } catch { + // Classified retryable, not swallowed into an automatic `/ingest`: a thrown push — an + // unreachable sink (503 when LITERS_SINK_ENABLED is unset), a full volume (507), a + // lease conflict, a rotated token, a background window expiring mid-push — retries as + // a cheap delta next sync while the lineage survives. If it keeps failing, the retry + // streak escalates to `/ingest` (see `upload`'s switch), so a persistently dead liters + // path still degrades to the proven whole-database upload rather than silence. + NSLog("liters: push failed (%@) — will retry next sync (streak %d/%d)", + String(describing: error), litersRetryStreak() + 1, litersRetryThreshold) + // Verbatim, not a category: the whole point is that the next person to look does not + // have to guess which of liters' failure modes this was. + noteLiters("failed: \(error)") + return .retryable("push threw: \(error)") + } + #else + _ = (client, walBytes, telemetry) + return .unavailable("LITERS not compiled in") + #endif + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/DeepBufferUploadPlan.swift b/Strand/CloudSync/DeepBufferUploadPlan.swift new file mode 100644 index 0000000000..6a8f9e19c4 --- /dev/null +++ b/Strand/CloudSync/DeepBufferUploadPlan.swift @@ -0,0 +1,142 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +// +// UPSTREAM BOUNDARY: the CAPTURE half (`Strand/BLE/PuffinDeepBufferLog.swift`) is upstream-safe and +// deliberately knows nothing about this file. Everything that moves a captured byte OFF the device +// lives here, under `Strand/CloudSync/`, behind `#if CLOUD_SYNC` — the same gate as every other +// cloud file. Nothing in `Strand/BLE/` imports, calls, or references anything below. +#if CLOUD_SYNC +import Foundation + +/// Pure planning logic for `DeepBufferUploader`: which byte range of the append-only deep-buffer log +/// to ship next, and how to tell one generation of that log from the next. No filesystem, no network, +/// no `UserDefaults` — every function here is a value-in/value-out transform, so the rules that decide +/// "have we already uploaded this?" are unit-testable with no strap, no server, and no device (the +/// BLE path itself cannot be tested at all — see CLAUDE.md's "BLE behavior cannot be CI-tested"). +/// +/// ## The problem this solves +/// +/// `PuffinDeepBufferLog` writes an APPEND-ONLY JSONL file that ALSO ROTATES: at a soft cap it renames +/// `puffin-deepbuffers.jsonl` → `puffin-deepbuffers.jsonl.1` (destroying the previous `.1`) and starts +/// a fresh file. Those two facts together are what make naive "upload from offset N" wrong: +/// +/// - Append-only alone would be easy — remember the offset, ship `[offset, size)`, advance. +/// - Rotation breaks that: after a rotation the live file restarts at size 0, so a remembered offset +/// of 90 MB would either skip the whole new generation forever, or (worse, once the new file grows +/// past 90 MB) resume mid-file and ship a byte range from a DIFFERENT generation as though it were +/// a continuation — silently corrupting the archive with no error anywhere. +/// +/// So an offset is only meaningful when paired with the identity of the file it indexes into. That +/// pair — `(generation, uploadedOffset)` — is the whole design. +/// +/// ## Why the identity is (creationDate, inode) +/// +/// `rename(2)` PRESERVES the inode, so a generation keeps its identity across the `.jsonl` → `.jsonl.1` +/// rotation. That is the property that lets a drain that fell behind pick up the un-uploaded tail of a +/// rotated-away generation from `.jsonl.1` at exactly the offset it left off — no special case, no +/// separate bookkeeping, the watermark simply still matches. +/// +/// The creation date is in the id because an inode ALONE is not stable over time: rotation +/// `removeItem`s the old `.1`, freeing that inode for the filesystem to hand straight back to the next +/// freshly-created `.jsonl`. Two different generations with the same inode number would collide in the +/// watermark map, and the new generation would inherit the old one's offset — skipping real data. The +/// creation timestamp disambiguates them. `sanitizedWatermark` below is the belt-and-braces backstop +/// for any residual confusion. +/// +/// ## Why READ-ONLY, never seal-and-rotate +/// +/// The classic log-shipping design seals a segment (rename it aside) and uploads whole sealed files. +/// That is WRONG here: the BLE path holds an open `FileHandle` to the live log, and POSIX semantics +/// mean a writer keeps writing into the renamed inode — so a "sealed" segment would keep growing after +/// it was sealed, and coordinating a close/reopen would mean reaching into the offload ack path. The +/// capture path must never be blocked, slowed, or coupled to the network (see `PuffinDeepBufferLog`'s +/// doc comment on why its hex encoder is hand-rolled: it runs inside the ack path and a few hundred +/// microseconds per buffer already matters). Reading a byte range through a SEPARATE read handle is +/// invisible to the writer: the uploader never mutates the file, never holds the writer's handle, and +/// never needs the writer to know it exists. +enum DeepBufferUploadPlan { + + /// The `/OpenWhoop/` filename `PuffinDeepBufferLog` writes, and the `.1` its rotation + /// renames to. DUPLICATED here rather than read from `PuffinDeepBufferLog.logURL()` (which is + /// `private`) ON PURPOSE: making it internal would edit an upstream-safe BLE file for a + /// cloud-only reason, and that file must stay free of any reason to think about upload. The path + /// is a frozen on-disk contract — it is named in `PuffinDeepBufferLog`'s own doc comment and is + /// where the only copy of already-freed strap buffers lives — so the drift risk is low and + /// one-directional. `DeepBufferUploaderTests.testLogFilenamesMatchTheCaptureContract` pins it. + static let liveFilename = "puffin-deepbuffers.jsonl" + static let rotatedFilename = "puffin-deepbuffers.jsonl.1" + static let logDirectoryName = "OpenWhoop" + + /// Raw bytes read (and then compressed) per upload. 16 MB raw ≈ 4 MB on the wire at the ~4x these + /// hex payloads compress to, which is the real constraint: this drains from a `BGAppRefreshTask`, + /// whose budget is ~30 s of wall clock. A single 90 MB whole-file upload could never finish inside + /// that window and would be killed and restarted from zero forever; 16 MB slices each land in ~1 s + /// on LTE, and — because the watermark advances per chunk — a wake that is cut off part-way still + /// makes permanent forward progress instead of losing everything it did. Chunking is what makes + /// the background lane viable at all, not just what avoids re-uploading. + static let maxChunkBytes = 16 * 1024 * 1024 + + /// Chunks per drain, bounding one wake's work at 128 MB raw / ~32 MB sent. Enough to clear a + /// steady-state backlog (the strap banks ~40 KB/s of hex, so even a full day is a handful of runs) + /// without ever letting a single background wake run unbounded. + static let maxChunksPerRun = 8 + + /// A stable identity for one generation of the log file: `-`. + /// Millisecond precision (not seconds) so a delete/create pair inside the same second still yields + /// distinct ids if the filesystem also recycles the inode; see the type's doc comment for why + /// neither component is sufficient alone. + static func generationId(creationDate: Date?, inode: UInt64) -> String { + let ms = Int64(((creationDate ?? .distantPast).timeIntervalSince1970 * 1000).rounded()) + return "\(ms)-\(inode)" + } + + /// Clamp a remembered watermark to what the file can actually support. + /// + /// For an append-only file a watermark ABOVE the current size is impossible — the file only ever + /// grows — so seeing one means the identity is lying to us: an inode got recycled inside the same + /// millisecond, `UserDefaults` was restored onto a different device, or the file was replaced + /// out-of-band. Re-uploading a generation from 0 is cheap and idempotent (the server keys objects + /// by `(generation, byteStart)` and overwrites); resuming from a bogus offset would ship a byte + /// range that means nothing. Take the safe branch. + static func sanitizedWatermark(uploaded: Int, fileSize: Int) -> Int { + (uploaded < 0 || uploaded > fileSize) ? 0 : uploaded + } + + /// The next byte range to ship for one generation, or nil when the watermark has caught up with + /// the file. Half-open `[start, end)`, capped at `maxChunk`. + static func nextRange(uploaded: Int, fileSize: Int, maxChunk: Int = maxChunkBytes) -> Range? { + let start = sanitizedWatermark(uploaded: uploaded, fileSize: fileSize) + guard start < fileSize else { return nil } + return start.. Int { + guard let idx = data.lastIndex(of: 0x0A) else { return 0 } + return data.distance(from: data.startIndex, to: idx) + 1 + } + + /// Drop watermarks for generations that no longer exist on disk. Rotation destroys `.jsonl.1` + /// permanently, so its watermark is dead weight the moment the file is gone; without this the map + /// grows without bound for the life of the install (one entry per rotation, forever) inside a + /// `UserDefaults` value that is read and rewritten on every drain. + static func prunedWatermarks(_ marks: [String: Int], live: Set) -> [String: Int] { + marks.filter { live.contains($0.key) } + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/DeepBufferUploader.swift b/Strand/CloudSync/DeepBufferUploader.swift new file mode 100644 index 0000000000..9f8f122e68 --- /dev/null +++ b/Strand/CloudSync/DeepBufferUploader.swift @@ -0,0 +1,259 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +// +// UPSTREAM BOUNDARY: see DeepBufferUploadPlan.swift's header. The capture half never references this. +#if CLOUD_SYNC +import Foundation +import Compression + +/// Abstracts the deep-buffer chunk POST so `DeepBufferUploader` is testable with no network — the same +/// seam shape as `CloudIngesting` (`CloudSyncUploader`) and `CloudEditFetching` (`CloudSyncCoordinator`). +/// `CloudSyncClient` conforms in `CloudSyncClient+DeepBuffer.swift`. +protocol DeepBufferIngesting { + func uploadDeepBuffer(generation: String, byteStart: Int, byteEnd: Int, + compressed: Data) async throws -> DeepBufferUploadReceipt +} +extension CloudSyncClient: DeepBufferIngesting {} + +/// What the server says it did with a chunk. `duplicate` is not a failure: the drain is at-least-once +/// (the watermark only advances on a confirmed 2xx, so a response lost in flight re-sends the same +/// range), and the server keys objects by `(generation, byteStart)` — so a re-send is an idempotent +/// overwrite, and saying so lets the drain log the difference instead of double-counting bytes. +struct DeepBufferUploadReceipt: Equatable { + let storedBytes: Int + let lines: Int + let duplicate: Bool +} + +/// One generation of the on-disk log, as `scanGenerations` found it. +struct DeepBufferGeneration: Equatable { + let id: String + let url: URL + let size: Int +} + +/// The outcome of one drain, for the status line and the tests. +struct DeepBufferDrainSummary: Equatable { + var chunks = 0 + var rawBytes = 0 + var sentBytes = 0 + var lines = 0 + var duplicates = 0 + /// True when the drain stopped because it hit `maxChunksPerRun`, not because it ran out of data — + /// i.e. there is more backlog and the next wake should run again immediately rather than wait. + var moreRemaining = false + var error: String? + + var isEmpty: Bool { chunks == 0 && error == nil } +} + +/// Where the per-generation byte watermarks live. A protocol so tests get a deterministic in-memory +/// map instead of racing the test host's real `UserDefaults`. +protocol DeepBufferWatermarkStoring: AnyObject { + func watermarks() -> [String: Int] + func setWatermarks(_ marks: [String: Int]) +} + +/// Production watermark storage. `UserDefaults`, not a `WhoopStore` cursor — the same call as +/// `CloudSyncModel.lastUploadTokenKey` (see its doc comment): this is per-device bookkeeping about an +/// upload THIS device performed, not data the store owns, and `cursors` only holds `Int` values keyed +/// by a single name anyway, where this needs a whole generation→offset map. +final class DeepBufferUserDefaultsWatermarks: DeepBufferWatermarkStoring { + static let key = "cloudsync.deepbuf.watermarks" + private let defaults: UserDefaults + init(defaults: UserDefaults = .standard) { self.defaults = defaults } + + func watermarks() -> [String: Int] { + (defaults.dictionary(forKey: Self.key) as? [String: Int]) ?? [:] + } + func setWatermarks(_ marks: [String: Int]) { + defaults.set(marks, forKey: Self.key) + } +} + +/// Ships `PuffinDeepBufferLog`'s append-only JSONL to the noop-cloud server in compressed, +/// line-aligned byte-range chunks, resuming from a per-generation watermark so nothing is ever sent +/// twice and nothing is skipped. The correctness argument for the `(generation, offset)` pair, and for +/// reading the live file rather than sealing segments, is in `DeepBufferUploadPlan`'s doc comment. +/// +/// A namespace of static functions with no state of its own, like `CloudSyncUploader` — the only +/// durable state is the watermark map, which is injected. +/// +/// NEVER touches the BLE path: it opens its own read-only `FileHandle`, reads a bounded byte range, +/// and closes it. `PuffinDeepBufferLog` (which is `@MainActor`, and whose `appendIfDeepBuffer` runs +/// inside the offload ack path) is not referenced, imported, or awaited from anywhere in here. This +/// type is deliberately NOT `@MainActor`: the read + deflate of a 16 MB slice is real CPU work and +/// belongs on a cooperative thread, not on the actor the capture path is trying to use. +enum DeepBufferUploader { + + /// `/OpenWhoop/` — the directory `PuffinDeepBufferLog` writes into. Resolved here + /// rather than borrowed from that (upstream-safe, `private`) type; see + /// `DeepBufferUploadPlan.liveFilename`'s doc comment for why the duplication is deliberate. + static func defaultLogDirectory() throws -> URL { + try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, + appropriateFor: nil, create: true) + .appendingPathComponent(DeepBufferUploadPlan.logDirectoryName, isDirectory: true) + } + + /// The generations present in `dir`, ROTATED FIRST then live. + /// + /// Rotated-first is the load-bearing order: `.jsonl.1` is the generation the NEXT rotation deletes + /// forever, so it is the data most at risk and must drain first. It is also the chronologically + /// older one, so this keeps the uploaded archive in wall-clock order for free. + /// + /// Deduped by generation id: rotation runs on the MainActor while this scan runs off it, so a scan + /// that interleaves with a rename can in principle observe one inode under both filenames. Keeping + /// the first occurrence (the rotated path) rather than uploading the same generation twice under + /// two names costs nothing and removes the race from every caller's reasoning. + static func scanGenerations(in dir: URL) -> [DeepBufferGeneration] { + var seen = Set() + var out: [DeepBufferGeneration] = [] + for name in [DeepBufferUploadPlan.rotatedFilename, DeepBufferUploadPlan.liveFilename] { + let url = dir.appendingPathComponent(name) + guard let attrs = try? FileManager.default.attributesOfItem(atPath: url.path), + let size = (attrs[.size] as? NSNumber)?.intValue, + let inode = (attrs[.systemFileNumber] as? NSNumber)?.uint64Value else { continue } + let id = DeepBufferUploadPlan.generationId(creationDate: attrs[.creationDate] as? Date, + inode: inode) + guard !seen.contains(id) else { continue } + seen.insert(id) + out.append(DeepBufferGeneration(id: id, url: url, size: size)) + } + return out + } + + /// Read `range` from `url` through a private read handle, then cut the result back to its last + /// complete line (see `DeepBufferUploadPlan.lastCompleteLineEnd`). Returns nil when the range + /// yields no complete line — e.g. a single in-progress line longer than the read, or a file that + /// vanished under a rotation between the scan and here. + /// + /// `expecting` is the generation id the caller scanned, and this re-checks it THROUGH THE OPEN + /// DESCRIPTOR. That is the whole point: the scan binds a generation's identity to (creationDate, + /// inode), but this opens by PATH, and the gap between the two spans a network upload — seconds, + /// not microseconds. `PuffinDeepBufferLog` rotates by renaming `.jsonl` -> `.jsonl.1`, so a + /// rotation landing in that gap leaves the path perfectly valid and pointing at a DIFFERENT file. + /// The read then succeeds and returns the wrong generation's bytes, which the caller staples into + /// the previous generation's archive under its id AND acks — advancing that generation's watermark + /// past a tail it never uploaded. Two silent corruptions, no error anywhere, and the archive is the + /// only copy of buffers the strap's ack-trim has already freed. + /// + /// A rename cannot change what an open fd refers to, so fstat-ing the descriptor after opening is + /// authoritative where a path re-check would still race. Mismatch => bail; the next drain rescans + /// and picks the generation up at its true watermark. + static func readLineAlignedChunk(url: URL, range: Range, expecting generation: String?) -> Data? { + guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } + defer { try? handle.close() } + if let expected = generation { + var st = stat() + guard fstat(handle.fileDescriptor, &st) == 0 else { return nil } + let birth = Date(timeIntervalSince1970: TimeInterval(st.st_birthtimespec.tv_sec) + + TimeInterval(st.st_birthtimespec.tv_nsec) / 1_000_000_000) + guard DeepBufferUploadPlan.generationId(creationDate: birth, inode: UInt64(st.st_ino)) == expected, + range.upperBound <= Int(st.st_size) else { return nil } + } + guard (try? handle.seek(toOffset: UInt64(range.lowerBound))) != nil, + let raw = try? handle.read(upToCount: range.count), !raw.isEmpty else { return nil } + let end = DeepBufferUploadPlan.lastCompleteLineEnd(raw) + guard end > 0 else { return nil } + return raw.prefix(end) + } + + /// Raw DEFLATE, via Apple's Compression framework. + /// + /// `.zlib` here is a MISNOMER in Apple's own API: `COMPRESSION_ZLIB` emits a RAW DEFLATE stream + /// with NO zlib header or Adler-32 trailer, which is why the server inflates it with + /// `zlib.inflateRawSync` and not `inflateSync`. Getting this pair wrong fails at runtime with an + /// "incorrect header check", never at compile time, on both sides — it is pinned end-to-end by + /// `DeepBufferCompressionTests` (Swift side) and `test/deepbuf.test.ts` (the Node inflate of real + /// Swift-produced bytes). + /// + /// The phone has CPU to spare for this and it is nowhere near the BLE critical path (see the type's + /// doc comment) — the whole point of paying it here is that hex JSONL is the most compressible + /// thing this app produces: every payload byte is two ASCII nibbles, so half the entropy is + /// structural before the IMU's own sample-to-sample correlation is considered. + static func deflate(_ data: Data) -> Data? { + try? (data as NSData).compressed(using: .zlib) as Data + } + + /// Drain every generation from its watermark up to the file's current end, oldest first, stopping + /// at `maxChunks`. + /// + /// The watermark advances ONLY after a confirmed 2xx, and only for the range that call actually + /// acknowledged. That single rule is what makes the whole lane safe under failure: a dropped + /// connection, a killed background task, a 500, or a phone that runs out of budget mid-drain all + /// leave the watermark exactly where the last CONFIRMED byte ended, so the next run re-sends that + /// range and nothing else. Combined with the server's `(generation, byteStart)` idempotency this is + /// at-least-once delivery onto an idempotent sink — i.e. effectively exactly-once — without the + /// phone having to track anything but one integer per generation. + /// + /// A per-generation failure STOPS that generation (its later bytes cannot be shipped before its + /// earlier ones without leaving a hole the watermark can't express) but the summary still reports + /// whatever landed before it, so partial progress is never rolled back. + @discardableResult + static func drain(client: any DeepBufferIngesting, + watermarks: any DeepBufferWatermarkStoring, + directory: URL? = nil, + maxChunks: Int = DeepBufferUploadPlan.maxChunksPerRun, + maxChunkBytes: Int = DeepBufferUploadPlan.maxChunkBytes) async -> DeepBufferDrainSummary { + var summary = DeepBufferDrainSummary() + let dir: URL + do { dir = try directory ?? defaultLogDirectory() } catch { + summary.error = "Couldn't locate the deep-buffer log directory. \(error.localizedDescription)" + return summary + } + let generations = scanGenerations(in: dir) + guard !generations.isEmpty else { return summary } + + // Prune BEFORE draining, against the generations actually on disk right now — a rotation that + // destroyed `.jsonl.1` also destroyed any reason to remember its offset. + var marks = DeepBufferUploadPlan.prunedWatermarks(watermarks.watermarks(), + live: Set(generations.map(\.id))) + + for generation in generations { + while summary.chunks < maxChunks { + let uploaded = marks[generation.id] ?? 0 + guard let range = DeepBufferUploadPlan.nextRange(uploaded: uploaded, + fileSize: generation.size, + maxChunk: maxChunkBytes) else { break } + guard let chunk = readLineAlignedChunk(url: generation.url, range: range, + expecting: generation.id) else { + // No complete line in this range: either the writer is mid-line at the very tail + // (normal — the next drain gets it once the line is finished) or the file rotated + // away underneath us. Neither is an error; move to the next generation. + break + } + guard let body = deflate(chunk) else { + summary.error = "Couldn't compress the deep-buffer chunk." + watermarks.setWatermarks(marks) + return summary + } + let byteStart = range.lowerBound + let byteEnd = byteStart + chunk.count // the LINE-ALIGNED end, not range.upperBound + do { + let receipt = try await client.uploadDeepBuffer(generation: generation.id, + byteStart: byteStart, + byteEnd: byteEnd, + compressed: body) + marks[generation.id] = byteEnd + watermarks.setWatermarks(marks) // persist per chunk: a kill mid-drain keeps progress + summary.chunks += 1 + summary.rawBytes += chunk.count + summary.sentBytes += body.count + summary.lines += receipt.lines + if receipt.duplicate { summary.duplicates += 1 } + } catch { + summary.error = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + watermarks.setWatermarks(marks) + return summary + } + } + if summary.chunks >= maxChunks { break } + } + watermarks.setWatermarks(marks) + // Anything still behind its file's end means the cap cut us short, not that we finished. + summary.moreRemaining = generations.contains { (marks[$0.id] ?? 0) < $0.size } + return summary + } +} +#endif // CLOUD_SYNC diff --git a/Strand/CloudSync/Generated/liters_ffi.swift b/Strand/CloudSync/Generated/liters_ffi.swift new file mode 100644 index 0000000000..5fe6329841 --- /dev/null +++ b/Strand/CloudSync/Generated/liters_ffi.swift @@ -0,0 +1,4460 @@ +// GENERATED by Rust/build-ios.sh from the pinned liters-ffi. Do not edit by hand. +// The #if is added by that script, not by uniffi: see the script for why it is here. +#if LITERS +// This file was autogenerated by some hot garbage in the `uniffi` crate. +// Trust me, you don't want to mess with it! + +// swiftlint:disable all +import Foundation + +// Depending on the consumer's build setup, the low-level FFI code +// might be in a separate module, or it might be compiled inline into +// this module. This is a bit of light hackery to work with both. +#if canImport(liters_ffiFFI) +import liters_ffiFFI +#endif + +fileprivate extension RustBuffer { + // Allocate a new buffer, copying the contents of a `UInt8` array. + init(bytes: [UInt8]) { + let rbuf = bytes.withUnsafeBufferPointer { ptr in + RustBuffer.from(ptr) + } + self.init(capacity: rbuf.capacity, len: rbuf.len, data: rbuf.data) + } + + static func empty() -> RustBuffer { + RustBuffer(capacity: 0, len:0, data: nil) + } + + static func from(_ ptr: UnsafeBufferPointer) -> RustBuffer { + try! rustCall { ffi_liters_ffi_rustbuffer_from_bytes(ForeignBytes(bufferPointer: ptr), $0) } + } + + // Frees the buffer in place. + // The buffer must not be used after this is called. + func deallocate() { + try! rustCall { ffi_liters_ffi_rustbuffer_free(self, $0) } + } +} + +fileprivate extension ForeignBytes { + init(bufferPointer: UnsafeBufferPointer) { + self.init(len: Int32(bufferPointer.count), data: bufferPointer.baseAddress) + } + + init(rawBufferPointer: UnsafeRawBufferPointer) { + self.init( + len: Int32(rawBufferPointer.count), + data: rawBufferPointer.baseAddress?.assumingMemoryBound(to: UInt8.self) + ) + } +} + +// Converter for `&[u8]` / `[ByRef] bytes` arguments. +// +// Conforms to `FfiConverter` so the compiler enforces the full converter +// method set. Only the scope-bound `lower(_:_body:)` overload is sound — +// zero-copy byte buffers only flow foreign -> Rust, and only in argument +// position. The four protocol-witness methods (`lift`, `lower`, `read`, +// `write`) `fatalError` at runtime if anyone reaches them. +// +// The scope-bound `lower` takes a closure because the `ForeignBytes` +// pointer is only guaranteed valid for the duration of +// `Data.withUnsafeBytes`. Callers must run the full FFI call inside +// the closure body. +fileprivate enum FfiConverterByRefBytes: FfiConverter { + typealias SwiftType = Data + typealias FfiType = ForeignBytes + + static func lower(_ value: Data, _ body: (ForeignBytes) throws -> R) rethrows -> R { + return try value.withUnsafeBytes { rawBuf in + try body(ForeignBytes(rawBufferPointer: rawBuf)) + } + } + + static func lower(_ value: Data) -> ForeignBytes { + fatalError("ByRef bytes cannot use the plain lower: returning ForeignBytes escapes the Data.withUnsafeBytes scope. Use the scope-bound lower(_:_body:) overload instead.") + } + + static func lift(_ value: ForeignBytes) throws -> Data { + fatalError("ByRef bytes cannot be lifted: zero-copy &[u8] only flows foreign->Rust") + } + + static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Data { + fatalError("ByRef bytes cannot be read from a buffer: zero-copy &[u8] is only supported in argument position, not nested in records/options/etc.") + } + + static func write(_ value: Data, into buf: inout [UInt8]) { + fatalError("ByRef bytes cannot be written to a buffer: zero-copy &[u8] is only supported in argument position, not nested in records/options/etc.") + } +} + +// For every type used in the interface, we provide helper methods for conveniently +// lifting and lowering that type from C-compatible data, and for reading and writing +// values of that type in a buffer. + +// Helper classes/extensions that don't change. +// Someday, this will be in a library of its own. + +fileprivate extension Data { + init(rustBuffer: RustBuffer) { + self.init( + bytesNoCopy: rustBuffer.data!, + count: Int(rustBuffer.len), + deallocator: .none + ) + } +} + +// Define reader functionality. Normally this would be defined in a class or +// struct, but we use standalone functions instead in order to make external +// types work. +// +// With external types, one swift source file needs to be able to call the read +// method on another source file's FfiConverter, but then what visibility +// should Reader have? +// - If Reader is fileprivate, then this means the read() must also +// be fileprivate, which doesn't work with external types. +// - If Reader is internal/public, we'll get compile errors since both source +// files will try define the same type. +// +// Instead, the read() method and these helper functions input a tuple of data + +fileprivate func createReader(data: Data) -> (data: Data, offset: Data.Index) { + (data: data, offset: 0) +} + +// Reads an integer at the current offset, in big-endian order, and advances +// the offset on success. Throws if reading the integer would move the +// offset past the end of the buffer. +fileprivate func readInt(_ reader: inout (data: Data, offset: Data.Index)) throws -> T { + let range = reader.offset...size + guard reader.data.count >= range.upperBound else { + throw UniffiInternalError.bufferOverflow + } + if T.self == UInt8.self { + let value = reader.data[reader.offset] + reader.offset += 1 + return value as! T + } + var value: T = 0 + let _ = withUnsafeMutableBytes(of: &value, { reader.data.copyBytes(to: $0, from: range)}) + reader.offset = range.upperBound + return value.bigEndian +} + +// Reads an arbitrary number of bytes, to be used to read +// raw bytes, this is useful when lifting strings +fileprivate func readBytes(_ reader: inout (data: Data, offset: Data.Index), count: Int) throws -> Array { + let range = reader.offset..<(reader.offset+count) + guard reader.data.count >= range.upperBound else { + throw UniffiInternalError.bufferOverflow + } + var value = [UInt8](repeating: 0, count: count) + value.withUnsafeMutableBufferPointer({ buffer in + reader.data.copyBytes(to: buffer, from: range) + }) + reader.offset = range.upperBound + return value +} + +// Reads a float at the current offset. +fileprivate func readFloat(_ reader: inout (data: Data, offset: Data.Index)) throws -> Float { + return Float(bitPattern: try readInt(&reader)) +} + +// Reads a float at the current offset. +fileprivate func readDouble(_ reader: inout (data: Data, offset: Data.Index)) throws -> Double { + return Double(bitPattern: try readInt(&reader)) +} + +// Indicates if the offset has reached the end of the buffer. +fileprivate func hasRemaining(_ reader: (data: Data, offset: Data.Index)) -> Bool { + return reader.offset < reader.data.count +} + +// Define writer functionality. Normally this would be defined in a class or +// struct, but we use standalone functions instead in order to make external +// types work. See the above discussion on Readers for details. + +fileprivate func createWriter() -> [UInt8] { + return [] +} + +fileprivate func writeBytes(_ writer: inout [UInt8], _ byteArr: S) where S: Sequence, S.Element == UInt8 { + writer.append(contentsOf: byteArr) +} + +// Writes an integer in big-endian order. +// +// Warning: make sure what you are trying to write +// is in the correct type! +fileprivate func writeInt(_ writer: inout [UInt8], _ value: T) { + var value = value.bigEndian + withUnsafeBytes(of: &value) { writer.append(contentsOf: $0) } +} + +fileprivate func writeFloat(_ writer: inout [UInt8], _ value: Float) { + writeInt(&writer, value.bitPattern) +} + +fileprivate func writeDouble(_ writer: inout [UInt8], _ value: Double) { + writeInt(&writer, value.bitPattern) +} + +// Protocol for types that transfer other types across the FFI. This is +// analogous to the Rust trait of the same name. +fileprivate protocol FfiConverter { + associatedtype FfiType + associatedtype SwiftType + + static func lift(_ value: FfiType) throws -> SwiftType + static func lower(_ value: SwiftType) -> FfiType + static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType + static func write(_ value: SwiftType, into buf: inout [UInt8]) +} + +// Types conforming to `Primitive` pass themselves directly over the FFI. +fileprivate protocol FfiConverterPrimitive: FfiConverter where FfiType == SwiftType { } + +extension FfiConverterPrimitive { +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public static func lift(_ value: FfiType) throws -> SwiftType { + return value + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public static func lower(_ value: SwiftType) -> FfiType { + return value + } +} + +// Types conforming to `FfiConverterRustBuffer` lift and lower into a `RustBuffer`. +// Used for complex types where it's hard to write a custom lift/lower. +fileprivate protocol FfiConverterRustBuffer: FfiConverter where FfiType == RustBuffer {} + +extension FfiConverterRustBuffer { +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public static func lift(_ buf: RustBuffer) throws -> SwiftType { + var reader = createReader(data: Data(rustBuffer: buf)) + let value = try read(from: &reader) + if hasRemaining(reader) { + throw UniffiInternalError.incompleteData + } + buf.deallocate() + return value + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public static func lower(_ value: SwiftType) -> RustBuffer { + var writer = createWriter() + write(value, into: &writer) + return RustBuffer(bytes: writer) + } +} +// An error type for FFI errors. These errors occur at the UniFFI level, not +// the library level. +fileprivate enum UniffiInternalError: LocalizedError { + case bufferOverflow + case incompleteData + case unexpectedOptionalTag + case unexpectedEnumCase + case unexpectedNullPointer + case unexpectedRustCallStatusCode + case unexpectedRustCallError + case unexpectedStaleHandle + case rustPanic(_ message: String) + + public var errorDescription: String? { + switch self { + case .bufferOverflow: return "Reading the requested value would read past the end of the buffer" + case .incompleteData: return "The buffer still has data after lifting its containing value" + case .unexpectedOptionalTag: return "Unexpected optional tag; should be 0 or 1" + case .unexpectedEnumCase: return "Raw enum value doesn't match any cases" + case .unexpectedNullPointer: return "Raw pointer value was null" + case .unexpectedRustCallStatusCode: return "Unexpected RustCallStatus code" + case .unexpectedRustCallError: return "CALL_ERROR but no errorClass specified" + case .unexpectedStaleHandle: return "The object in the handle map has been dropped already" + case let .rustPanic(message): return message + } + } +} + +fileprivate extension NSLock { + func withLock(f: () throws -> T) rethrows -> T { + self.lock() + defer { self.unlock() } + return try f() + } +} + +fileprivate let CALL_SUCCESS: Int8 = 0 +fileprivate let CALL_ERROR: Int8 = 1 +fileprivate let CALL_UNEXPECTED_ERROR: Int8 = 2 +fileprivate let CALL_CANCELLED: Int8 = 3 + +fileprivate extension RustCallStatus { + init() { + self.init( + code: CALL_SUCCESS, + errorBuf: RustBuffer.init( + capacity: 0, + len: 0, + data: nil + ) + ) + } +} + +private func rustCall(_ callback: (UnsafeMutablePointer) -> T) throws -> T { + let neverThrow: ((RustBuffer) throws -> Never)? = nil + return try makeRustCall(callback, errorHandler: neverThrow) +} + +private func rustCallWithError( + _ errorHandler: @escaping (RustBuffer) throws -> E, + _ callback: (UnsafeMutablePointer) -> T) throws -> T { + try makeRustCall(callback, errorHandler: errorHandler) +} + +private func makeRustCall( + _ callback: (UnsafeMutablePointer) -> T, + errorHandler: ((RustBuffer) throws -> E)? +) throws -> T { + uniffiEnsureLitersFfiInitialized() + var callStatus = RustCallStatus.init() + let returnedVal = callback(&callStatus) + try uniffiCheckCallStatus(callStatus: callStatus, errorHandler: errorHandler) + return returnedVal +} + +private func uniffiCheckCallStatus( + callStatus: RustCallStatus, + errorHandler: ((RustBuffer) throws -> E)? +) throws { + switch callStatus.code { + case CALL_SUCCESS: + return + + case CALL_ERROR: + if let errorHandler = errorHandler { + throw try errorHandler(callStatus.errorBuf) + } else { + callStatus.errorBuf.deallocate() + throw UniffiInternalError.unexpectedRustCallError + } + + case CALL_UNEXPECTED_ERROR: + // When the rust code sees a panic, it tries to construct a RustBuffer + // with the message. But if that code panics, then it just sends back + // an empty buffer. + if callStatus.errorBuf.len > 0 { + throw UniffiInternalError.rustPanic(try FfiConverterString.lift(callStatus.errorBuf)) + } else { + callStatus.errorBuf.deallocate() + throw UniffiInternalError.rustPanic("Rust panic") + } + + case CALL_CANCELLED: + fatalError("Cancellation not supported yet") + + default: + throw UniffiInternalError.unexpectedRustCallStatusCode + } +} + +private func uniffiTraitInterfaceCall( + callStatus: UnsafeMutablePointer, + makeCall: () throws -> T, + writeReturn: (T) -> () +) { + do { + try writeReturn(makeCall()) + } catch let error { + callStatus.pointee.code = CALL_UNEXPECTED_ERROR + callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) + } +} + +private func uniffiTraitInterfaceCallWithError( + callStatus: UnsafeMutablePointer, + makeCall: () throws -> T, + writeReturn: (T) -> (), + lowerError: (E) -> RustBuffer +) { + do { + try writeReturn(makeCall()) + } catch let error as E { + callStatus.pointee.code = CALL_ERROR + callStatus.pointee.errorBuf = lowerError(error) + } catch { + callStatus.pointee.code = CALL_UNEXPECTED_ERROR + callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) + } +} +// Initial value and increment amount for handles. +// These ensure that SWIFT handles always have the lowest bit set +fileprivate let UNIFFI_HANDLEMAP_INITIAL: UInt64 = 1 +fileprivate let UNIFFI_HANDLEMAP_DELTA: UInt64 = 2 + +fileprivate final class UniffiHandleMap: @unchecked Sendable { + // All mutation happens with this lock held, which is why we implement @unchecked Sendable. + private let lock = NSLock() + private var map: [UInt64: T] = [:] + private var currentHandle: UInt64 = UNIFFI_HANDLEMAP_INITIAL + + func insert(obj: T) -> UInt64 { + lock.withLock { + return doInsert(obj) + } + } + + // Low-level insert function, this assumes `lock` is held. + private func doInsert(_ obj: T) -> UInt64 { + let handle = currentHandle + currentHandle += UNIFFI_HANDLEMAP_DELTA + map[handle] = obj + return handle + } + + func get(handle: UInt64) throws -> T { + try lock.withLock { + guard let obj = map[handle] else { + throw UniffiInternalError.unexpectedStaleHandle + } + return obj + } + } + + func clone(handle: UInt64) throws -> UInt64 { + try lock.withLock { + guard let obj = map[handle] else { + throw UniffiInternalError.unexpectedStaleHandle + } + return doInsert(obj) + } + } + + @discardableResult + func remove(handle: UInt64) throws -> T { + try lock.withLock { + guard let obj = map.removeValue(forKey: handle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return obj + } + } + + var count: Int { + get { + map.count + } + } +} + + +// Public interface members begin here. +// Magic number for the Rust proxy to call using the same mechanism as every other method, +// to free the callback once it's dropped by Rust. +private let IDX_CALLBACK_FREE: Int32 = 0 +// Callback return codes +private let UNIFFI_CALLBACK_SUCCESS: Int32 = 0 +private let UNIFFI_CALLBACK_ERROR: Int32 = 1 +private let UNIFFI_CALLBACK_UNEXPECTED_ERROR: Int32 = 2 + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterUInt16: FfiConverterPrimitive { + typealias FfiType = UInt16 + typealias SwiftType = UInt16 + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt16 { + return try lift(readInt(&buf)) + } + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterUInt32: FfiConverterPrimitive { + typealias FfiType = UInt32 + typealias SwiftType = UInt32 + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt32 { + return try lift(readInt(&buf)) + } + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterUInt64: FfiConverterPrimitive { + typealias FfiType = UInt64 + typealias SwiftType = UInt64 + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt64 { + return try lift(readInt(&buf)) + } + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterDouble: FfiConverterPrimitive { + typealias FfiType = Double + typealias SwiftType = Double + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Double { + return try lift(readDouble(&buf)) + } + + public static func write(_ value: Double, into buf: inout [UInt8]) { + writeDouble(&buf, lower(value)) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterBool : FfiConverter { + typealias FfiType = Int8 + typealias SwiftType = Bool + + public static func lift(_ value: Int8) throws -> Bool { + return value != 0 + } + + public static func lower(_ value: Bool) -> Int8 { + return value ? 1 : 0 + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Bool { + return try lift(readInt(&buf)) + } + + public static func write(_ value: Bool, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterString: FfiConverter { + typealias SwiftType = String + typealias FfiType = RustBuffer + + public static func lift(_ value: RustBuffer) throws -> String { + defer { + value.deallocate() + } + if value.data == nil { + return String() + } + let bytes = UnsafeBufferPointer(start: value.data!, count: Int(value.len)) + // Use Swift's native UTF-8 decoder; `String(bytes:encoding:.utf8)` goes + // through Foundation's NSString and silently strips a leading U+FEFF BOM. + // Invalid UTF-8 substitutes U+FFFD instead of trapping (unreachable + // given Rust's `String` invariant). + return String(decoding: bytes, as: UTF8.self) + } + + public static func lower(_ value: String) -> RustBuffer { + return value.utf8CString.withUnsafeBufferPointer { ptr in + // The swift string gives us int8_t, we want uint8_t. + ptr.withMemoryRebound(to: UInt8.self) { ptr in + // The swift string gives us a trailing null byte, we don't want it. + let buf = UnsafeBufferPointer(rebasing: ptr.prefix(upTo: ptr.count - 1)) + return RustBuffer.from(buf) + } + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> String { + let len: Int32 = try readInt(&buf) + // See `lift` above for why we avoid Foundation's NSString-backed decoder here. + return String(decoding: try readBytes(&buf, count: Int(len)), as: UTF8.self) + } + + public static func write(_ value: String, into buf: inout [UInt8]) { + let len = Int32(value.utf8.count) + writeInt(&buf, len) + writeBytes(&buf, value.utf8) + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterData: FfiConverterRustBuffer { + typealias SwiftType = Data + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Data { + let len: Int32 = try readInt(&buf) + return Data(try readBytes(&buf, count: Int(len))) + } + + public static func write(_ value: Data, into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + writeBytes(&buf, value) + } +} + + + + +/** + * The host HTTP client: ONE shared instance for the whole process. Backed by + * one platform client (e.g. a single `OkHttpClient`) so requests to one + * authority coalesce onto a single connection. `body` is present only for + * `PUT` pushes. + */ +public protocol HttpClient: AnyObject, Sendable { + + func execute(request: HttpRequest, body: HttpRequestBody?) throws -> HttpResponse + +} +/** + * The host HTTP client: ONE shared instance for the whole process. Backed by + * one platform client (e.g. a single `OkHttpClient`) so requests to one + * authority coalesce onto a single connection. `body` is present only for + * `PUT` pushes. + */ +open class HttpClientImpl: HttpClient, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_httpclient(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_httpclient(handle, $0) } + } + + + + +open func execute(request: HttpRequest, body: HttpRequestBody?)throws -> HttpResponse { + return try FfiConverterTypeHttpResponse_lift(try rustCallWithError(FfiConverterTypeHttpError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httpclient_execute( + self.uniffiCloneHandle(), + FfiConverterTypeHttpRequest_lower(request), + FfiConverterOptionTypeHttpRequestBody.lower(body),uniffiCallStatus + ) +}) +} + + + +} + + + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct UniffiCallbackInterfaceHttpClient { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + // + // Store the vtable directly. + static let vtable: UniffiVTableCallbackInterfaceHttpClient = UniffiVTableCallbackInterfaceHttpClient( + uniffiFree: { (uniffiHandle: UInt64) -> () in + do { + try FfiConverterTypeHttpClient.handleMap.remove(handle: uniffiHandle) + } catch { + print("Uniffi callback interface HttpClient: handle missing in uniffiFree") + } + }, + uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in + do { + return try FfiConverterTypeHttpClient.handleMap.clone(handle: uniffiHandle) + } catch { + fatalError("Uniffi callback interface HttpClient: handle missing in uniffiClone") + } + }, + execute: { ( + uniffiHandle: UInt64, + request: RustBuffer, + body: RustBuffer, + uniffiOutReturn: UnsafeMutablePointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> HttpResponse in + guard let uniffiObj = try? FfiConverterTypeHttpClient.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return try uniffiObj.execute( + request: try FfiConverterTypeHttpRequest_lift(request), + body: try FfiConverterOptionTypeHttpRequestBody.lift(body) + ) + } + + + let writeReturn = { uniffiOutReturn.pointee = FfiConverterTypeHttpResponse_lower($0) } + uniffiTraitInterfaceCallWithError( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn, + lowerError: FfiConverterTypeHttpError_lower + ) + } + ) + + // Rust stores this pointer for future callback invocations, so it must live + // for the process lifetime (not just for the init function call). + // + // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. + // This is safe because the pointee is initialized once during static init + // and never mutated by either side of the FFI. Its fields are C function pointers. + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: vtable) + return UnsafePointer(ptr) + }() +} + +private func uniffiCallbackInitHttpClient() { + uniffi_liters_ffi_fn_init_callback_vtable_httpclient(UniffiCallbackInterfaceHttpClient.vtablePtr) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpClient: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() + + typealias FfiType = UInt64 + typealias SwiftType = HttpClient + + public static func lift(_ handle: UInt64) throws -> HttpClient { + if ((handle & 1) == 0) { + // Rust-generated handle, construct a new class that uses the handle to implement the + // interface + return HttpClientImpl(unsafeFromHandle: handle) + } else { + // Swift-generated handle, get the object from the handle map + return try handleMap.remove(handle: handle) + } + } + + public static func lower(_ value: HttpClient) -> UInt64 { + if let rustImpl = value as? HttpClientImpl { + // Rust-implemented object. Clone the handle and return it + return rustImpl.uniffiCloneHandle() + } else { + // Swift object, generate a new vtable handle and return that. + return handleMap.insert(obj: value) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpClient { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: HttpClient, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpClient_lift(_ handle: UInt64) throws -> HttpClient { + return try FfiConverterTypeHttpClient.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpClient_lower(_ value: HttpClient) -> UInt64 { + return FfiConverterTypeHttpClient.lower(value) +} + + + + + + +/** + * The request-body source for a `PUT`, implemented by liters and *called by + * the host*: the host pulls bytes as it streams the request body, so a push + * is never buffered whole. `read` returns up to `max` bytes; an empty result + * is end-of-body. Throwing aborts the upload. + */ +public protocol HttpRequestBodyProtocol: AnyObject, Sendable { + + func read(max: UInt32) throws -> Data + +} +/** + * The request-body source for a `PUT`, implemented by liters and *called by + * the host*: the host pulls bytes as it streams the request body, so a push + * is never buffered whole. `read` returns up to `max` bytes; an empty result + * is end-of-body. Throwing aborts the upload. + */ +open class HttpRequestBody: HttpRequestBodyProtocol, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_httprequestbody(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_httprequestbody(handle, $0) } + } + + + + +open func read(max: UInt32)throws -> Data { + return try FfiConverterData.lift(try rustCallWithError(FfiConverterTypeHttpError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httprequestbody_read( + self.uniffiCloneHandle(), + FfiConverterUInt32.lower(max),uniffiCallStatus + ) +}) +} + + + +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpRequestBody: FfiConverter { + typealias FfiType = UInt64 + typealias SwiftType = HttpRequestBody + + public static func lift(_ handle: UInt64) throws -> HttpRequestBody { + return HttpRequestBody(unsafeFromHandle: handle) + } + + public static func lower(_ value: HttpRequestBody) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpRequestBody { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: HttpRequestBody, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpRequestBody_lift(_ handle: UInt64) throws -> HttpRequestBody { + return try FfiConverterTypeHttpRequestBody.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpRequestBody_lower(_ value: HttpRequestBody) -> UInt64 { + return FfiConverterTypeHttpRequestBody.lower(value) +} + + + + + + +/** + * A response, implemented by the host over its platform client's response. + */ +public protocol HttpResponse: AnyObject, Sendable { + + func status() -> UInt16 + + func headers() -> [HttpHeader] + + /** + * Pulls the next body bytes, blocking until data, an idle tick + * (`long_lived` only), or end-of-body. Throwing ends the body with a + * transport/protocol error. + */ + func readBody(max: UInt32) throws -> BodyChunk + + /** + * Aborts the in-flight call (e.g. OkHttp `Call.cancel()`). Best-effort and + * idempotent; must not throw. + */ + func cancel() + +} +/** + * A response, implemented by the host over its platform client's response. + */ +open class HttpResponseImpl: HttpResponse, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_httpresponse(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_httpresponse(handle, $0) } + } + + + + +open func status() -> UInt16 { + return try! FfiConverterUInt16.lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httpresponse_status( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + +open func headers() -> [HttpHeader] { + return try! FfiConverterSequenceTypeHttpHeader.lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httpresponse_headers( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Pulls the next body bytes, blocking until data, an idle tick + * (`long_lived` only), or end-of-body. Throwing ends the body with a + * transport/protocol error. + */ +open func readBody(max: UInt32)throws -> BodyChunk { + return try FfiConverterTypeBodyChunk_lift(try rustCallWithError(FfiConverterTypeHttpError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httpresponse_read_body( + self.uniffiCloneHandle(), + FfiConverterUInt32.lower(max),uniffiCallStatus + ) +}) +} + + /** + * Aborts the in-flight call (e.g. OkHttp `Call.cancel()`). Best-effort and + * idempotent; must not throw. + */ +open func cancel() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_httpresponse_cancel( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + + +} + + + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct UniffiCallbackInterfaceHttpResponse { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + // + // Store the vtable directly. + static let vtable: UniffiVTableCallbackInterfaceHttpResponse = UniffiVTableCallbackInterfaceHttpResponse( + uniffiFree: { (uniffiHandle: UInt64) -> () in + do { + try FfiConverterTypeHttpResponse.handleMap.remove(handle: uniffiHandle) + } catch { + print("Uniffi callback interface HttpResponse: handle missing in uniffiFree") + } + }, + uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in + do { + return try FfiConverterTypeHttpResponse.handleMap.clone(handle: uniffiHandle) + } catch { + fatalError("Uniffi callback interface HttpResponse: handle missing in uniffiClone") + } + }, + status: { ( + uniffiHandle: UInt64, + uniffiOutReturn: UnsafeMutablePointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> UInt16 in + guard let uniffiObj = try? FfiConverterTypeHttpResponse.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.status( + ) + } + + + let writeReturn = { uniffiOutReturn.pointee = FfiConverterUInt16.lower($0) } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + }, + headers: { ( + uniffiHandle: UInt64, + uniffiOutReturn: UnsafeMutablePointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> [HttpHeader] in + guard let uniffiObj = try? FfiConverterTypeHttpResponse.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.headers( + ) + } + + + let writeReturn = { uniffiOutReturn.pointee = FfiConverterSequenceTypeHttpHeader.lower($0) } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + }, + readBody: { ( + uniffiHandle: UInt64, + max: UInt32, + uniffiOutReturn: UnsafeMutablePointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> BodyChunk in + guard let uniffiObj = try? FfiConverterTypeHttpResponse.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return try uniffiObj.readBody( + max: try FfiConverterUInt32.lift(max) + ) + } + + + let writeReturn = { uniffiOutReturn.pointee = FfiConverterTypeBodyChunk_lower($0) } + uniffiTraitInterfaceCallWithError( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn, + lowerError: FfiConverterTypeHttpError_lower + ) + }, + cancel: { ( + uniffiHandle: UInt64, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeHttpResponse.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.cancel( + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + } + ) + + // Rust stores this pointer for future callback invocations, so it must live + // for the process lifetime (not just for the init function call). + // + // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. + // This is safe because the pointee is initialized once during static init + // and never mutated by either side of the FFI. Its fields are C function pointers. + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: vtable) + return UnsafePointer(ptr) + }() +} + +private func uniffiCallbackInitHttpResponse() { + uniffi_liters_ffi_fn_init_callback_vtable_httpresponse(UniffiCallbackInterfaceHttpResponse.vtablePtr) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpResponse: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() + + typealias FfiType = UInt64 + typealias SwiftType = HttpResponse + + public static func lift(_ handle: UInt64) throws -> HttpResponse { + if ((handle & 1) == 0) { + // Rust-generated handle, construct a new class that uses the handle to implement the + // interface + return HttpResponseImpl(unsafeFromHandle: handle) + } else { + // Swift-generated handle, get the object from the handle map + return try handleMap.remove(handle: handle) + } + } + + public static func lower(_ value: HttpResponse) -> UInt64 { + if let rustImpl = value as? HttpResponseImpl { + // Rust-implemented object. Clone the handle and return it + return rustImpl.uniffiCloneHandle() + } else { + // Swift object, generate a new vtable handle and return that. + return handleMap.insert(obj: value) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpResponse { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: HttpResponse, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpResponse_lift(_ handle: UInt64) throws -> HttpResponse { + return try FfiConverterTypeHttpResponse.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpResponse_lower(_ value: HttpResponse) -> UInt64 { + return FfiConverterTypeHttpResponse.lower(value) +} + + + + + + +/** + * Runs replication for any number of registered databases on background + * threads: each database either pushes to a bucket or follows one, with + * automatic retry/backoff, and per-database or global sleep/resume for + * mobile power management. `sleep()` cancels in-flight transfers and + * releases the database's WAL lock and file descriptors; `resume()` + * schedules an immediate catch-up round. + * + * All methods return immediately except `unregister()` and `shutdown()`, + * which join worker threads (bounded by the storage backend's cancellation + * latency — a few seconds worst case over HTTP). Call `shutdown()` on app + * teardown for deterministic cleanup; it also runs when the object is + * destroyed. + */ +public protocol LitersManagerProtocol: AnyObject, Sendable { + + /** + * Nudges a push registration to run a round now (also the retry path + * out of `Failed`). Ignored while sleeping — resume first. Errors on + * an unknown id or a follow registration. + */ + func pushNow(id: String) throws + + /** + * Registers a local path to follow (materialize) a bucket and starts + * its worker. `db_path` need not exist yet — the first sync restores + * it. Errors on a duplicate id or an already-registered path. + */ + func registerFollow(id: String, dbPath: String, storage: Storage, options: FollowOptions) throws + + /** + * Registers a local database to push to `storage` and starts its + * worker. The database file must exist. Errors on a duplicate id or an + * already-registered path. For HTTP storage the manager persists a + * per-database writer id next to the database and sends it for + * server-side fencing. + */ + func registerPush(id: String, dbPath: String, storage: Storage, options: PushOptions) throws + + /** + * Wakes a sleeping database and schedules an immediate catch-up round. + * On a running-but-`Failed` database this acts as a retry nudge. + * Idempotent. + */ + func resume(id: String) throws + + /** + * `resume()` for every registered database (app returned to + * foreground). + */ + func resumeAll() + + /** + * Installs (or clears, with `null`) the event listener. See + * `ManagerListener` for the threading rules. + */ + func setListener(listener: ManagerListener?) + + /** + * Cancels every session and joins every worker thread. Idempotent; + * also runs when the object is destroyed. + */ + func shutdown() + + /** + * Puts one database to sleep: aborts its in-flight transfer, releases + * its WAL lock and file descriptors, and stops all storage traffic + * until `resume()`. Returns immediately; teardown completes on the + * worker. Idempotent. + */ + func sleep(id: String) throws + + /** + * `sleep()` for every registered database (app went to background). + */ + func sleepAll() + + /** + * Status of one registered database, or `null` for an unknown id. + */ + func status(id: String) -> DbStatus? + + /** + * Statuses of all registered databases, sorted by id. + */ + func statuses() -> [DbStatus] + + /** + * Forces a follow registration to resync immediately by restarting its + * follow session. Also the retry path out of `Failed`. Ignored while + * sleeping. Errors on an unknown id or a push registration. + */ + func syncNow(id: String) throws + + /** + * Removes a registration: cancels its session, joins its worker, and + * releases its database. Blocks briefly; the id becomes free for + * re-registration. Unknown id errors. + */ + func unregister(id: String) throws + +} +/** + * Runs replication for any number of registered databases on background + * threads: each database either pushes to a bucket or follows one, with + * automatic retry/backoff, and per-database or global sleep/resume for + * mobile power management. `sleep()` cancels in-flight transfers and + * releases the database's WAL lock and file descriptors; `resume()` + * schedules an immediate catch-up round. + * + * All methods return immediately except `unregister()` and `shutdown()`, + * which join worker threads (bounded by the storage backend's cancellation + * latency — a few seconds worst case over HTTP). Call `shutdown()` on app + * teardown for deterministic cleanup; it also runs when the object is + * destroyed. + */ +open class LitersManager: LitersManagerProtocol, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_litersmanager(self.handle, $0) } + } + /** + * A manager with default options (liters backoff, no default push + * interval — databases push on `push_now()` unless registered with + * their own interval) and no host HTTP transport. + */ +public convenience init() { + let handle = + try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_litersmanager_new(uniffiCallStatus + ) +} + self.init(unsafeFromHandle: handle) +} + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_litersmanager(handle, $0) } + } + + + /** + * A manager with explicit options and, optionally, a host HTTP transport. + * + * Pass `http_client` (one shared instance backed by a single platform + * HTTP client, e.g. one `OkHttpClient`) to have every HTTP follow/push run + * through it: all followers to one authority then coalesce onto a single + * HTTP/2 connection, with the platform owning TLS, the system trust store, + * and keepalive. `null` keeps the built-in socket transport (one + * `Connection: close` request per call, `http://` only). + */ +public static func withOptions(options: ManagerOptions, httpClient: HttpClient?) -> LitersManager { + return try! FfiConverterTypeLitersManager_lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_litersmanager_with_options( + FfiConverterTypeManagerOptions_lower(options), + FfiConverterOptionTypeHttpClient.lower(httpClient),uniffiCallStatus + ) +}) +} + + + + /** + * Nudges a push registration to run a round now (also the retry path + * out of `Failed`). Ignored while sleeping — resume first. Errors on + * an unknown id or a follow registration. + */ +open func pushNow(id: String)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_push_now( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +} +} + + /** + * Registers a local path to follow (materialize) a bucket and starts + * its worker. `db_path` need not exist yet — the first sync restores + * it. Errors on a duplicate id or an already-registered path. + */ +open func registerFollow(id: String, dbPath: String, storage: Storage, options: FollowOptions)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_register_follow( + self.uniffiCloneHandle(), + FfiConverterString.lower(id), + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage), + FfiConverterTypeFollowOptions_lower(options),uniffiCallStatus + ) +} +} + + /** + * Registers a local database to push to `storage` and starts its + * worker. The database file must exist. Errors on a duplicate id or an + * already-registered path. For HTTP storage the manager persists a + * per-database writer id next to the database and sends it for + * server-side fencing. + */ +open func registerPush(id: String, dbPath: String, storage: Storage, options: PushOptions)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_register_push( + self.uniffiCloneHandle(), + FfiConverterString.lower(id), + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage), + FfiConverterTypePushOptions_lower(options),uniffiCallStatus + ) +} +} + + /** + * Wakes a sleeping database and schedules an immediate catch-up round. + * On a running-but-`Failed` database this acts as a retry nudge. + * Idempotent. + */ +open func resume(id: String)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_resume( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +} +} + + /** + * `resume()` for every registered database (app returned to + * foreground). + */ +open func resumeAll() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_resume_all( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Installs (or clears, with `null`) the event listener. See + * `ManagerListener` for the threading rules. + */ +open func setListener(listener: ManagerListener?) {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_set_listener( + self.uniffiCloneHandle(), + FfiConverterOptionTypeManagerListener.lower(listener),uniffiCallStatus + ) +} +} + + /** + * Cancels every session and joins every worker thread. Idempotent; + * also runs when the object is destroyed. + */ +open func shutdown() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_shutdown( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Puts one database to sleep: aborts its in-flight transfer, releases + * its WAL lock and file descriptors, and stops all storage traffic + * until `resume()`. Returns immediately; teardown completes on the + * worker. Idempotent. + */ +open func sleep(id: String)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_sleep( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +} +} + + /** + * `sleep()` for every registered database (app went to background). + */ +open func sleepAll() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_sleep_all( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Status of one registered database, or `null` for an unknown id. + */ +open func status(id: String) -> DbStatus? { + return try! FfiConverterOptionTypeDbStatus.lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_status( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +}) +} + + /** + * Statuses of all registered databases, sorted by id. + */ +open func statuses() -> [DbStatus] { + return try! FfiConverterSequenceTypeDbStatus.lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_statuses( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Forces a follow registration to resync immediately by restarting its + * follow session. Also the retry path out of `Failed`. Ignored while + * sleeping. Errors on an unknown id or a push registration. + */ +open func syncNow(id: String)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_sync_now( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +} +} + + /** + * Removes a registration: cancels its session, joins its worker, and + * releases its database. Blocks briefly; the id becomes free for + * re-registration. Unknown id errors. + */ +open func unregister(id: String)throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersmanager_unregister( + self.uniffiCloneHandle(), + FfiConverterString.lower(id),uniffiCallStatus + ) +} +} + + + +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeLitersManager: FfiConverter { + typealias FfiType = UInt64 + typealias SwiftType = LitersManager + + public static func lift(_ handle: UInt64) throws -> LitersManager { + return LitersManager(unsafeFromHandle: handle) + } + + public static func lower(_ value: LitersManager) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LitersManager { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: LitersManager, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersManager_lift(_ handle: UInt64) throws -> LitersManager { + return try FfiConverterTypeLitersManager.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersManager_lower(_ value: LitersManager) -> UInt64 { + return FfiConverterTypeLitersManager.lower(value) +} + + + + + + +/** + * A local read-only materialization of a bucket. `sync()` restores on first + * use, then applies changes incrementally. Open `db_path()` read-only with + * your platform SQLite; do not hold read transactions across `sync()` calls + * from the same process. For continuous (live) following, register the + * database with `LitersManager.register_follow` instead of polling. + */ +public protocol LitersReplicaProtocol: AnyObject, Sendable { + + /** + * Aborts the in-flight `sync`, if any: it returns `Cancelled` promptly. + * Each call installs a fresh token at its start, so cancel only ever + * affects the operation currently running. A cancelled sync resumes + * exactly where it stopped on the next call. + */ + func cancel() + + /** + * Tears the replica down: cancels any in-flight sync, waits for it to + * return, then drops the storage client. Subsequent calls (except + * `db_path()`) error with "LitersReplica is closed" — including calls + * already queued behind the cancelled one. Idempotent. + */ + func close() + + /** + * Path of the local database file to open read-only. + */ + func dbPath() -> String + + /** + * Last applied TXID (zero if never synced). + */ + func position() throws -> UInt64 + + /** + * Deletes the local replica; the next `sync()` restores from scratch. + */ + func reset() throws + + /** + * Brings the local replica up to date. Short and resumable; safe to + * call from a background task with a tight deadline. + */ + func sync() throws -> SyncSummary + +} +/** + * A local read-only materialization of a bucket. `sync()` restores on first + * use, then applies changes incrementally. Open `db_path()` read-only with + * your platform SQLite; do not hold read transactions across `sync()` calls + * from the same process. For continuous (live) following, register the + * database with `LitersManager.register_follow` instead of polling. + */ +open class LitersReplica: LitersReplicaProtocol, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_litersreplica(self.handle, $0) } + } + /** + * `auto_reset`: on bucket divergence, silently delete the local replica + * and re-restore instead of raising `Diverged`. + */ +public convenience init(dbPath: String, storage: Storage, autoReset: Bool)throws { + let handle = + try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_litersreplica_new( + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage), + FfiConverterBool.lower(autoReset),uniffiCallStatus + ) +} + self.init(unsafeFromHandle: handle) +} + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_litersreplica(handle, $0) } + } + + + /** + * [`LitersReplica::new`], with the app's own HTTP client executing every + * `Storage::Http` request. See [`LitersWriter::new_with_http_client`] for + * what the built-in transport cannot do; `null` is exactly + * [`LitersReplica::new`]. + */ +public static func newWithHttpClient(dbPath: String, storage: Storage, autoReset: Bool, httpClient: HttpClient?)throws -> LitersReplica { + return try FfiConverterTypeLitersReplica_lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_litersreplica_new_with_http_client( + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage), + FfiConverterBool.lower(autoReset), + FfiConverterOptionTypeHttpClient.lower(httpClient),uniffiCallStatus + ) +}) +} + + + + /** + * Aborts the in-flight `sync`, if any: it returns `Cancelled` promptly. + * Each call installs a fresh token at its start, so cancel only ever + * affects the operation currently running. A cancelled sync resumes + * exactly where it stopped on the next call. + */ +open func cancel() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_cancel( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Tears the replica down: cancels any in-flight sync, waits for it to + * return, then drops the storage client. Subsequent calls (except + * `db_path()`) error with "LitersReplica is closed" — including calls + * already queued behind the cancelled one. Idempotent. + */ +open func close() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_close( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Path of the local database file to open read-only. + */ +open func dbPath() -> String { + return try! FfiConverterString.lift(try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_db_path( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Last applied TXID (zero if never synced). + */ +open func position()throws -> UInt64 { + return try FfiConverterUInt64.lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_position( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Deletes the local replica; the next `sync()` restores from scratch. + */ +open func reset()throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_reset( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Brings the local replica up to date. Short and resumable; safe to + * call from a background task with a tight deadline. + */ +open func sync()throws -> SyncSummary { + return try FfiConverterTypeSyncSummary_lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_litersreplica_sync( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + + +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeLitersReplica: FfiConverter { + typealias FfiType = UInt64 + typealias SwiftType = LitersReplica + + public static func lift(_ handle: UInt64) throws -> LitersReplica { + return LitersReplica(unsafeFromHandle: handle) + } + + public static func lower(_ value: LitersReplica) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LitersReplica { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: LitersReplica, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersReplica_lift(_ handle: UInt64) throws -> LitersReplica { + return try FfiConverterTypeLitersReplica.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersReplica_lower(_ value: LitersReplica) -> UInt64 { + return FfiConverterTypeLitersReplica.lower(value) +} + + + + + + +/** + * Replicates a local, app-owned SQLite database to a bucket. Call `push()` + * after commits (or batched); call `maintain()` opportunistically (wifi + + * charging) to compact and enforce retention. For multiple databases with + * scheduled pushes and sleep/resume, use `LitersManager` instead. + */ +public protocol LitersWriterProtocol: AnyObject, Sendable { + + /** + * Aborts the in-flight `push`/`maintain`/`snapshot`, if any: it returns + * `Cancelled` promptly (mid-transfer on the HTTP backend). Each call + * installs a fresh token at its start, so cancel only ever affects the + * operation currently running — never a future call. Cancel-then-retry + * is indistinguishable from kill-then-restart. + */ + func cancel() + + /** + * Tears the writer down: cancels any in-flight operation, waits for it + * to return, then releases the WAL read lock, SQLite connections, and + * file descriptors. Subsequent calls error with "LitersWriter is + * closed" — including calls already queued behind the cancelled one, + * which bail instead of running (so this never blocks for more than + * the single in-flight operation's cancellation latency). Idempotent. + */ + func close() + + /** + * Runs due compaction, snapshotting, and retention with litestream's + * default cadences. + */ + func maintain() throws -> MaintenanceSummary + + /** + * Current local replication position. + */ + func position() throws -> UInt64 + + /** + * Captures all committed changes into the bucket. Short and resumable: + * on failure, the next push picks up exactly where this one stopped. + */ + func push() throws -> PushSummary + + /** + * Recovery for a "local ltx file missing or corrupt" error: wipes local + * replication state so the next push re-derives everything from the + * bucket (snapshotting if needed). Local database content is untouched. + */ + func resetLocal() throws + + /** + * Forces a full snapshot to the bucket now. + */ + func snapshot() throws -> UInt64? + +} +/** + * Replicates a local, app-owned SQLite database to a bucket. Call `push()` + * after commits (or batched); call `maintain()` opportunistically (wifi + + * charging) to compact and enforce retention. For multiple databases with + * scheduled pushes and sleep/resume, use `LitersManager` instead. + */ +open class LitersWriter: LitersWriterProtocol, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_literswriter(self.handle, $0) } + } + /** + * Opens a writer for an existing SQLite database. Switches the database + * to WAL mode and takes over checkpointing: do not run your own + * `wal_checkpoint`, and **set `wal_autocheckpoint = 0` on every + * connection your app opens to this database**. + * + * That second half is not optional on mobile, which is the platform this + * crate exists for. The writer's read lock does starve a foreign + * autocheckpoint *while it is held* — but `LitersManager.sleep()` (and + * dropping the writer, and the app being killed) releases it, by design: + * see `Manager::sleep`, which "drops its Writer — releasing the + * WAL-pinning read transaction and every fd — and parks with zero storage + * traffic". On iOS that is most of the app's life. A foreign + * autocheckpoint that fires in that window restarts the WAL, and the next + * push finds its resume frame gone: `verify()` reports "wal truncated by + * another process" or "wal overwritten by another process" and recovers + * the only way it can, by writing a **full snapshot of the database**. + * + * Note also that `wal_autocheckpoint` is a per-*connection* setting, so a + * connection pool must apply it to every connection it opens, not once at + * startup. + * + * Turning autocheckpoint off makes liters the only checkpointer, which is + * the intent — but liters' own thresholds + * (`WriterOptions::min_checkpoint_page_n`, `truncate_page_n`) live inside + * `push()`. If pushes stop, nothing checkpoints and the WAL grows without + * bound, so an app that disables autocheckpoint should also keep its own + * size floor on the `-wal` file. + * + * Performs no network I/O: opening succeeds offline, and pushes + * accumulate local L0 files until the bucket becomes reachable (the + * first successful push then uploads the backlog). + */ +public convenience init(dbPath: String, storage: Storage)throws { + let handle = + try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_literswriter_new( + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage),uniffiCallStatus + ) +} + self.init(unsafeFromHandle: handle) +} + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_literswriter(handle, $0) } + } + + + /** + * [`LitersWriter::new`], with the app's own HTTP client executing every + * `Storage::Http` request. + * + * Without this, a `Storage::Http` writer runs on the built-in socket + * transport: `http://` only (an `https://` url is rejected, not silently + * downgraded), one `Connection: close` request per call, and — the part + * that matters on iOS — no way to hand the transfer to a background + * `URLSession`, so an upload in flight when the app suspends is lost. + * Pass the same shared `HttpClient` instance you would give + * `LitersManager`, so writers and followers to one authority coalesce + * onto a single connection. + * + * `null` is exactly [`LitersWriter::new`]. + */ +public static func newWithHttpClient(dbPath: String, storage: Storage, httpClient: HttpClient?)throws -> LitersWriter { + return try FfiConverterTypeLitersWriter_lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_constructor_literswriter_new_with_http_client( + FfiConverterString.lower(dbPath), + FfiConverterTypeStorage_lower(storage), + FfiConverterOptionTypeHttpClient.lower(httpClient),uniffiCallStatus + ) +}) +} + + + + /** + * Aborts the in-flight `push`/`maintain`/`snapshot`, if any: it returns + * `Cancelled` promptly (mid-transfer on the HTTP backend). Each call + * installs a fresh token at its start, so cancel only ever affects the + * operation currently running — never a future call. Cancel-then-retry + * is indistinguishable from kill-then-restart. + */ +open func cancel() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_cancel( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Tears the writer down: cancels any in-flight operation, waits for it + * to return, then releases the WAL read lock, SQLite connections, and + * file descriptors. Subsequent calls error with "LitersWriter is + * closed" — including calls already queued behind the cancelled one, + * which bail instead of running (so this never blocks for more than + * the single in-flight operation's cancellation latency). Idempotent. + */ +open func close() {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_close( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Runs due compaction, snapshotting, and retention with litestream's + * default cadences. + */ +open func maintain()throws -> MaintenanceSummary { + return try FfiConverterTypeMaintenanceSummary_lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_maintain( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Current local replication position. + */ +open func position()throws -> UInt64 { + return try FfiConverterUInt64.lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_position( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Captures all committed changes into the bucket. Short and resumable: + * on failure, the next push picks up exactly where this one stopped. + */ +open func push()throws -> PushSummary { + return try FfiConverterTypePushSummary_lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_push( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + /** + * Recovery for a "local ltx file missing or corrupt" error: wipes local + * replication state so the next push re-derives everything from the + * bucket (snapshotting if needed). Local database content is untouched. + */ +open func resetLocal()throws {try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_reset_local( + self.uniffiCloneHandle(),uniffiCallStatus + ) +} +} + + /** + * Forces a full snapshot to the bucket now. + */ +open func snapshot()throws -> UInt64? { + return try FfiConverterOptionUInt64.lift(try rustCallWithError(FfiConverterTypeLitersError_lift) { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_literswriter_snapshot( + self.uniffiCloneHandle(),uniffiCallStatus + ) +}) +} + + + +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeLitersWriter: FfiConverter { + typealias FfiType = UInt64 + typealias SwiftType = LitersWriter + + public static func lift(_ handle: UInt64) throws -> LitersWriter { + return LitersWriter(unsafeFromHandle: handle) + } + + public static func lower(_ value: LitersWriter) -> UInt64 { + return value.uniffiCloneHandle() + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LitersWriter { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: LitersWriter, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersWriter_lift(_ handle: UInt64) throws -> LitersWriter { + return try FfiConverterTypeLitersWriter.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersWriter_lower(_ value: LitersWriter) -> UInt64 { + return FfiConverterTypeLitersWriter.lower(value) +} + + + + + + +/** + * Receives manager events. Callbacks are delivered on liters worker + * threads (and, for the `Sleeping` state change, on the thread that called + * `sleep()`): they must return quickly, must never block, and must not + * call back into `unregister()`/`shutdown()` — hand the event to your own + * queue/handler instead. + */ +public protocol ManagerListener: AnyObject, Sendable { + + func onEvent(event: ManagerEvent) + +} +/** + * Receives manager events. Callbacks are delivered on liters worker + * threads (and, for the `Sleeping` state change, on the thread that called + * `sleep()`): they must return quickly, must never block, and must not + * call back into `unregister()`/`shutdown()` — hand the event to your own + * queue/handler instead. + */ +open class ManagerListenerImpl: ManagerListener, @unchecked Sendable { + fileprivate let handle: UInt64 + + /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public struct NoHandle { + public init() {} + } + + // TODO: We'd like this to be `private` but for Swifty reasons, + // we can't implement `FfiConverter` without making this `required` and we can't + // make it `required` without making it `public`. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + required public init(unsafeFromHandle handle: UInt64) { + self.handle = handle + } + + // This constructor can be used to instantiate a fake object. + // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + // + // - Warning: + // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noHandle: NoHandle) { + self.handle = 0 + } + +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public func uniffiCloneHandle() -> UInt64 { + return try! rustCall { uniffi_liters_ffi_fn_clone_managerlistener(self.handle, $0) } + } + // No primary constructor declared for this class. + + deinit { + if handle == 0 { + // Mock objects have handle=0 don't try to free them + return + } + + try! rustCall { uniffi_liters_ffi_fn_free_managerlistener(handle, $0) } + } + + + + +open func onEvent(event: ManagerEvent) {try! rustCall() { + uniffiCallStatus in + uniffi_liters_ffi_fn_method_managerlistener_on_event( + self.uniffiCloneHandle(), + FfiConverterTypeManagerEvent_lower(event),uniffiCallStatus + ) +} +} + + + +} + + + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct UniffiCallbackInterfaceManagerListener { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + // + // Store the vtable directly. + static let vtable: UniffiVTableCallbackInterfaceManagerListener = UniffiVTableCallbackInterfaceManagerListener( + uniffiFree: { (uniffiHandle: UInt64) -> () in + do { + try FfiConverterTypeManagerListener.handleMap.remove(handle: uniffiHandle) + } catch { + print("Uniffi callback interface ManagerListener: handle missing in uniffiFree") + } + }, + uniffiClone: { (uniffiHandle: UInt64) -> UInt64 in + do { + return try FfiConverterTypeManagerListener.handleMap.clone(handle: uniffiHandle) + } catch { + fatalError("Uniffi callback interface ManagerListener: handle missing in uniffiClone") + } + }, + onEvent: { ( + uniffiHandle: UInt64, + event: RustBuffer, + uniffiOutReturn: UnsafeMutableRawPointer, + uniffiCallStatus: UnsafeMutablePointer + ) in + let makeCall = { + () throws -> () in + guard let uniffiObj = try? FfiConverterTypeManagerListener.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return uniffiObj.onEvent( + event: try FfiConverterTypeManagerEvent_lift(event) + ) + } + + + let writeReturn = { () } + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + } + ) + + // Rust stores this pointer for future callback invocations, so it must live + // for the process lifetime (not just for the init function call). + // + // `nonisolated(unsafe)` is needed under Swift 6 strict concurrency. + // This is safe because the pointee is initialized once during static init + // and never mutated by either side of the FFI. Its fields are C function pointers. + nonisolated(unsafe) static let vtablePtr: UnsafePointer = { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: vtable) + return UnsafePointer(ptr) + }() +} + +private func uniffiCallbackInitManagerListener() { + uniffi_liters_ffi_fn_init_callback_vtable_managerlistener(UniffiCallbackInterfaceManagerListener.vtablePtr) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeManagerListener: FfiConverter { + fileprivate static let handleMap = UniffiHandleMap() + + typealias FfiType = UInt64 + typealias SwiftType = ManagerListener + + public static func lift(_ handle: UInt64) throws -> ManagerListener { + if ((handle & 1) == 0) { + // Rust-generated handle, construct a new class that uses the handle to implement the + // interface + return ManagerListenerImpl(unsafeFromHandle: handle) + } else { + // Swift-generated handle, get the object from the handle map + return try handleMap.remove(handle: handle) + } + } + + public static func lower(_ value: ManagerListener) -> UInt64 { + if let rustImpl = value as? ManagerListenerImpl { + // Rust-implemented object. Clone the handle and return it + return rustImpl.uniffiCloneHandle() + } else { + // Swift object, generate a new vtable handle and return that. + return handleMap.insert(obj: value) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ManagerListener { + let handle: UInt64 = try readInt(&buf) + return try lift(handle) + } + + public static func write(_ value: ManagerListener, into buf: inout [UInt8]) { + writeInt(&buf, lower(value)) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerListener_lift(_ handle: UInt64) throws -> ManagerListener { + return try FfiConverterTypeManagerListener.lift(handle) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerListener_lower(_ value: ManagerListener) -> UInt64 { + return FfiConverterTypeManagerListener.lower(value) +} + + + + +/** + * Retry schedule for transient failures: exponential growth from + * `initial_ms` by `multiplier` per consecutive failure, saturating at + * `max_ms`, with ±`jitter` randomization. The defaults are the liters + * defaults (500ms → 60s, ×2, ±25%). + */ +public struct Backoff: Equatable, Hashable { + /** + * Delay before the first retry, in milliseconds. + */ + public var initialMs: UInt64 + /** + * Ceiling (pre-jitter) the schedule saturates at, in milliseconds. + */ + public var maxMs: UInt64 + /** + * Growth factor per consecutive failure. + */ + public var multiplier: Double + /** + * Jitter fraction in [0, 1]; 0 disables jitter. + */ + public var jitter: Double + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * Delay before the first retry, in milliseconds. + */initialMs: UInt64 = UInt64(500), + /** + * Ceiling (pre-jitter) the schedule saturates at, in milliseconds. + */maxMs: UInt64 = UInt64(60000), + /** + * Growth factor per consecutive failure. + */multiplier: Double = Double(2.0), + /** + * Jitter fraction in [0, 1]; 0 disables jitter. + */jitter: Double = Double(0.25)) { + self.initialMs = initialMs + self.maxMs = maxMs + self.multiplier = multiplier + self.jitter = jitter + } + + + + +} + +#if compiler(>=6) +extension Backoff: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeBackoff: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Backoff { + return + try Backoff( + initialMs: FfiConverterUInt64.read(from: &buf), + maxMs: FfiConverterUInt64.read(from: &buf), + multiplier: FfiConverterDouble.read(from: &buf), + jitter: FfiConverterDouble.read(from: &buf) + ) + } + + public static func write(_ value: Backoff, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.initialMs, into: &buf) + FfiConverterUInt64.write(value.maxMs, into: &buf) + FfiConverterDouble.write(value.multiplier, into: &buf) + FfiConverterDouble.write(value.jitter, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBackoff_lift(_ buf: RustBuffer) throws -> Backoff { + return try FfiConverterTypeBackoff.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBackoff_lower(_ value: Backoff) -> RustBuffer { + return FfiConverterTypeBackoff.lower(value) +} + + +/** + * Point-in-time snapshot of one registered database. + */ +public struct DbStatus: Equatable, Hashable { + public var id: String + public var role: DbRole + public var state: DbState + /** + * Replication position (TXID): for pushes, the last successful push; + * for follows, the last applied transaction. `null` until the first + * push/apply. + */ + public var position: UInt64? + /** + * Most recent error message; cleared by the next successful round. + */ + public var lastError: String? + /** + * Completion time of the last successful push/apply, in milliseconds + * since the Unix epoch. + */ + public var lastActivityMs: UInt64? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(id: String, role: DbRole, state: DbState, + /** + * Replication position (TXID): for pushes, the last successful push; + * for follows, the last applied transaction. `null` until the first + * push/apply. + */position: UInt64?, + /** + * Most recent error message; cleared by the next successful round. + */lastError: String?, + /** + * Completion time of the last successful push/apply, in milliseconds + * since the Unix epoch. + */lastActivityMs: UInt64?) { + self.id = id + self.role = role + self.state = state + self.position = position + self.lastError = lastError + self.lastActivityMs = lastActivityMs + } + + + + +} + +#if compiler(>=6) +extension DbStatus: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeDbStatus: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DbStatus { + return + try DbStatus( + id: FfiConverterString.read(from: &buf), + role: FfiConverterTypeDbRole.read(from: &buf), + state: FfiConverterTypeDbState.read(from: &buf), + position: FfiConverterOptionUInt64.read(from: &buf), + lastError: FfiConverterOptionString.read(from: &buf), + lastActivityMs: FfiConverterOptionUInt64.read(from: &buf) + ) + } + + public static func write(_ value: DbStatus, into buf: inout [UInt8]) { + FfiConverterString.write(value.id, into: &buf) + FfiConverterTypeDbRole.write(value.role, into: &buf) + FfiConverterTypeDbState.write(value.state, into: &buf) + FfiConverterOptionUInt64.write(value.position, into: &buf) + FfiConverterOptionString.write(value.lastError, into: &buf) + FfiConverterOptionUInt64.write(value.lastActivityMs, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbStatus_lift(_ buf: RustBuffer) throws -> DbStatus { + return try FfiConverterTypeDbStatus.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbStatus_lower(_ value: DbStatus) -> RustBuffer { + return FfiConverterTypeDbStatus.lower(value) +} + + +/** + * Per-database options for `LitersManager.register_follow`. + */ +public struct FollowOptions: Equatable, Hashable { + /** + * On bucket divergence, silently delete the local replica and + * re-restore instead of parking in the `Failed` state. + */ + public var autoReset: Bool + /** + * Poll cadence in milliseconds for waiting out an empty bucket and for + * backends without live streaming. `null` = 1000. + */ + public var pollIntervalMs: UInt64? + /** + * Backoff for transient reconnects inside the follow loop; `null` uses + * the manager's backoff. + */ + public var retry: Backoff? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * On bucket divergence, silently delete the local replica and + * re-restore instead of parking in the `Failed` state. + */autoReset: Bool = false, + /** + * Poll cadence in milliseconds for waiting out an empty bucket and for + * backends without live streaming. `null` = 1000. + */pollIntervalMs: UInt64? = nil, + /** + * Backoff for transient reconnects inside the follow loop; `null` uses + * the manager's backoff. + */retry: Backoff? = nil) { + self.autoReset = autoReset + self.pollIntervalMs = pollIntervalMs + self.retry = retry + } + + + + +} + +#if compiler(>=6) +extension FollowOptions: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeFollowOptions: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> FollowOptions { + return + try FollowOptions( + autoReset: FfiConverterBool.read(from: &buf), + pollIntervalMs: FfiConverterOptionUInt64.read(from: &buf), + retry: FfiConverterOptionTypeBackoff.read(from: &buf) + ) + } + + public static func write(_ value: FollowOptions, into buf: inout [UInt8]) { + FfiConverterBool.write(value.autoReset, into: &buf) + FfiConverterOptionUInt64.write(value.pollIntervalMs, into: &buf) + FfiConverterOptionTypeBackoff.write(value.retry, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeFollowOptions_lift(_ buf: RustBuffer) throws -> FollowOptions { + return try FfiConverterTypeFollowOptions.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeFollowOptions_lower(_ value: FollowOptions) -> RustBuffer { + return FfiConverterTypeFollowOptions.lower(value) +} + + +/** + * One HTTP header, carried across the FFI in both directions. + */ +public struct HttpHeader: Equatable, Hashable { + public var name: String + public var value: String + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(name: String, value: String) { + self.name = name + self.value = value + } + + + + +} + +#if compiler(>=6) +extension HttpHeader: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpHeader: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpHeader { + return + try HttpHeader( + name: FfiConverterString.read(from: &buf), + value: FfiConverterString.read(from: &buf) + ) + } + + public static func write(_ value: HttpHeader, into buf: inout [UInt8]) { + FfiConverterString.write(value.name, into: &buf) + FfiConverterString.write(value.value, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpHeader_lift(_ buf: RustBuffer) throws -> HttpHeader { + return try FfiConverterTypeHttpHeader.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpHeader_lower(_ value: HttpHeader) -> RustBuffer { + return FfiConverterTypeHttpHeader.lower(value) +} + + +/** + * A request for the host `HttpClient` to execute. `headers` are semantic only + * (`authorization`, the fencing headers); the host adds its own transport + * headers (`host`, framing, connection). `url` is absolute and may be + * `https`. + */ +public struct HttpRequest: Equatable, Hashable { + public var method: String + public var url: String + public var headers: [HttpHeader] + /** + * `true` only for the long-lived `/stream` follow: the host keeps the + * response open and returns `BodyChunk.Idle` on its read-timeout tick + * (roughly once a second) instead of failing, so liters can run its + * dead-man timer and stay cancel-responsive. For `false` the host uses + * its normal read timeout and *fails* (throws) a stalled read. + */ + public var longLived: Bool + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(method: String, url: String, headers: [HttpHeader], + /** + * `true` only for the long-lived `/stream` follow: the host keeps the + * response open and returns `BodyChunk.Idle` on its read-timeout tick + * (roughly once a second) instead of failing, so liters can run its + * dead-man timer and stay cancel-responsive. For `false` the host uses + * its normal read timeout and *fails* (throws) a stalled read. + */longLived: Bool) { + self.method = method + self.url = url + self.headers = headers + self.longLived = longLived + } + + + + +} + +#if compiler(>=6) +extension HttpRequest: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpRequest: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpRequest { + return + try HttpRequest( + method: FfiConverterString.read(from: &buf), + url: FfiConverterString.read(from: &buf), + headers: FfiConverterSequenceTypeHttpHeader.read(from: &buf), + longLived: FfiConverterBool.read(from: &buf) + ) + } + + public static func write(_ value: HttpRequest, into buf: inout [UInt8]) { + FfiConverterString.write(value.method, into: &buf) + FfiConverterString.write(value.url, into: &buf) + FfiConverterSequenceTypeHttpHeader.write(value.headers, into: &buf) + FfiConverterBool.write(value.longLived, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpRequest_lift(_ buf: RustBuffer) throws -> HttpRequest { + return try FfiConverterTypeHttpRequest.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpRequest_lower(_ value: HttpRequest) -> RustBuffer { + return FfiConverterTypeHttpRequest.lower(value) +} + + +public struct MaintenanceSummary: Equatable, Hashable { + public var compactedLevels: Data + public var snapshotTxid: UInt64? + public var deletedFiles: UInt64 + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init(compactedLevels: Data, snapshotTxid: UInt64?, deletedFiles: UInt64) { + self.compactedLevels = compactedLevels + self.snapshotTxid = snapshotTxid + self.deletedFiles = deletedFiles + } + + + + +} + +#if compiler(>=6) +extension MaintenanceSummary: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeMaintenanceSummary: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> MaintenanceSummary { + return + try MaintenanceSummary( + compactedLevels: FfiConverterData.read(from: &buf), + snapshotTxid: FfiConverterOptionUInt64.read(from: &buf), + deletedFiles: FfiConverterUInt64.read(from: &buf) + ) + } + + public static func write(_ value: MaintenanceSummary, into buf: inout [UInt8]) { + FfiConverterData.write(value.compactedLevels, into: &buf) + FfiConverterOptionUInt64.write(value.snapshotTxid, into: &buf) + FfiConverterUInt64.write(value.deletedFiles, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeMaintenanceSummary_lift(_ buf: RustBuffer) throws -> MaintenanceSummary { + return try FfiConverterTypeMaintenanceSummary.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeMaintenanceSummary_lower(_ value: MaintenanceSummary) -> RustBuffer { + return FfiConverterTypeMaintenanceSummary.lower(value) +} + + +/** + * Manager-wide defaults (see `LitersManager.with_options`). + */ +public struct ManagerOptions: Equatable, Hashable { + /** + * Backoff for transient failures; overridable per push registration + * and used as the follow-loop retry schedule. `null` = liters + * defaults. + */ + public var backoff: Backoff? + /** + * Fallback push cadence (milliseconds) for registrations whose own + * `push_interval_ms` is `null`. When both are `null`, those databases + * push only on `push_now()`. + */ + public var defaultPushIntervalMs: UInt64? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * Backoff for transient failures; overridable per push registration + * and used as the follow-loop retry schedule. `null` = liters + * defaults. + */backoff: Backoff? = nil, + /** + * Fallback push cadence (milliseconds) for registrations whose own + * `push_interval_ms` is `null`. When both are `null`, those databases + * push only on `push_now()`. + */defaultPushIntervalMs: UInt64? = nil) { + self.backoff = backoff + self.defaultPushIntervalMs = defaultPushIntervalMs + } + + + + +} + +#if compiler(>=6) +extension ManagerOptions: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeManagerOptions: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ManagerOptions { + return + try ManagerOptions( + backoff: FfiConverterOptionTypeBackoff.read(from: &buf), + defaultPushIntervalMs: FfiConverterOptionUInt64.read(from: &buf) + ) + } + + public static func write(_ value: ManagerOptions, into buf: inout [UInt8]) { + FfiConverterOptionTypeBackoff.write(value.backoff, into: &buf) + FfiConverterOptionUInt64.write(value.defaultPushIntervalMs, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerOptions_lift(_ buf: RustBuffer) throws -> ManagerOptions { + return try FfiConverterTypeManagerOptions.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerOptions_lower(_ value: ManagerOptions) -> RustBuffer { + return FfiConverterTypeManagerOptions.lower(value) +} + + +/** + * Per-database options for `LitersManager.register_push`. + */ +public struct PushOptions: Equatable, Hashable { + /** + * Push cadence in milliseconds. `null` falls back to the manager's + * `default_push_interval_ms`; if that is also `null`, this database + * pushes only on `push_now()`. With an interval set, the first push + * runs immediately after registration. + */ + public var pushIntervalMs: UInt64? + /** + * When set, maintenance (compaction/snapshot/retention with + * litestream's default cadences) is attempted after a successful push + * once this many milliseconds have passed since the last attempt. A + * maintenance failure is reported but never blocks future pushes. + */ + public var maintenanceIntervalMs: UInt64? + /** + * Per-database transient-failure backoff; `null` uses the manager's. + */ + public var backoff: Backoff? + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * Push cadence in milliseconds. `null` falls back to the manager's + * `default_push_interval_ms`; if that is also `null`, this database + * pushes only on `push_now()`. With an interval set, the first push + * runs immediately after registration. + */pushIntervalMs: UInt64? = nil, + /** + * When set, maintenance (compaction/snapshot/retention with + * litestream's default cadences) is attempted after a successful push + * once this many milliseconds have passed since the last attempt. A + * maintenance failure is reported but never blocks future pushes. + */maintenanceIntervalMs: UInt64? = nil, + /** + * Per-database transient-failure backoff; `null` uses the manager's. + */backoff: Backoff? = nil) { + self.pushIntervalMs = pushIntervalMs + self.maintenanceIntervalMs = maintenanceIntervalMs + self.backoff = backoff + } + + + + +} + +#if compiler(>=6) +extension PushOptions: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypePushOptions: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PushOptions { + return + try PushOptions( + pushIntervalMs: FfiConverterOptionUInt64.read(from: &buf), + maintenanceIntervalMs: FfiConverterOptionUInt64.read(from: &buf), + backoff: FfiConverterOptionTypeBackoff.read(from: &buf) + ) + } + + public static func write(_ value: PushOptions, into buf: inout [UInt8]) { + FfiConverterOptionUInt64.write(value.pushIntervalMs, into: &buf) + FfiConverterOptionUInt64.write(value.maintenanceIntervalMs, into: &buf) + FfiConverterOptionTypeBackoff.write(value.backoff, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePushOptions_lift(_ buf: RustBuffer) throws -> PushOptions { + return try FfiConverterTypePushOptions.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePushOptions_lower(_ value: PushOptions) -> RustBuffer { + return FfiConverterTypePushOptions.lower(value) +} + + +public struct PushSummary: Equatable, Hashable { + /** + * Local replication position (TXID) after the push. + */ + public var txid: UInt64 + /** + * Whether new committed content was captured into an L0 file. + */ + public var synced: Bool + /** + * Number of files uploaded by this push. + */ + public var uploaded: UInt64 + /** + * Bucket-side max TXID after the push. + */ + public var remoteTxid: UInt64 + /** + * Whether a WAL checkpoint ran. + */ + public var checkpointed: Bool + /** + * Whether this push wrote a **full snapshot of the database** instead of + * an incremental WAL delta. + * + * The one field worth logging on every push. A snapshot ships the whole + * database, so a device that snapshots on most pushes is doing full + * uploads with extra steps — and on iOS that is a live possibility on + * every relaunch, because the `synced_to_wal_end` shortcut is + * deliberately not persisted across a writer close/reopen. + */ + public var snapshotted: Bool + /** + * Why, when `snapshotted`. One of a small fixed set from liters' verify + * decision tree, so it is safe to aggregate on. + */ + public var snapshotReason: String? + /** + * Bytes of LTX uploaded by this push, across `uploaded` files. + */ + public var bytesUploaded: UInt64 + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * Local replication position (TXID) after the push. + */txid: UInt64, + /** + * Whether new committed content was captured into an L0 file. + */synced: Bool, + /** + * Number of files uploaded by this push. + */uploaded: UInt64, + /** + * Bucket-side max TXID after the push. + */remoteTxid: UInt64, + /** + * Whether a WAL checkpoint ran. + */checkpointed: Bool, + /** + * Whether this push wrote a **full snapshot of the database** instead of + * an incremental WAL delta. + * + * The one field worth logging on every push. A snapshot ships the whole + * database, so a device that snapshots on most pushes is doing full + * uploads with extra steps — and on iOS that is a live possibility on + * every relaunch, because the `synced_to_wal_end` shortcut is + * deliberately not persisted across a writer close/reopen. + */snapshotted: Bool, + /** + * Why, when `snapshotted`. One of a small fixed set from liters' verify + * decision tree, so it is safe to aggregate on. + */snapshotReason: String?, + /** + * Bytes of LTX uploaded by this push, across `uploaded` files. + */bytesUploaded: UInt64) { + self.txid = txid + self.synced = synced + self.uploaded = uploaded + self.remoteTxid = remoteTxid + self.checkpointed = checkpointed + self.snapshotted = snapshotted + self.snapshotReason = snapshotReason + self.bytesUploaded = bytesUploaded + } + + + + +} + +#if compiler(>=6) +extension PushSummary: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypePushSummary: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> PushSummary { + return + try PushSummary( + txid: FfiConverterUInt64.read(from: &buf), + synced: FfiConverterBool.read(from: &buf), + uploaded: FfiConverterUInt64.read(from: &buf), + remoteTxid: FfiConverterUInt64.read(from: &buf), + checkpointed: FfiConverterBool.read(from: &buf), + snapshotted: FfiConverterBool.read(from: &buf), + snapshotReason: FfiConverterOptionString.read(from: &buf), + bytesUploaded: FfiConverterUInt64.read(from: &buf) + ) + } + + public static func write(_ value: PushSummary, into buf: inout [UInt8]) { + FfiConverterUInt64.write(value.txid, into: &buf) + FfiConverterBool.write(value.synced, into: &buf) + FfiConverterUInt64.write(value.uploaded, into: &buf) + FfiConverterUInt64.write(value.remoteTxid, into: &buf) + FfiConverterBool.write(value.checkpointed, into: &buf) + FfiConverterBool.write(value.snapshotted, into: &buf) + FfiConverterOptionString.write(value.snapshotReason, into: &buf) + FfiConverterUInt64.write(value.bytesUploaded, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePushSummary_lift(_ buf: RustBuffer) throws -> PushSummary { + return try FfiConverterTypePushSummary.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypePushSummary_lower(_ value: PushSummary) -> RustBuffer { + return FfiConverterTypePushSummary.lower(value) +} + + +public struct SyncSummary: Equatable, Hashable { + /** + * Whether a full restore ran (vs. incremental application). + */ + public var restored: Bool + public var fromTxid: UInt64 + public var toTxid: UInt64 + + // Default memberwise initializers are never public by default, so we + // declare one manually. + public init( + /** + * Whether a full restore ran (vs. incremental application). + */restored: Bool, fromTxid: UInt64, toTxid: UInt64) { + self.restored = restored + self.fromTxid = fromTxid + self.toTxid = toTxid + } + + + + +} + +#if compiler(>=6) +extension SyncSummary: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeSyncSummary: FfiConverterRustBuffer { + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SyncSummary { + return + try SyncSummary( + restored: FfiConverterBool.read(from: &buf), + fromTxid: FfiConverterUInt64.read(from: &buf), + toTxid: FfiConverterUInt64.read(from: &buf) + ) + } + + public static func write(_ value: SyncSummary, into buf: inout [UInt8]) { + FfiConverterBool.write(value.restored, into: &buf) + FfiConverterUInt64.write(value.fromTxid, into: &buf) + FfiConverterUInt64.write(value.toTxid, into: &buf) + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeSyncSummary_lift(_ buf: RustBuffer) throws -> SyncSummary { + return try FfiConverterTypeSyncSummary.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeSyncSummary_lower(_ value: SyncSummary) -> RustBuffer { + return FfiConverterTypeSyncSummary.lower(value) +} + + +/** + * One read from a response body. + */ + +public enum BodyChunk: Equatable, Hashable { + + /** + * Some decoded body bytes (must be non-empty). + */ + case data(bytes: Data + ) + /** + * No bytes right now, connection alive — an idle tick on a `long_lived` + * body. Only valid for `long_lived` requests. + */ + case idle + /** + * The body is complete. + */ + case eof + + + + + +} + +#if compiler(>=6) +extension BodyChunk: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeBodyChunk: FfiConverterRustBuffer { + typealias SwiftType = BodyChunk + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> BodyChunk { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .data(bytes: try FfiConverterData.read(from: &buf) + ) + + case 2: return .idle + + case 3: return .eof + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: BodyChunk, into buf: inout [UInt8]) { + switch value { + + + case let .data(bytes): + writeInt(&buf, Int32(1)) + FfiConverterData.write(bytes, into: &buf) + + + case .idle: + writeInt(&buf, Int32(2)) + + + case .eof: + writeInt(&buf, Int32(3)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBodyChunk_lift(_ buf: RustBuffer) throws -> BodyChunk { + return try FfiConverterTypeBodyChunk.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeBodyChunk_lower(_ value: BodyChunk) -> RustBuffer { + return FfiConverterTypeBodyChunk.lower(value) +} + + + +/** + * Whether a registration pushes or follows. + */ + +public enum DbRole: Equatable, Hashable { + + case push + case follow + + + + + +} + +#if compiler(>=6) +extension DbRole: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeDbRole: FfiConverterRustBuffer { + typealias SwiftType = DbRole + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DbRole { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .push + + case 2: return .follow + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: DbRole, into buf: inout [UInt8]) { + switch value { + + + case .push: + writeInt(&buf, Int32(1)) + + + case .follow: + writeInt(&buf, Int32(2)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbRole_lift(_ buf: RustBuffer) throws -> DbRole { + return try FfiConverterTypeDbRole.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbRole_lower(_ value: DbRole) -> RustBuffer { + return FfiConverterTypeDbRole.lower(value) +} + + + +/** + * Lifecycle state of one registered database. + */ + +public enum DbState: Equatable, Hashable { + + /** + * Registered and waiting for the next tick/nudge (push entries only; + * an active follower is `Working` for its whole session). + */ + case idle + /** + * A push/maintain round or follow session is in progress. + */ + case working + /** + * A transient failure is waiting out its backoff delay. `until_ms` is + * milliseconds since the Unix epoch; `attempt` is the + * consecutive-failure count (1 after the first failure). + */ + case backingOff(untilMs: UInt64, attempt: UInt32 + ) + /** + * Slept via `sleep()`: session cancelled, Writer dropped, no storage + * traffic until `resume()`. + */ + case sleeping + /** + * A fatal (non-transient) error parked the worker; it retries once per + * nudge (`push_now`/`sync_now`) or on `resume()`. + */ + case failed + + + + + +} + +#if compiler(>=6) +extension DbState: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeDbState: FfiConverterRustBuffer { + typealias SwiftType = DbState + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DbState { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .idle + + case 2: return .working + + case 3: return .backingOff(untilMs: try FfiConverterUInt64.read(from: &buf), attempt: try FfiConverterUInt32.read(from: &buf) + ) + + case 4: return .sleeping + + case 5: return .failed + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: DbState, into buf: inout [UInt8]) { + switch value { + + + case .idle: + writeInt(&buf, Int32(1)) + + + case .working: + writeInt(&buf, Int32(2)) + + + case let .backingOff(untilMs,attempt): + writeInt(&buf, Int32(3)) + FfiConverterUInt64.write(untilMs, into: &buf) + FfiConverterUInt32.write(attempt, into: &buf) + + + case .sleeping: + writeInt(&buf, Int32(4)) + + + case .failed: + writeInt(&buf, Int32(5)) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbState_lift(_ buf: RustBuffer) throws -> DbState { + return try FfiConverterTypeDbState.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeDbState_lower(_ value: DbState) -> RustBuffer { + return FfiConverterTypeDbState.lower(value) +} + + + +/** + * Errors the host transport can report. Thrown from a callback method or + * returned as the error arm; anything the host throws that is not one of + * these is mapped to `Transport` (see the `From` impl below) rather than + * crashing the worker. + */ +public +enum HttpError: Swift.Error, Equatable, Hashable, Foundation.LocalizedError { + + + + /** + * A network/transport failure (connect, reset, timeout). Retryable — + * maps to liters' `Unavailable`. + */ + case Transport(message: String + ) + /** + * A protocol/format failure (bad framing). Maps to liters' `Other`. + */ + case Protocol(message: String + ) + + + + + + + public var errorDescription: String? { + String(reflecting: self) + } + +} + +#if compiler(>=6) +extension HttpError: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeHttpError: FfiConverterRustBuffer { + typealias SwiftType = HttpError + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> HttpError { + let variant: Int32 = try readInt(&buf) + switch variant { + + + + + case 1: return .Transport( + message: try FfiConverterString.read(from: &buf) + ) + case 2: return .Protocol( + message: try FfiConverterString.read(from: &buf) + ) + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: HttpError, into buf: inout [UInt8]) { + switch value { + + + + + + case let .Transport(message): + writeInt(&buf, Int32(1)) + FfiConverterString.write(message, into: &buf) + + + case let .Protocol(message): + writeInt(&buf, Int32(2)) + FfiConverterString.write(message, into: &buf) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpError_lift(_ buf: RustBuffer) throws -> HttpError { + return try FfiConverterTypeHttpError.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeHttpError_lower(_ value: HttpError) -> RustBuffer { + return FfiConverterTypeHttpError.lower(value) +} + + +public +enum LitersError: Swift.Error, Equatable, Hashable, Foundation.LocalizedError { + + + + /** + * The bucket's history no longer matches the local state; call + * `Replica.reset()` (or construct with `auto_reset`) to re-restore. + */ + case Diverged(message: String + ) + /** + * Storage/network failure (including local I/O); usually transient — + * retry the same call later. + */ + case Storage(message: String + ) + /** + * The server rejected a write for consistency/ownership reasons: + * another writer holds the bucket lease, or the push was + * non-monotonic. Not retryable as-is — the next push after the local + * lineage check re-derives the correct baseline. + */ + case Conflict(message: String + ) + /** + * Authentication required or rejected; fix the auth token. + */ + case Unauthorized(message: String + ) + /** + * The operation was aborted by `cancel()` (or `close()`). State is + * crash-safe: retrying is indistinguishable from resuming after a + * process kill. + */ + case Cancelled + case Other(message: String + ) + + + + + + + public var errorDescription: String? { + String(reflecting: self) + } + +} + +#if compiler(>=6) +extension LitersError: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeLitersError: FfiConverterRustBuffer { + typealias SwiftType = LitersError + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> LitersError { + let variant: Int32 = try readInt(&buf) + switch variant { + + + + + case 1: return .Diverged( + message: try FfiConverterString.read(from: &buf) + ) + case 2: return .Storage( + message: try FfiConverterString.read(from: &buf) + ) + case 3: return .Conflict( + message: try FfiConverterString.read(from: &buf) + ) + case 4: return .Unauthorized( + message: try FfiConverterString.read(from: &buf) + ) + case 5: return .Cancelled + case 6: return .Other( + message: try FfiConverterString.read(from: &buf) + ) + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: LitersError, into buf: inout [UInt8]) { + switch value { + + + + + + case let .Diverged(message): + writeInt(&buf, Int32(1)) + FfiConverterString.write(message, into: &buf) + + + case let .Storage(message): + writeInt(&buf, Int32(2)) + FfiConverterString.write(message, into: &buf) + + + case let .Conflict(message): + writeInt(&buf, Int32(3)) + FfiConverterString.write(message, into: &buf) + + + case let .Unauthorized(message): + writeInt(&buf, Int32(4)) + FfiConverterString.write(message, into: &buf) + + + case .Cancelled: + writeInt(&buf, Int32(5)) + + + case let .Other(message): + writeInt(&buf, Int32(6)) + FfiConverterString.write(message, into: &buf) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersError_lift(_ buf: RustBuffer) throws -> LitersError { + return try FfiConverterTypeLitersError.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeLitersError_lower(_ value: LitersError) -> RustBuffer { + return FfiConverterTypeLitersError.lower(value) +} + + +/** + * Events delivered to a `ManagerListener`. One database's events arrive in + * order; followers do not emit per-transaction position events (poll + * `status()` for live positions) — `SyncCompleted` is reserved for future + * use. + */ + +public enum ManagerEvent: Equatable, Hashable { + + case stateChanged(id: String, state: DbState + ) + case pushCompleted(id: String, result: PushSummary + ) + case syncCompleted(id: String, result: SyncSummary + ) + case error(id: String, message: String, transient: Bool + ) + + + + + +} + +#if compiler(>=6) +extension ManagerEvent: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeManagerEvent: FfiConverterRustBuffer { + typealias SwiftType = ManagerEvent + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ManagerEvent { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .stateChanged(id: try FfiConverterString.read(from: &buf), state: try FfiConverterTypeDbState.read(from: &buf) + ) + + case 2: return .pushCompleted(id: try FfiConverterString.read(from: &buf), result: try FfiConverterTypePushSummary.read(from: &buf) + ) + + case 3: return .syncCompleted(id: try FfiConverterString.read(from: &buf), result: try FfiConverterTypeSyncSummary.read(from: &buf) + ) + + case 4: return .error(id: try FfiConverterString.read(from: &buf), message: try FfiConverterString.read(from: &buf), transient: try FfiConverterBool.read(from: &buf) + ) + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: ManagerEvent, into buf: inout [UInt8]) { + switch value { + + + case let .stateChanged(id,state): + writeInt(&buf, Int32(1)) + FfiConverterString.write(id, into: &buf) + FfiConverterTypeDbState.write(state, into: &buf) + + + case let .pushCompleted(id,result): + writeInt(&buf, Int32(2)) + FfiConverterString.write(id, into: &buf) + FfiConverterTypePushSummary.write(result, into: &buf) + + + case let .syncCompleted(id,result): + writeInt(&buf, Int32(3)) + FfiConverterString.write(id, into: &buf) + FfiConverterTypeSyncSummary.write(result, into: &buf) + + + case let .error(id,message,transient): + writeInt(&buf, Int32(4)) + FfiConverterString.write(id, into: &buf) + FfiConverterString.write(message, into: &buf) + FfiConverterBool.write(transient, into: &buf) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerEvent_lift(_ buf: RustBuffer) throws -> ManagerEvent { + return try FfiConverterTypeManagerEvent.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeManagerEvent_lower(_ value: ManagerEvent) -> RustBuffer { + return FfiConverterTypeManagerEvent.lower(value) +} + + + +/** + * Where a database's replica lives. + */ + +public enum Storage: Equatable, Hashable { + + /** + * Local directory in litestream's `file` layout (testing, shared + * containers). + */ + case dir(path: String + ) + /** + * Another liters instance serving a bucket over HTTP + * (`http://host:port[/path]`; one database of a multi-DB server is + * addressed as `http://host:port/db/{name}`). Valid as a Replica + * source — `sync()` to poll, or follow live via + * `LitersManager.register_follow`. Valid as a Writer destination when + * the server runs with `writable: true` — that is push replication: + * this device dials out and pushes, the listening liters receives. + * Read-only servers reject the first push with a clear error. + * + * `auth_token` is sent as `Authorization: Bearer ` on every + * request, for servers started with an auth token. (Adding this field + * is a source-level break for Swift/Kotlin callers constructing + * `Storage.Http` positionally; pass `null`/`nil` to keep the old + * behavior.) + */ + case http(url: String, authToken: String? + ) + + + + + +} + +#if compiler(>=6) +extension Storage: Sendable {} +#endif + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public struct FfiConverterTypeStorage: FfiConverterRustBuffer { + typealias SwiftType = Storage + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Storage { + let variant: Int32 = try readInt(&buf) + switch variant { + + case 1: return .dir(path: try FfiConverterString.read(from: &buf) + ) + + case 2: return .http(url: try FfiConverterString.read(from: &buf), authToken: try FfiConverterOptionString.read(from: &buf) + ) + + default: throw UniffiInternalError.unexpectedEnumCase + } + } + + public static func write(_ value: Storage, into buf: inout [UInt8]) { + switch value { + + + case let .dir(path): + writeInt(&buf, Int32(1)) + FfiConverterString.write(path, into: &buf) + + + case let .http(url,authToken): + writeInt(&buf, Int32(2)) + FfiConverterString.write(url, into: &buf) + FfiConverterOptionString.write(authToken, into: &buf) + + } + } +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeStorage_lift(_ buf: RustBuffer) throws -> Storage { + return try FfiConverterTypeStorage.lift(buf) +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +public func FfiConverterTypeStorage_lower(_ value: Storage) -> RustBuffer { + return FfiConverterTypeStorage.lower(value) +} + + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionUInt64: FfiConverterRustBuffer { + typealias SwiftType = UInt64? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterUInt64.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterUInt64.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionString: FfiConverterRustBuffer { + typealias SwiftType = String? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterString.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterString.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeHttpClient: FfiConverterRustBuffer { + typealias SwiftType = HttpClient? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeHttpClient.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeHttpClient.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeHttpRequestBody: FfiConverterRustBuffer { + typealias SwiftType = HttpRequestBody? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeHttpRequestBody.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeHttpRequestBody.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeManagerListener: FfiConverterRustBuffer { + typealias SwiftType = ManagerListener? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeManagerListener.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeManagerListener.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeBackoff: FfiConverterRustBuffer { + typealias SwiftType = Backoff? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeBackoff.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeBackoff.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterOptionTypeDbStatus: FfiConverterRustBuffer { + typealias SwiftType = DbStatus? + + public static func write(_ value: SwiftType, into buf: inout [UInt8]) { + guard let value = value else { + writeInt(&buf, Int8(0)) + return + } + writeInt(&buf, Int8(1)) + FfiConverterTypeDbStatus.write(value, into: &buf) + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { + switch try readInt(&buf) as Int8 { + case 0: return nil + case 1: return try FfiConverterTypeDbStatus.read(from: &buf) + default: throw UniffiInternalError.unexpectedOptionalTag + } + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeDbStatus: FfiConverterRustBuffer { + typealias SwiftType = [DbStatus] + + public static func write(_ value: [DbStatus], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeDbStatus.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [DbStatus] { + let len: Int32 = try readInt(&buf) + var seq = [DbStatus]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeDbStatus.read(from: &buf)) + } + return seq + } +} + +#if swift(>=5.8) +@_documentation(visibility: private) +#endif +fileprivate struct FfiConverterSequenceTypeHttpHeader: FfiConverterRustBuffer { + typealias SwiftType = [HttpHeader] + + public static func write(_ value: [HttpHeader], into buf: inout [UInt8]) { + let len = Int32(value.count) + writeInt(&buf, len) + for item in value { + FfiConverterTypeHttpHeader.write(item, into: &buf) + } + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [HttpHeader] { + let len: Int32 = try readInt(&buf) + var seq = [HttpHeader]() + seq.reserveCapacity(Int(len)) + for _ in 0 ..< len { + seq.append(try FfiConverterTypeHttpHeader.read(from: &buf)) + } + return seq + } +} + +private enum InitializationResult { + case ok + case contractVersionMismatch + case apiChecksumMismatch +} +// Use a global variable to perform the versioning checks. Swift ensures that +// the code inside is only computed once. +private let initializationResult: InitializationResult = { + // Get the bindings contract version from our ComponentInterface + let bindings_contract_version = 30 + // Get the scaffolding contract version by calling the into the dylib + let scaffolding_contract_version = ffi_liters_ffi_uniffi_contract_version() + if bindings_contract_version != scaffolding_contract_version { + return InitializationResult.contractVersionMismatch + } + if (uniffi_liters_ffi_checksum_method_httpclient_execute() != 20111) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_httprequestbody_read() != 60181) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_httpresponse_status() != 55118) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_httpresponse_headers() != 31518) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_httpresponse_read_body() != 54488) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_httpresponse_cancel() != 48762) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_push_now() != 60078) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_register_follow() != 44870) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_register_push() != 15559) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_resume() != 64190) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_resume_all() != 38642) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_set_listener() != 45114) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_shutdown() != 37060) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_sleep() != 32012) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_sleep_all() != 64691) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_status() != 22869) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_statuses() != 53853) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_sync_now() != 2233) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersmanager_unregister() != 3105) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_cancel() != 41779) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_close() != 45861) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_db_path() != 55063) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_position() != 41654) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_reset() != 44487) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_litersreplica_sync() != 37122) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_cancel() != 9936) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_close() != 22696) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_maintain() != 48767) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_position() != 6071) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_push() != 60273) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_reset_local() != 52534) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_literswriter_snapshot() != 32323) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_method_managerlistener_on_event() != 29970) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_litersmanager_new() != 6165) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_litersmanager_with_options() != 34269) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_litersreplica_new() != 23861) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_litersreplica_new_with_http_client() != 6821) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_literswriter_new() != 51245) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_liters_ffi_checksum_constructor_literswriter_new_with_http_client() != 15490) { + return InitializationResult.apiChecksumMismatch + } + + uniffiCallbackInitHttpClient() + uniffiCallbackInitHttpResponse() + uniffiCallbackInitManagerListener() + return InitializationResult.ok +}() + +// Make the ensure init function public so that other modules which have external type references to +// our types can call it. +public func uniffiEnsureLitersFfiInitialized() { + switch initializationResult { + case .ok: + break + case .contractVersionMismatch: + fatalError("UniFFI contract version mismatch: try cleaning and rebuilding your project") + case .apiChecksumMismatch: + fatalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project") + } +} + +// swiftlint:enable all +#endif diff --git a/Strand/CloudSync/LitersHTTPClient.swift b/Strand/CloudSync/LitersHTTPClient.swift new file mode 100644 index 0000000000..3143673ffb --- /dev/null +++ b/Strand/CloudSync/LitersHTTPClient.swift @@ -0,0 +1,205 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig) AND LITERS is set (by the generated +// Config/LitersLocal.xcconfig, i.e. only after Rust/build-ios.sh has produced the xcframework). +// A default build contains none of this code. +#if CLOUD_SYNC && LITERS +import Foundation + +/// The host HTTP transport liters runs every `Storage.http` request through. +/// +/// ## Why this file has to exist at all +/// +/// `LitersWriter(dbPath:storage:)` without a host client uses liters' built-in socket transport, +/// which is **`http://` only** — `HttpReplicaClient::with_options` *rejects* an `https://` URL +/// rather than silently downgrading it. `noop-cloud` is HTTPS. So on this app the choice is not +/// "nicer transport vs. built-in"; without this class the liters path cannot reach the server at +/// all. +/// +/// ## The shape liters asks for, and why it is blocking +/// +/// `execute` is synchronous by contract: liters calls it from its own worker while holding the +/// writer lock, and expects a response object it then *pulls* bytes from. That is deliberate — it +/// is what lets a 300 MB snapshot push stream instead of buffering. The bridge here is therefore a +/// `DispatchSemaphore` around a `URLSession` task, and it is a hard error to call it on the main +/// thread: `CloudSyncUploader` only ever reaches it from a `Task`'s background executor, and the +/// `precondition` makes a future misuse fail loudly rather than deadlock the UI. +/// +/// ## What this is NOT +/// +/// Not a background `URLSession`. A background session delivers completion through the app +/// delegate, minutes later, to a process that may have been relaunched — there is no way to return +/// that from a synchronous `execute`, and pretending otherwise would be worse than not having it. +/// The consequence is stated plainly: an upload in flight when iOS suspends the app is lost, and +/// the next push resumes from the last committed TXID. **That is exactly today's `/ingest` +/// behaviour** (`CloudSyncClient` also uses a foreground session), so this path is not a regression +/// — but it is the reason `liters`' own docs push you toward a background session, and it is the +/// first thing to revisit if pushes start dying at suspend. +final class LitersURLSessionClient: HttpClient, @unchecked Sendable { + /// One instance per process, one `URLSession` inside it, so every request to the server + /// coalesces onto one connection — which is what liters' `HttpClient` doc asks for. + static let shared = LitersURLSessionClient() + + /// Generous, because a snapshot push is the whole database. `timeoutIntervalForRequest` is an + /// *inactivity* timeout, not a total-transfer one, so this does not cap a slow large upload; + /// `timeoutIntervalForResource` does, and 1 hour is the ceiling on one push attempt. + private let session: URLSession + + init(session: URLSession? = nil) { + if let session { + self.session = session + } else { + let cfg = URLSessionConfiguration.default + cfg.timeoutIntervalForRequest = 120 + cfg.timeoutIntervalForResource = 3600 + cfg.httpMaximumConnectionsPerHost = 2 + cfg.waitsForConnectivity = true + self.session = URLSession(configuration: cfg) + } + } + + func execute(request: HttpRequest, body: HttpRequestBody?) throws -> HttpResponse { + precondition(!Thread.isMainThread, + "LitersURLSessionClient.execute blocks; it must never run on the main thread") + + guard let url = URL(string: request.url) else { + // `HttpError.Protocol(...)` will not parse — Swift reads `Type.Protocol` as the + // metatype syntax regardless of backticks — so the case is reached through a + // leading-dot member reference on an explicitly typed value. + let e: HttpError = .Protocol(message: "liters produced an unparseable URL: \(request.url)") + throw e + } + var req = URLRequest(url: url) + req.httpMethod = request.method + for h in request.headers { req.setValue(h.value, forHTTPHeaderField: h.name) } + + // A PUT body is pulled from liters, never handed over whole: `HttpRequestBody.read(max:)` + // is the source and a 300 MB snapshot must not be materialised as `Data`. Bound streams + // give `URLSession` an `InputStream` while a producer thread fills the other end. + if let body { + req.httpBodyStream = Self.stream(pulling: body) + } + + let sem = DispatchSemaphore(value: 0) + var out: (data: Data, response: HTTPURLResponse)? + var failure: Error? + + let task = session.dataTask(with: req) { data, response, error in + if let error { + failure = error + } else if let http = response as? HTTPURLResponse { + out = (data ?? Data(), http) + } else { + let e: HttpError = .Protocol(message: "no HTTP response") + failure = e + } + sem.signal() + } + task.resume() + sem.wait() + + if let failure { + // Transport, not Protocol: liters treats Transport as retryable, and a dropped + // connection or a suspended radio is exactly that. A push is idempotent by key, so a + // retry of a half-sent PUT is safe. + throw HttpError.Transport(message: (failure as NSError).localizedDescription) + } + guard let out else { throw HttpError.Transport(message: "no response and no error") } + + var headers: [HttpHeader] = [] + for (k, v) in out.response.allHeaderFields { + guard let name = k as? String, let value = v as? String else { continue } + // `content-length` and `content-encoding` are rewritten below, not forwarded. URLSession + // owns the transfer: it transparently negotiates and undoes `Content-Encoding`, and it + // does not reliably surface `Content-Length` at all — an HTTP/2 response, or one it + // decompressed, arrives with the header missing or describing the *compressed* size. + // Either way the number would not describe `out.data`. + // + // This is not cosmetic: liters' client requires a length on a listing response and + // fails the whole push with "storage: list level 0: missing content-length" without + // one. Found by running it — the first real push through this class died here. + let lower = name.lowercased() + if lower == "content-length" || lower == "content-encoding" { continue } + headers.append(HttpHeader(name: name, value: value)) + } + // The authoritative length: what the caller will actually be able to read. + headers.append(HttpHeader(name: "content-length", value: String(out.data.count))) + return BufferedHTTPResponse(status: UInt16(out.response.statusCode), + headerList: headers, + body: out.data) + } + + /// Bridges liters' pull-based body into the push-based `InputStream` `URLSession` wants. + /// + /// `Stream.getBoundStreams` gives a connected pair with a fixed-size kernel-ish buffer; the + /// producer thread blocks on `write` when the consumer is behind, which is the backpressure + /// that keeps memory flat regardless of push size. + private static func stream(pulling body: HttpRequestBody) -> InputStream { + var input: InputStream? + var output: OutputStream? + Stream.getBoundStreams(withBufferSize: 65_536, inputStream: &input, outputStream: &output) + guard let input, let output else { + // Cannot happen; a zero-byte body is still a legal body and lets the PUT fail on the + // server's terms rather than crashing the app here. + return InputStream(data: Data()) + } + let pump = Thread { + output.open() + defer { output.close() } + while true { + let chunk: Data + do { + chunk = try body.read(max: 65_536) + } catch { + break // aborts the upload; URLSession sees a truncated body and errors + } + if chunk.isEmpty { break } // documented end-of-body + var sent = 0 + chunk.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return } + while sent < chunk.count { + let n = output.write(base + sent, maxLength: chunk.count - sent) + if n <= 0 { sent = chunk.count; return } // consumer went away + sent += n + } + } + } + } + pump.name = "liters-body-pump" + pump.stackSize = 512 * 1024 + pump.start() + return input + } +} + +/// liters' responses on the push path are small (a status line or a short JSON error), so the whole +/// body is collected and then served through the pull interface. The long-lived `/stream` follow — +/// the one case where buffering would be wrong — is a *Replica* concern; this app only pushes. +private final class BufferedHTTPResponse: HttpResponse, @unchecked Sendable { + private let statusCode: UInt16 + private let headerList: [HttpHeader] + private var body: Data + private let lock = NSLock() + + init(status: UInt16, headerList: [HttpHeader], body: Data) { + self.statusCode = status + self.headerList = headerList + self.body = body + } + + func status() -> UInt16 { statusCode } + func headers() -> [HttpHeader] { headerList } + + func readBody(max: UInt32) throws -> BodyChunk { + lock.lock() + defer { lock.unlock() } + if body.isEmpty { return .eof } + let n = Swift.min(Int(max), body.count) + let chunk = body.prefix(n) + body = body.dropFirst(n) + return .data(bytes: Data(chunk)) + } + + /// Nothing to abort: the transfer already completed before this object existed. + func cancel() {} +} +#endif // CLOUD_SYNC && LITERS diff --git a/Strand/CloudSync/LitersReplicator.swift b/Strand/CloudSync/LitersReplicator.swift new file mode 100644 index 0000000000..ecf9148579 --- /dev/null +++ b/Strand/CloudSync/LitersReplicator.swift @@ -0,0 +1,116 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig) AND LITERS is set (by the generated Config/LitersLocal.xcconfig, i.e. only +// after Rust/build-ios.sh has produced the xcframework). A default build contains none of this code. +#if CLOUD_SYNC && LITERS +import Foundation + +/// Owns the one `LitersWriter` for the app's whole lifetime and performs one push per sync. +/// +/// ## Why this is a process-lifetime singleton and not a per-push object +/// +/// This is the single most load-bearing decision in the liters path, and it was measured rather +/// than assumed. `liters::Writer` resumes an incremental push from a WAL offset it recorded on the +/// previous push. Dropping the writer releases the WAL-pinning read lock, and SQLite checkpoints +/// and truncates the WAL when the *last* connection to a database closes — `wal_autocheckpoint = 0` +/// does not prevent that, it only stops the threshold-triggered one. The next open therefore finds +/// a WAL whose header salt has been reset and can only recover by shipping the whole database. +/// +/// Measured on VK's real 782 MB database against a real `noop-cloud` `/liters` endpoint +/// (2026-07-27): a writer opened and dropped around each push snapshotted **every single time** — +/// 5 pushes, 5 snapshots, ~312 MB each. The same database with one writer held across pushes +/// snapshotted once (the first, which has to) and then ran incremental: 151–176 KB per 10-minute +/// delta, ~2.0 MB per 4-hour delta. That is the difference between page replication and a full +/// upload with extra steps, and it is entirely a function of writer lifetime. +/// +/// So: opened lazily on the first push, kept, and never closed except by `reset()` — which exists +/// for the recovery path, not for routine use. +/// +/// ## Threading +/// +/// `LitersWriter`'s methods are synchronous and block. `push()` is called from a `Task`'s +/// background executor by `CloudSyncUploader`, and serialized here by an `NSLock` because the FFI +/// object's own lock would otherwise queue a second push behind an in-flight one with no timeout of +/// ours. One sync at a time is already guaranteed upstream by `CloudSyncGate`; the lock is a +/// belt-and-braces against a future second caller. +final class LitersReplicator: @unchecked Sendable { + static let shared = LitersReplicator() + + private let lock = NSLock() + private var writer: LitersWriter? + /// The endpoint the live writer was opened against. A settings change (new server or rotated + /// token) has to rebuild the writer, or pushes would keep going to the old destination. + private var openedFor: String? + + private init() {} + + /// `/liters` — the mount `noop-cloud`'s Express proxy serves. Everything after it is + /// liters' own grammar (`/ltx/0/{min}-{max}.ltx`) and is not this app's business. + static func endpoint(base: URL) -> String { + base.appendingPathComponent("liters").absoluteString + } + + /// One push. Returns liters' own summary so the caller records the real numbers rather than a + /// guess. Throws `LitersError` — the caller is expected to fall back to `/ingest`. + func push(databasePath: String, endpoint: String, token: String) throws -> PushSummary { + lock.lock() + defer { lock.unlock() } + + if openedFor != endpoint { closeLocked() } + if writer == nil { + writer = try LitersWriter.newWithHttpClient( + dbPath: databasePath, + storage: .http(url: endpoint, authToken: token), + httpClient: LitersURLSessionClient.shared) + openedFor = endpoint + } + guard let writer else { throw LitersError.Other(message: "writer unavailable") } + return try writer.push() + } + + /// **`maintain()` is deliberately never called from the push path**, and that is a decision, + /// not an omission. + /// + /// liters' `maintain()` runs compaction, retention *and* snapshotting on litestream's default + /// cadence — `snapshot_interval` is 24 h — and `LitersWriter.maintain()` hardcodes + /// `MaintenanceOptions::default()`, so the interval is not reachable from Swift. Measured on + /// the real 782 MB database (2026-07-27): one `maintain()` run took **177 s** and uploaded a + /// fresh **312 MB** snapshot. A `BGAppRefreshTask` gets tens of seconds, so this cannot run + /// there at all; and calling it daily would put a 312 MB upload back into a scheme whose entire + /// point is not doing 165 MB uploads. + /// + /// The cost of not calling it is bucket growth, and it is small and bounded on this + /// deployment: the bucket is a transport, not an archive — the server keeps a live + /// `mirror.sqlite` and only ever applies files forward. One base snapshot (~312 MB) plus + /// ~2 MB per 4-hour sync is ~12 MB/day, i.e. ~4.7 GB after a year on a 10.5 GB volume, and it + /// is swept server-side rather than paid for on the phone's radio. Revisit only if a *fresh* + /// restore (as opposed to incremental follow) ever becomes a normal operation. + + /// Whether a writer is currently open. Diagnostics only — a `false` here after a successful + /// sync means something closed it, which is the state that costs a snapshot. + var isOpen: Bool { + lock.lock(); defer { lock.unlock() } + return writer != nil + } + + /// Recovery for liters' one non-transient local error ("local ltx file missing or corrupt"): + /// wipes local replication state so the next push re-derives everything from the bucket. The + /// database content is untouched. Costs one snapshot. + func resetLocalState() { + lock.lock(); defer { lock.unlock() } + try? writer?.resetLocal() + } + + /// Drops the writer, releasing the WAL read lock and every fd. **Costs a snapshot on the next + /// push** (see the type doc), so this is for teardown and endpoint changes, not for tidiness. + func close() { + lock.lock(); defer { lock.unlock() } + closeLocked() + } + + private func closeLocked() { + writer?.close() + writer = nil + openedFor = nil + } +} +#endif // CLOUD_SYNC && LITERS diff --git a/Strand/CloudSync/SyncPushTelemetry.swift b/Strand/CloudSync/SyncPushTelemetry.swift new file mode 100644 index 0000000000..61f7f61d20 --- /dev/null +++ b/Strand/CloudSync/SyncPushTelemetry.swift @@ -0,0 +1,216 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation + +/// One observed page-replication push, recorded locally so the device trial can answer the single +/// question that gates adopting `liters`: **how often does a push degrade to a full snapshot?** +/// +/// A snapshotting push ships the entire database. If that is rare, incremental page replication is +/// a 25–380× win over today's whole-file upload. If it is routine, `liters` is a full upload with +/// extra steps and the trial should stop there. Nothing about `PushSummary`'s other fields answers +/// this — a snapshot and a delta differ only in size, and size alone cannot distinguish "a big +/// day" from "re-uploaded everything". +/// +/// Fields mirror `liters`' FFI `PushSummary` one-for-one plus two the app supplies, so wiring the +/// real thing later is a struct literal and nothing else. Deliberately a plain value type with no +/// `liters` import: this compiles and is tested with no Rust in the project. +struct SyncPushObservation: Codable, Equatable, Sendable { + /// When the push completed. + var at: Date + /// True when the push wrote a full snapshot of the database rather than a WAL delta. + var snapshotted: Bool + /// `liters`' reason from its `verify()` decision tree, e.g. "wal truncated by another process". + /// One of a small fixed set, so it is safe to group on. `nil` when not snapshotting. + var snapshotReason: String? + /// Bytes of LTX uploaded. + var bytesUploaded: Int64 + /// Size of the `-wal` file when the push ran — the other half of the story, since a large WAL + /// is what precedes a foreign checkpoint and therefore a snapshot. + var walBytes: Int64 + /// Seconds since the previous recorded push; `nil` for the first one after install. + var secondsSinceLastPush: Double? + /// `liters`' replication position after the push, for correlating with the server. + var txid: UInt64 +} + +/// Aggregate view of the trial so far. `snapshotRate` is the go/no-go number. +struct SyncPushStats: Equatable { + var pushes: Int + var snapshots: Int + var bytesUploaded: Int64 + var bytesUploadedBySnapshots: Int64 + /// Count per `snapshotReason`, so a bad rate can be attributed to a branch rather than guessed at. + var reasonCounts: [String: Int] + var maxWalBytes: Int64 + var medianSecondsBetweenPushes: Double? + + /// Fraction of pushes that shipped the whole database. This is what decides the trial. + var snapshotRate: Double { pushes == 0 ? 0 : Double(snapshots) / Double(pushes) } + /// Fraction of uploaded bytes spent on snapshots — the cost, as opposed to the frequency. A low + /// rate can still be a bad deal if each snapshot is 766 MB. + var snapshotByteShare: Double { + bytesUploaded == 0 ? 0 : Double(bytesUploadedBySnapshots) / Double(bytesUploaded) + } +} + +/// Append-only, bounded, local recorder for `SyncPushObservation`s. +/// +/// ## Why not a table in the database +/// +/// The obvious place is a `syncPushLog` table in `whoop.sqlite`. That would be wrong twice over. +/// Every insert is another commit in the database whose *commit rate* is the thing being measured +/// (~14 dirtied pages each, which is exactly what drives WAL growth), and every row would then be +/// replicated by the replicator it is measuring — the instrument would be part of the experiment. +/// A small JSON file in Application Support is outside the replication set entirely. +/// +/// ## Cost +/// +/// One `stat(2)` for the WAL size plus one atomic rewrite of a file that is capped at +/// `maxRecords` (200 ≈ 30 KB). At the expected push cadence — single-digit per day, ~2/hour even +/// on an aggressive schedule — that is immaterial, and it is bounded no matter how long the trial +/// runs. Trimming drops old *records* but never the counters, so `stats` stays correct over the +/// whole trial while the record window stays small. +/// +/// Not an actor: pushes are already serialized by the replicator, and an `NSLock` keeps this usable +/// from a `BGTask` completion handler without an `await`. +final class SyncPushTelemetry: @unchecked Sendable { + /// Records kept on disk. Older ones are dropped; the aggregate counters are not. + static let maxRecords = 200 + + private let url: URL + private let lock = NSLock() + private var state: State + + private struct State: Codable { + var version: Int = 1 + var records: [SyncPushObservation] = [] + /// Monotonic totals, kept across trimming so `stats` covers the whole trial. + var totalPushes: Int = 0 + var totalSnapshots: Int = 0 + var totalBytes: Int64 = 0 + var totalSnapshotBytes: Int64 = 0 + var reasonCounts: [String: Int] = [:] + var maxWalBytes: Int64 = 0 + } + + /// The default location: `Application Support/CloudSync/push-telemetry.json`. Not in Caches — + /// the OS may evict Caches under pressure, and losing the trial's data to a cache purge would + /// be a silent, unrecoverable hole in the one measurement the trial exists for. + static func defaultURL() throws -> URL { + let dir = try FileManager.default + .url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + .appendingPathComponent("CloudSync", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + return dir.appendingPathComponent("push-telemetry.json") + } + + /// Loads existing state, or starts empty. A corrupt or unreadable file starts empty rather than + /// throwing: losing telemetry must never break a sync. + init(url: URL) { + self.url = url + if let data = try? Data(contentsOf: url), + let decoded = try? JSONDecoder().decode(State.self, from: data) { + self.state = decoded + } else { + self.state = State() + } + } + + /// Records one push. `walBytes` comes from `WhoopStore.walFileSizeBytes()`; + /// `secondsSinceLastPush` is derived here so callers cannot forget it. + /// + /// Best-effort persistence: a failed write loses the record, never the sync. + @discardableResult + func record(at now: Date = Date(), + snapshotted: Bool, + snapshotReason: String?, + bytesUploaded: Int64, + walBytes: Int64, + txid: UInt64) -> SyncPushObservation { + lock.lock() + defer { lock.unlock() } + + let previous = state.records.last?.at + let observation = SyncPushObservation( + at: now, + snapshotted: snapshotted, + snapshotReason: snapshotted ? snapshotReason : nil, + bytesUploaded: bytesUploaded, + walBytes: walBytes, + secondsSinceLastPush: previous.map { now.timeIntervalSince($0) }, + txid: txid) + + state.records.append(observation) + if state.records.count > Self.maxRecords { + state.records.removeFirst(state.records.count - Self.maxRecords) + } + state.totalPushes += 1 + state.totalBytes += bytesUploaded + if snapshotted { + state.totalSnapshots += 1 + state.totalSnapshotBytes += bytesUploaded + state.reasonCounts[snapshotReason ?? "unspecified", default: 0] += 1 + } + state.maxWalBytes = max(state.maxWalBytes, walBytes) + persist() + return observation + } + + /// The whole trial so far, including pushes whose records have since been trimmed. + var stats: SyncPushStats { + lock.lock() + defer { lock.unlock() } + let gaps = state.records.compactMap(\.secondsSinceLastPush).sorted() + return SyncPushStats( + pushes: state.totalPushes, + snapshots: state.totalSnapshots, + bytesUploaded: state.totalBytes, + bytesUploadedBySnapshots: state.totalSnapshotBytes, + reasonCounts: state.reasonCounts, + maxWalBytes: state.maxWalBytes, + medianSecondsBetweenPushes: gaps.isEmpty ? nil : gaps[gaps.count / 2]) + } + + /// The retained window, oldest first. For a diagnostics screen or a manual export. + var recentRecords: [SyncPushObservation] { + lock.lock() + defer { lock.unlock() } + return state.records + } + + /// A one-line summary for the log, so the trial is readable without a UI. + var oneLineSummary: String { + let s = stats + let pct = String(format: "%.1f%%", s.snapshotRate * 100) + let bytePct = String(format: "%.1f%%", s.snapshotByteShare * 100) + let reasons = s.reasonCounts.sorted { $0.value > $1.value } + .map { "\($0.key)=\($0.value)" }.joined(separator: ", ") + return "pushes=\(s.pushes) snapshots=\(s.snapshots) (\(pct) of pushes, \(bytePct) of bytes) " + + "maxWal=\(s.maxWalBytes)B" + + (reasons.isEmpty ? "" : " reasons[\(reasons)]") + } + + /// Discards everything. For starting a clean trial run. + func reset() { + lock.lock() + defer { lock.unlock() } + state = State() + persist() + } + + /// Atomic: write a sibling temp file and rename, so a kill mid-write cannot leave a truncated + /// JSON that the next launch silently discards as corrupt. + private func persist() { + guard let data = try? JSONEncoder().encode(state) else { return } + let tmp = url.appendingPathExtension("tmp") + do { + try data.write(to: tmp, options: .atomic) + _ = try? FileManager.default.replaceItemAt(url, withItemAt: tmp) + } catch { + try? FileManager.default.removeItem(at: tmp) + } + } +} +#endif diff --git a/Strand/CloudSync/SyncReplicationTrial.swift b/Strand/CloudSync/SyncReplicationTrial.swift new file mode 100644 index 0000000000..d1f460cdea --- /dev/null +++ b/Strand/CloudSync/SyncReplicationTrial.swift @@ -0,0 +1,116 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (by the untracked +// OuraSecrets.xcconfig — see OuraConfig.xcconfig). A default build contains none of this code, +// keeping "fully offline" a byte-level property of the shipped binary, not a runtime promise. +#if CLOUD_SYNC +import Foundation +import WhoopStore + +/// The on/off switch and launch hook for the page-replication device trial. +/// +/// ## What the trial is for +/// +/// `noop-cloud`'s current sync uploads the whole database (153.4 MB, 90–150 s) on every sync. The +/// replacement ships WAL frames instead. The one number that decides whether that is a 25–380× win +/// or a full upload with extra steps is **how often a push degrades to a full snapshot**, and it +/// cannot be reasoned about — iOS relaunches the process constantly, and a replicator that loses its +/// resume offset re-uploads everything. It has to be measured on VK's device, on VK's data. +/// +/// Enabling the trial does two things and nothing else: +/// +/// 1. hands WAL checkpointing to an external replicator (`StoreReplication.configure(.external)`), +/// which is the precondition for a replicator to see every transaction; and +/// 2. records one `SyncPushObservation` per push (`SyncPushTelemetry`). +/// +/// It uploads nothing differently. Today's `/ingest` path is untouched, so the trial's downside is +/// bounded to "the WAL is now checkpointed by the backstop at 64 MiB instead of by SQLite at 4 MB." +/// +/// ## Default: off +/// +/// `.external` is a real commitment — with autocheckpoint disabled, WAL growth is bounded only by +/// `WalBackstopPolicy` (see `WalCheckpointing`'s obligation note, and the measured 166 MB/day). It +/// stays off until VK turns it on, on his own device, and it is a plain `UserDefaults` flag rather +/// than a Keychain item because it is a preference, not a secret. +enum SyncReplicationTrial { + /// `UserDefaults` key. Absent (the shipped state) reads as `false`. + static let enabledKey = "cloudsync.replicationTrialEnabled" + + /// Whether the trial is on. Reading this is *not* the same as asking whether `.external` is in + /// force — a flip only takes effect at the next launch, because `applyAtLaunch` runs before any + /// store opens and a live `DatabasePool`'s `wal_autocheckpoint` cannot be changed after the fact. + /// Ask `StoreReplication.walCheckpointing` for the state that is actually in force. + static var isEnabled: Bool { + get { UserDefaults.standard.bool(forKey: enabledKey) } + set { UserDefaults.standard.set(newValue, forKey: enabledKey) } + } + + /// Whether the trial is actually *running in this process*, as opposed to merely switched on. + /// + /// The two differ for exactly one launch after every flip, in both directions, and the difference is + /// the thing a user is most likely to get wrong: `applyAtLaunch` runs once in `init()` and + /// `wal_autocheckpoint` is a per-connection PRAGMA applied at pool-open time, so flipping the toggle + /// cannot retroactively change a pool that is already open. The Test Centre card shows this next to + /// the toggle so "on" and "measuring" are never conflated. + static var isInForce: Bool { + if case .external = StoreReplication.walCheckpointing { return true } + return false + } + + /// Install the process-wide store policy. **Must be called before the first `WhoopStore` open**, + /// i.e. at the top of the app's `init()`, ahead of `AppModel()`. + /// + /// Both real openers (`Repository.ensureStore()`, `BLEManager.bootstrapStore()`) are `async` and + /// therefore strictly later than `init()`, including the CoreBluetooth state-restoration path — + /// UIKit constructs the `App` before it delivers a restoration callback. If that ever stops being + /// true, `StoreReplication.configuredAfterFirstOpen` flips to `true` and says so rather than + /// leaving a half-configured process to be inferred from a bad snapshot rate. + /// + /// Deliberately does nothing when the trial is off: the `.automatic` branch would set the same + /// values `StoreReplication` already ships, and not calling `configure` at all keeps + /// `openedStoreCount`/`configuredAfterFirstOpen` meaningful for a build that never opted in. + static func applyAtLaunch(enabled: Bool = isEnabled) { + guard enabled else { return } + StoreReplication.configure(walCheckpointing: .external, walBackstop: .standard) + // Deliberately does not build `statusLine` here: that touches `shared`, and this runs on the + // launch-critical path before the first frame. The interesting half of the trial's state is + // logged after each push, where file I/O has already happened. + NSLog("SyncReplicationTrial: ON — external WAL checkpointing, backstop at %lld bytes", + StoreReplication.walBackstop.ceilingBytes) + } + + /// The shared recorder, opened lazily so a build with the trial off never creates the file. + /// `nil` if Application Support is unavailable (it never is on a real device) — telemetry is + /// best-effort and must never be able to fail a sync. + /// + /// Also `nil` under XCTest, for the same reason `CloudSyncModel.autoSyncIfDue` refuses to run + /// there: `StrandTests` executes inside the full app via `TEST_HOST`, so it resolves the developer + /// Mac's REAL Application Support directory. Without this, every test that exercises an upload + /// would append junk records to the live trial file and corrupt the one measurement this exists to + /// produce. Tests that want to assert on telemetry inject their own instance over a temp URL. + static let shared: SyncPushTelemetry? = { + guard !CloudSyncModel.isRunningUnderXCTest else { return nil } + guard let url = try? SyncPushTelemetry.defaultURL() else { return nil } + return SyncPushTelemetry(url: url) + }() + + /// One line describing the trial's state, for the log and for a diagnostics row. Combines the + /// *intent* (the flag), the *reality* (what the store actually opened with), and the *result* + /// (the telemetry aggregate) — the three are not the same and have to be read together. + static var statusLine: String { + let inForce: String + switch StoreReplication.walCheckpointing { + case .external: inForce = "external" + case .automatic: inForce = "automatic" + } + let late = StoreReplication.configuredAfterFirstOpen ? " LATE-CONFIGURE" : "" + let telemetry = shared.map { " " + $0.oneLineSummary } ?? "" + // The FOURTH fact, and the one that was missing: what the last attempt actually did. The + // three above describe intent, reality and result-so-far, and all three read "healthy" while + // every push fails — a failing push records no telemetry at all, so `pushes=0` means both + // "never tried" and "tried and failed every time". See + // `CloudSyncUploader.litersBreadcrumbKey`. + let last = UserDefaults.standard.string(forKey: CloudSyncUploader.litersBreadcrumbKey) + let outcome = last.map { "\nlast liters attempt: \($0)" } ?? "\nlast liters attempt: none recorded" + return "replicationTrial enabled=\(isEnabled) checkpointing=\(inForce)\(late)\(telemetry)\(outcome)" + } +} +#endif diff --git a/Strand/Collect/RawHistoryArchive.swift b/Strand/Collect/RawHistoryArchive.swift index bff5ab0bb3..76d2c447d6 100644 --- a/Strand/Collect/RawHistoryArchive.swift +++ b/Strand/Collect/RawHistoryArchive.swift @@ -18,11 +18,16 @@ import WhoopProtocol struct RawHistoryArchive { /// File name under `/com.noopapp.noop/`. static let fileName = "rejected_history.jsonl" - /// Soft cap (~5 MB). When appending would push the file past this, the archive EVICTS oldest - /// surplus lines to make room rather than refusing the write — but only down to a per-version - /// retention floor (see `perVersionFloor`), so a brand-new layout version is never binned merely - /// because common versions filled the file. Only when the incoming frames alone can't fit even an - /// empty archive does `archive` skip them (reported as unarchived). (#344) + /// Soft cap (~5 MB). When appending would push the file past this, the archive EVICTS surplus lines + /// to make room rather than refusing the write — weakest-value first, and never below a per-version + /// retention floor, so a brand-new layout version is never binned merely because common versions + /// filled the file (see `evictLines` for the full priority order). Only when the incoming frames + /// alone can't fit even an empty archive does `archive` skip them (reported as unarchived). (#344) + /// + /// Deliberately NOT raised to survive a placeholder flood. At ~3.2 KB per hex line a 1 Hz stream of + /// 1584-byte records fills any cap in hours, so a bigger number buys a linear factor of time and + /// costs every user disk — while `zeroPayloadFloor` removes the pressure outright by evicting the + /// informationless records first. Value, not volume, is what was missing. static let maxBytes = 5 * 1024 * 1024 /// Per distinct layout VERSION, keep at least this many of the newest archived lines, immune to @@ -32,6 +37,48 @@ struct RawHistoryArchive { /// can hold at most `floor × distinctVersions` lines, a tiny set. (#344) static let perVersionFloor = 64 + /// Per-version retention floor for lines whose PAYLOAD REGION IS ENTIRELY ZERO — far smaller than + /// `perVersionFloor`, because such a record carries no information beyond "the strap banked a record + /// at time T". + /// + /// Measured motivation: with `enable_raw_data_w_ecg` armed, a WHOOP MG banks one 1584-byte record + /// PER SECOND whose every byte from offset 21 to the CRC32 trailer is zero (invariant across 1,307 + /// captured frames). At ~3.2 KB per hex JSONL line that fills the whole 5 MB archive in ~25 minutes, + /// so under a purely age-ordered policy an actual ECG record captured during the electrode-contact + /// window is evicted by placeholders within the half hour — the most likely reason the 2026-07-27 + /// contact frames were already gone by the 2026-08-03 analysis. Keeping a handful still proves the + /// artefact exists (and dates it); keeping thousands buys nothing and costs the evidence. + static let zeroPayloadFloor = 8 + + /// First byte of a record's PAYLOAD region — everything past the record header, ending at the 4-byte + /// CRC32 trailer. 21 on WHOOP 5/MG (magic @0, seq @11, unix ts @15–18, a constant @19–20), and 17 on + /// WHOOP 4: the same +4 that shifts the type byte 4→8 and the version byte 5→9, because the puffin + /// envelope is 4 bytes longer. Starting AFTER the header is load-bearing — a region that included the + /// non-zero seq/timestamp bytes would never test as all-zero and the rule below would never fire. + static func payloadStart(_ family: DeviceFamily) -> Int { family == .whoop5 ? 21 : 17 } + + /// Bytes of the CRC32 trailer excluded from the payload region. + static let crcTrailerBytes = 4 + + /// A payload region shorter than this is not judged at all (`hasZeroPayload` returns false): a + /// truncated or tiny frame is not evidence of an empty record, and must not lose retention priority + /// on the strength of a few zero bytes. + static let minPayloadBytesToJudge = 32 + + /// True when EVERY byte of the frame's payload region is zero — an informationless record. + /// + /// Deliberately the strict test: a single non-zero byte anywhere in the region makes the frame + /// informative and gives it full retention priority. "Mostly zero" is NOT the test — a novel record + /// that carries one small populated block inside an otherwise-empty buffer is precisely the thing + /// this archive exists to catch. + static func hasZeroPayload(_ frame: [UInt8], family: DeviceFamily) -> Bool { + let start = payloadStart(family) + let end = frame.count - crcTrailerBytes + guard end - start >= minPayloadBytesToJudge else { return false } + for i in start../com.noopapp.noop/`, created on demand. Overridable for tests. init(directory: URL? = nil, maxBytes: Int = RawHistoryArchive.maxBytes, - perVersionFloor: Int = RawHistoryArchive.perVersionFloor) { + perVersionFloor: Int = RawHistoryArchive.perVersionFloor, + zeroPayloadFloor: Int = RawHistoryArchive.zeroPayloadFloor) { if let directory { self.directory = directory } else { @@ -65,6 +114,7 @@ struct RawHistoryArchive { } self.maxBytes = maxBytes self.perVersionFloor = perVersionFloor + self.zeroPayloadFloor = zeroPayloadFloor } /// The archive file URL (does not create anything). @@ -135,7 +185,8 @@ struct RawHistoryArchive { // within the newest `perVersionFloor` of its version. `evictLines` is the pure, unit-tested core. let existing = (try? String(contentsOf: url, encoding: .utf8)) .map { $0.split(separator: "\n", omittingEmptySubsequences: true).map { String($0) + "\n" } } ?? [] - let kept = RawHistoryArchive.evictLines(existing + newLines, maxBytes: maxBytes, floor: perVersionFloor) + let kept = RawHistoryArchive.evictLines(existing + newLines, maxBytes: maxBytes, + floor: perVersionFloor, zeroFloor: zeroPayloadFloor) let data = Data(kept.joined().utf8) do { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -160,11 +211,26 @@ struct RawHistoryArchive { } } - /// The `VersionKey` of one archived JSONL line, re-derived from its stored frame + family; `nil` - /// for a malformed/unparseable line — those get NO retention floor (evict them first, never let - /// garbage occupy a floor slot a real rare version could use), and are always evictable before any - /// real, floor-protected line. - private static func lineVersionKey(_ line: String) -> VersionKey? { + /// How one archived line is classified for retention: which version bucket it belongs to, and + /// whether its payload region is entirely zero (informationless — see `hasZeroPayload`). + private struct LineClass { let key: VersionKey; let zeroPayload: Bool } + + /// Classify one archived JSONL line from its stored frame + family; `nil` for a + /// malformed/unparseable line — those get NO retention floor (evict them first, never let garbage + /// occupy a floor slot a real rare version could use), and are always evictable before any real, + /// floor-protected line. + private static func lineClass(_ line: String) -> LineClass? { + guard let (bytes, family) = parseArchiveLine(line) else { return nil } + return LineClass(key: VersionKey(family: family.rawValue, + version: versionByte(bytes, family: family)), + zeroPayload: hasZeroPayload(bytes, family: family)) + } + + /// The frame bytes + family of one archived JSONL line, or `nil` if the line is malformed. + /// Hand-parsed to match the hand-built writer: the only dynamic fields are `family` and the + /// [0-9a-f] `frameHex`. Shared by `readAll`, retention, and the operator summary so all three agree + /// on exactly which lines are readable. Twin of the Android `RawHistoryArchive.parseArchiveLine`. + static func parseArchiveLine(_ line: S) -> (frame: [UInt8], family: DeviceFamily)? { guard let fr = line.range(of: "\"family\":\""), let hr = line.range(of: "\"frameHex\":\"") else { return nil } let fam = String(line[fr.upperBound...].prefix { $0 != "\"" }) @@ -176,36 +242,75 @@ struct RawHistoryArchive { guard let b = UInt8(hex[i.. [String] { - // Mark the newest `floor` indices of each version as protected (scan newest→oldest). Malformed - // lines (nil key) are never protected — they are evicted before any real, floor-kept line. - let keys = lines.map { lineVersionKey($0) } + /// newline-terminated JSONL set, oldest-first (existing + incoming). + /// + /// Retention priority, weakest first — everything in a band is evicted before anything in the next: + /// 1. malformed lines (unparseable — no version, no floor); + /// 2. surplus lines whose payload region is ENTIRELY ZERO (`hasZeroPayload`), oldest first; + /// 3. surplus informative lines, oldest first; + /// 4. floor-protected lines, never evicted: the newest `zeroFloor` zero-payload lines of each + /// version and the newest `floor` informative lines of each version. + /// + /// Band 2 is what keeps an ECG capture recoverable. A 5/MG with `enable_raw_data_w_ecg` armed banks + /// one all-zero 1584-byte record per second; age-ordered eviction alone would flush a real + /// contact-window record out of a 5 MB archive within ~25 minutes of placeholders, and an all-zero + /// payload demonstrably carries nothing that would be worth that trade. Band 4's split floor means a + /// version whose records are ALL empty still keeps a few dated samples of the artefact, while the + /// full `floor` stays reserved for records that actually contain something. + /// + /// Bounded: if everything left is floor-protected it stops even if still over cap (the floor wins — + /// `(floor + zeroFloor) × distinctVersions` is tiny). Twin of the Android + /// `RawHistoryArchive.evictLines`, whose `zeroFloor` is defaulted for the same reason it is here — so + /// existing call sites do not have to churn. + static func evictLines(_ lines: [String], maxBytes: Int, floor: Int, + zeroFloor: Int = RawHistoryArchive.zeroPayloadFloor) -> [String] { + // Mark the newest `floor` informative and newest `zeroFloor` zero-payload indices of each version + // as protected (scan newest→oldest). Malformed lines (nil class) are never protected — they are + // evicted before any real, floor-kept line. + let classes = lines.map { lineClass($0) } var protectedIdx = Set() - var seen: [VersionKey: Int] = [:] + var seenInformative: [VersionKey: Int] = [:] + var seenZero: [VersionKey: Int] = [:] for idx in lines.indices.reversed() { - guard let k = keys[idx] else { continue } - let c = (seen[k] ?? 0) - if c < floor { protectedIdx.insert(idx); seen[k] = c + 1 } + guard let c = classes[idx] else { continue } + if c.zeroPayload { + let n = (seenZero[c.key] ?? 0) + if n < zeroFloor { protectedIdx.insert(idx); seenZero[c.key] = n + 1 } + } else { + let n = (seenInformative[c.key] ?? 0) + if n < floor { protectedIdx.insert(idx); seenInformative[c.key] = n + 1 } + } } var total = lines.reduce(0) { $0 + $1.utf8.count } guard total > maxBytes else { return lines } - // Evict oldest unprotected lines first. + let target = maxBytes - maxBytes / lowWaterDivisor + // Evict in weakest-first bands; within a band, oldest first (ascending index). var dropped = Set() - for idx in lines.indices where total > maxBytes { // ascending = oldest first - if protectedIdx.contains(idx) { continue } - dropped.insert(idx) - total -= lines[idx].utf8.count + func evict(_ admits: (Int) -> Bool) { + for idx in lines.indices where total > target { + if protectedIdx.contains(idx) || dropped.contains(idx) { continue } + guard admits(idx) else { continue } + dropped.insert(idx) + total -= lines[idx].utf8.count + } } + evict { classes[$0] == nil } // 1. malformed + evict { classes[$0]?.zeroPayload == true } // 2. zero-payload surplus + evict { _ in true } // 3. informative surplus return lines.enumerated().filter { !dropped.contains($0.offset) }.map(\.element) } @@ -214,22 +319,76 @@ struct RawHistoryArchive { /// match the hand-built writer: the only dynamic fields are `family` and the [0-9a-f] `frameHex`.) func readAll() -> [(frame: [UInt8], family: DeviceFamily)] { guard let text = try? String(contentsOf: fileURL, encoding: .utf8) else { return [] } - return text.split(separator: "\n").compactMap { line in - guard let fr = line.range(of: "\"family\":\""), - let hr = line.range(of: "\"frameHex\":\"") else { return nil } - let fam = String(line[fr.upperBound...].prefix { $0 != "\"" }) - let hex = line[hr.upperBound...].prefix { $0 != "\"" } - guard let family = DeviceFamily(rawValue: fam), hex.count % 2 == 0 else { return nil } - var bytes = [UInt8](); bytes.reserveCapacity(hex.count / 2); var i = hex.startIndex - while i < hex.endIndex { - let j = hex.index(i, offsetBy: 2) - guard let b = UInt8(hex[i.. String { + let text = (try? String(contentsOf: fileURL, encoding: .utf8)) ?? "" + let lines = text.split(separator: "\n", omittingEmptySubsequences: true).map(String.init) + let size = (try? FileManager.default.attributesOfItem(atPath: fileURL.path)) + .flatMap { $0[.size] as? Int } ?? 0 + return "\(fileURL.path)\n\(RawHistoryArchive.summaryText(lines: lines, sizeBytes: size, newest: newest))" + } + + /// The pure renderer behind `summaryText()` — takes the raw JSONL lines so it is unit-testable + /// without a file. `lines` is oldest-first, exactly as stored. + static func summaryText(lines: [String], sizeBytes: Int, newest: Int = 8) -> String { + guard !lines.isEmpty else { return "Raw history archive: EMPTY (no frames archived yet)." } + struct Bucket { var informative = 0; var zero = 0 } + var buckets: [String: Bucket] = [:] + var malformed = 0 + var rows: [String] = [] + for line in lines { + guard let (frame, family) = parseArchiveLine(line) else { malformed += 1; continue } + let version = versionByte(frame, family: family) + let label = "\(family.rawValue) v\(version)" + let isZero = hasZeroPayload(frame, family: family) + var b = buckets[label] ?? Bucket() + if isZero { b.zero += 1 } else { b.informative += 1 } + buckets[label] = b + // Sliding window over the newest `newest` readable lines (the file is oldest-first). + if newest > 0 { + if rows.count == newest { rows.removeFirst() } + rows.append(" \(stamp(line)) \(label) \(frame.count) B \(isZero ? "zero-payload" : "INFORMATIVE")") } - return (bytes, family) } + var out = "Raw history archive: \(lines.count) line(s), \(sizeBytes / 1024) KB" + out += "\n\nBy layout version:" + for label in buckets.keys.sorted() { + let b = buckets[label]! + out += "\n \(label) \(b.informative + b.zero) line(s) — \(b.informative) informative, \(b.zero) all-zero payload" + } + if malformed > 0 { out += "\n (malformed) \(malformed) line(s)" } + if !rows.isEmpty { + out += "\n\nNewest \(rows.count) capture(s), phone clock:\n" + rows.joined(separator: "\n") + } + return out } + /// `capturedAtMs` of one archived line rendered in the phone's local time, or `?` when unreadable. + private static func stamp(_ line: String) -> String { + guard let r = line.range(of: "\"capturedAtMs\":"), + let ms = Double(line[r.upperBound...].prefix { $0 == "." || ("0"..."9").contains($0) }) + else { return "????-??-?? ??:??:??" } + return stampFormatter.string(from: Date(timeIntervalSince1970: ms / 1000)) + } + + private static let stampFormatter: DateFormatter = { + let f = DateFormatter() + f.dateFormat = "yyyy-MM-dd HH:mm:ss" + return f + }() + /// Re-decode every archived frame through the CURRENT decoder and insert whatever now decodes. /// The strap freed these records when they were acked, so this archive is the ONLY way banked /// history backfills after a newly-landed layout (e.g. WHOOP 4.0 v25). Idempotent: offloaded rows diff --git a/Strand/Data/AppleHealthImport.swift b/Strand/Data/AppleHealthImport.swift index 7163c304bd..8d9a74c27c 100644 --- a/Strand/Data/AppleHealthImport.swift +++ b/Strand/Data/AppleHealthImport.swift @@ -48,7 +48,8 @@ enum AppleHealthImport { // #89: Apple Health steps must land in DailyMetric.steps too — the sourced-daily // arbitration resolves "steps" via metricValue(d) = d.steps, so leaving it nil (the // pre-fix state) meant imported Apple steps never surfaced there. - steps: d.steps.map { Int($0) }) + steps: d.steps.map { Int($0) }, + avgSdnn: d.hrvSDNN) // Apple HRV is SDNN — mirror into the SDNN field } let dmWritten = try await store.upsertDailyMetrics(dm, deviceId: deviceId) diff --git a/Strand/Data/DataBackup.swift b/Strand/Data/DataBackup.swift index 8187a1017b..28de506c39 100644 --- a/Strand/Data/DataBackup.swift +++ b/Strand/Data/DataBackup.swift @@ -199,6 +199,38 @@ enum DataBackup { } } + /// (Cloud sync, replicated stores) The same verified `.noopbak` as `writeBackup(checkpoint:to:)`, + /// but archiving a database the caller has already staged at `sourceURL` — and therefore taking + /// **no checkpoint at all**. + /// + /// Exists for exactly one caller: `CloudSyncUploader.defaultExporter` under + /// `WalCheckpointing.external`, where a checkpoint of the live store would restart the WAL under + /// a page replicator and cost its next push a full snapshot of the database (measured — see + /// `WhoopStore.writeConsistentCopy(to:)`). The staged file is a consistent full copy produced by + /// SQLite's Online Backup API, so it needs no flushing: it already carries everything the WAL + /// held. The caller owns `sourceURL`'s lifetime. + /// + /// Same `writeVerifiedBackupZip` as every other export, so the container, the entry names, the + /// settings sidecar and the `PRAGMA quick_check` gate are identical — a `.noopbak` produced this + /// way is indistinguishable from one produced by the checkpointing path. Never presents UI. + static func writeBackup(stagedDatabaseAt sourceURL: URL, to dest: URL) async -> BackupResult { + guard FileManager.default.fileExists(atPath: sourceURL.path) else { + return .failure(String(localized: "The staged database copy is missing.")) + } + do { + let fm = FileManager.default + if fm.fileExists(atPath: dest.path) { try fm.removeItem(at: dest) } + // Off the caller's thread for the same reason the interactive export is: a whole-database + // read plus DEFLATE. + try await Task.detached(priority: .utility) { + try writeVerifiedBackupZip(dbURL: sourceURL, to: dest, settingsJSON: currentSettingsJSON()) + }.value + return .exported(dest) + } catch { + return .failure(String(localized: "Backup failed: \(error.localizedDescription)")) + } + } + /// Test seam: write a `.noopbak` for an EXPLICIT source database (no checkpoint, no `StorePaths`), /// so a unit test can round-trip a throwaway SQLite through the exact ZIP container the app writes. /// `settings` (canonical `BackupSettings` keys) adds the `settings.json` entry; nil writes the diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index 9714d7f701..88dbaf927d 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -408,6 +408,17 @@ final class IntelligenceEngine: ObservableObject { // boundary inside the window is a negligible edge case for an hour-of-day band.) let tzOffset = TimeZone.current.secondsFromGMT() + // #tz-upload: stamp TODAY's local calendar day with the phone's IANA timezone so historical + // nights stay localisable once the user crosses zones (Pacific ↔ Eastern). Every timestamp in + // the DB is epoch-UTC and `tzOffset` above is only an integer offset — it can't distinguish + // "America/Los_Angeles" from any other −07:00 zone or survive a DST transition. analyzeRecent is + // the daily hook (idle tick + every post-sync rescore), and the upsert is keyed by day so + // repeated same-day passes overwrite one row rather than churning. Fire-and-forget: a failure + // here must never block scoring. The same identifier also rides each upload as the + // `X-Phone-Timezone` header (CloudSyncClient.ingest). + let todayLocal = AnalyticsEngine.dayString(Self.midnightLocal(now, offsetSec: tzOffset), offsetSec: tzOffset) + try? await store.upsertPhoneTimezone(day: todayLocal, tzId: TimeZone.current.identifier) + // ── Pass 1: analyse each offloaded night against the IMPORTED-ONLY baseline. For a BLE-only // user the imported daily rows are empty, so the HRV baseline isn't usable yet and recovery is // null here , but each night's avgHrv/restingHr are computed baseline-INDEPENDENTLY, so we @@ -471,10 +482,20 @@ final class IntelligenceEngine: ObservableObject { // Returns nil under `habitualMinDays` of history → cold-start: every `analyzeDay`/`sleepEditedDaily` // call below stays on the overnight-band bonus. The same value threads into both seams so analytics // and the Sleep tab resolve to the identical block. (#547) - let habitualMidsleepSec = await Self.computeHabitualMidsleep( + let (habitualMidsleepSec, nightlyHours) = await Self.computeHabitualSleep( store: store, importedId: deviceId, computedId: deviceId + "-noop", windowStart: nowLocalMidnight - maxDays * 86_400 - 30 * 3_600, windowEnd: now, offsetSec: tzOffset) + // Wave 0 (SL1/T1): personal sleep REGULARITY + population-anchored NEED, computed ONCE from the + // trailing per-night durations and threaded to every analyzeDay below (mirrors the midsleep + // learner just above — one personal trait per run, applied to the whole re-scored history so + // Rest stops running on a flat neutral-0.5 consistency and a fixed 8 h need). Recent 28-night + // window for regularity (a recent-behaviour signal); full history for the need's upper-quartile + // "unrestricted nights" estimate. Both degrade honestly on thin history (consistency → nil → + // neutral term; need → population default), so cold-start is unchanged. + let sleepConsistency = VitalityEngine.sleepConsistency(nightlyHours: Array(nightlyHours.suffix(28))) + let sleepNeedHours = AnalyticsEngine.Rest.personalizedNeedHours(nightlyHours: nightlyHours, + age: profile.age) // ── FIX 1 (main-actor jank): run the ENTIRE per-day enumeration OFF the main actor ─────────── // Every `await store.…` read inside this loop has its continuation RESUME on the main actor @@ -556,6 +577,12 @@ final class IntelligenceEngine: ObservableObject { // #93: WHOOP 4.0 raw SpO2 PPG samples for the night; analyzeDay banks the nightly red/IR ADC // means on the DailyMetric. Empty on a 5/MG (no v24 spo2 channels) → the raw means stay nil. let spo2 = (try? await store.spo2Samples(deviceId: owner, from: from, to: to, limit: 200_000)) ?? [] + // v34: the 5/MG's own per-second `@82` SpO2 percentages for the night; analyzeDay banks the + // ramp-trimmed nightly MEDIAN on `DailyMetric.spo2Pct`. Read from the durable table, which + // is never pruned — so re-scoring an old night still finds its readings, where the capped + // aux table they were forked from has held only ~7 days since v31. Empty on a WHOOP 4.0 and + // on windows offloaded before v34 → the percentage stays nil rather than fabricated. + let spo2Pct = (try? await store.spo2PctSamples(deviceId: owner, from: from, to: to, limit: 200_000)) ?? [] // #938: the strap family that WROTE this owner's skin-temp rows, so analyzeDay converts the raw // register on the right scale (5/MG banks centidegrees, a WHOOP 4.0 v24 banks a raw ADC). The // registry knows each device's model; unknown/non-WHOOP owners fall back to `.whoop5` (the prior @@ -675,6 +702,10 @@ final class IntelligenceEngine: ObservableObject { // ever runs AFTER whichever stager above just ran, and self-gates on the night's observed // gravity + step density, so flipping it on is a no-op for any night too sparse to trust. let useMotionAwareWake = PuffinExperiment.motionAwareWakeEnabled + // Wake-gap bridging (experimental, default OFF): read once, off the detached + // executor, like the two flags above. Assembles a night split by a brief get-up into + // one session with the wake staged inside — see `PuffinExperiment.wakeBridgeEnabled`. + let useWakeBridge = PuffinExperiment.wakeBridgeEnabled // Already OFF the main actor , score directly (the prior nested `Task.detached` here only // existed to hop off the main actor; the whole loop now runs off it, so the score is computed @@ -693,8 +724,11 @@ final class IntelligenceEngine: ObservableObject { skinTempFamily: skinFamily, // #938 skinTempAnchorRaw: skinAnchorRaw, // #938 second capture spo2: spo2, // #93 + spo2PctSamples: spo2Pct, // v34 profile: up, baselines: baselines1, maxHROverride: maxHR, tzOffsetSeconds: tzOffset, wristOff: wristOff, + sleepNeedHours: sleepNeedHours, + sleepConsistency: sleepConsistency, habitualMidsleepSec: habitualMidsleepSec, bandSleepState: bandSleepState, // #690: thread the V2 toggle into the NORMAL staging path so @@ -703,6 +737,8 @@ final class IntelligenceEngine: ObservableObject { // #364 follow-up: same threading for the motion-aware wake // refinement post-pass. useMotionAwareWake: useMotionAwareWake, + // Wake-gap bridging (experimental, default OFF). + useWakeBridge: useWakeBridge, traceSink: traceSink, hrvTraceSink: hrvTraceSink, // Per-window HRV detail ONLY for the most-recent night @@ -1846,10 +1882,15 @@ final class IntelligenceEngine: ObservableObject { return samples } - private static func computeHabitualMidsleep( + /// Habitual midsleep (local seconds) AND the trailing per-night sleep DURATIONS (hours, + /// chronological) from the stored sessions over the window — the longest block per LOCAL day, so + /// naps drop out. One read serves both the main-night midsleep learner (#547) and the personal + /// sleep-need + regularity that thread into `analyzeDay` (Wave 0 · SL1/T1). The midsleep result is + /// byte-identical to before; the nightly-hours output is the Swift-side extension. + private static func computeHabitualSleep( store: WhoopStore, importedId: String, computedId: String, windowStart: Int, windowEnd: Int, offsetSec: Int - ) async -> Int? { + ) async -> (midsleepSec: Int?, nightlyHours: [Double]) { let imported = (try? await store.sleepSessions(deviceId: importedId, from: windowStart, to: windowEnd, limit: 4000)) ?? [] let computed = (try? await store.sleepSessions(deviceId: computedId, from: windowStart, @@ -1861,14 +1902,30 @@ final class IntelligenceEngine: ObservableObject { // covers an imported night and its computed twin (the longest capture wins, exactly what the // per-day length rule chose anyway). let merged = SleepSessionDedup.dedupe(imported + computed).kept + // Longest block per LOCAL day (naps drop out), chosen by in-bed SPAN — reused for BOTH the + // midsleep learner and the per-night durations (Wave 0 · SL1/T1), so the two can never read a + // different history. For the DURATIONS we keep TST (span × efficiency), NOT the in-bed span: + // the need/regularity estimate must be in the same asleep-time units as the `tstSeconds` Rest + // scores against, or need reads systematically high (validated on real data — an in-bed span + // over-counts ~0.85 h vs TST). Efficiency is 0..1 (post the v26 unit-heal); a rare nil main + // night falls back to a typical 0.9. + var longestByDay: [String: (span: Int, tstHours: Double)] = [:] let blocks = merged.compactMap { s -> SleepStageTotals.HistoryBlock? in let start = s.effectiveStartTs, end = s.endTs guard end > start else { return nil } let mid = start + (end - start) / 2 let dayKey = AnalyticsEngine.dayString(mid, offsetSec: offsetSec) + let span = end - start + if span > (longestByDay[dayKey]?.span ?? 0) { + let eff = s.efficiency.flatMap { (0.0 < $0 && $0 <= 1.0) ? $0 : nil } ?? 0.9 + longestByDay[dayKey] = (span, Double(span) / 3600.0 * eff) + } return SleepStageTotals.HistoryBlock(start: start, end: end, dayKey: dayKey) } - return SleepStageTotals.habitualMidsleepSec(blocks, offsetSec: offsetSec) + let midsleep = SleepStageTotals.habitualMidsleepSec(blocks, offsetSec: offsetSec) + // Chronological (day-key string sort == date order) so a recent-window suffix is well-defined. + let nightlyHours = longestByDay.keys.sorted().compactMap { longestByDay[$0]?.tstHours } + return (midsleep, nightlyHours) } /// Floor a unix-seconds timestamp to 00:00:00 of its UTC calendar day. Mirrors the Android @@ -1902,7 +1959,7 @@ private extension DailyMetric { avgHrv: avgHrv, recovery: r, strain: strain, exerciseCount: exerciseCount, spo2Pct: spo2Pct, skinTempDevC: sd, respRateBpm: respRateBpm, steps: steps, activeKcalEst: activeKcalEst, - spo2Red: spo2Red, spo2Ir: spo2Ir) + spo2Red: spo2Red, spo2Ir: spo2Ir, avgSdnn: avgSdnn) } /// Rebuild with substituted sleep-derived fields (a user-corrected wake window), leaving every @@ -1913,7 +1970,7 @@ private extension DailyMetric { disturbances: disturbances, restingHr: restingHr, avgHrv: avgHrv, recovery: recovery, strain: strain, exerciseCount: exerciseCount, spo2Pct: spo2Pct, skinTempDevC: skinTempDevC, respRateBpm: respRateBpm, steps: steps, - activeKcalEst: activeKcalEst, spo2Red: spo2Red, spo2Ir: spo2Ir) + activeKcalEst: activeKcalEst, spo2Red: spo2Red, spo2Ir: spo2Ir, avgSdnn: avgSdnn) } } diff --git a/Strand/Data/Repository.swift b/Strand/Data/Repository.swift index 755073bdfa..11fa77d46c 100644 --- a/Strand/Data/Repository.swift +++ b/Strand/Data/Repository.swift @@ -594,6 +594,16 @@ final class Repository: ObservableObject { /// Checkpoint the WAL into the main DB file if the store is already open, so a file-level /// backup captures everything. No-op (returns false) if no handle exists yet , the caller /// then copies the on-disk files as-is, which still includes the -wal sidecar. + /// + /// Known residual hazard, deliberately not changed here: under `WalCheckpointing.external` this + /// checkpoint restarts the WAL underneath a page replicator and costs its next push a full + /// snapshot of the database (see `WhoopStore.writeConsistentCopy(to:)` for the measurement, and + /// `CloudSyncUploader.defaultExporter` for the pattern that avoids it). It is left alone because + /// its callers are the *user-initiated* file-level backups — the Export button, and + /// `BackupSync.catchUpIfDue`, which is gated on the auto-backup toggle AND a chosen folder AND a + /// full day elapsed. Those want the disk reclaimed, they are at most daily, and swapping them to + /// a staged full copy would make a manual export visibly slower for everyone to protect a trial + /// that only one device is running. Revisit if page replication becomes the default. func checkpointForBackup() async -> Bool { guard let store else { return false } do { try await store.checkpointWAL(); return true } catch { return false } @@ -911,6 +921,67 @@ final class Repository: ObservableObject { return await unionDailyMetrics(store: store, from: fromDay, to: toDay) } + /// The most recent night's HRV in BOTH metrics, plus the Apple reading for the same day when present — + /// backs the "Latest Night" dual-metric header + the honest Apple cross-check on the Cross-Device screen. + /// `rmssd` is the strap's `avgHrv` (beat-to-beat); `sdnn` is `avgSdnn` (the 5-min SDNN index, already + /// window-matched to Apple's short-window SDNN, so `sdnn` vs `appleSdnn` is a like-for-like check). + struct NightHRV: Equatable { + let day: String + let rmssd: Double? + let sdnn: Double? + let appleSdnn: Double? + } + + func latestNightHRV() async -> NightHRV? { + guard let store = await ensureStore() else { return nil } + var latest: DailyMetric? + for id in importedReadIds + computedReadIds { + let rows = (try? await store.dailyMetrics(deviceId: id, from: "2018-01-01", to: "9999-12-31")) ?? [] + for r in rows where r.avgHrv != nil { + if latest == nil || r.day > latest!.day { latest = r } + } + } + guard let night = latest else { return nil } + let appleRows = (try? await store.dailyMetrics(deviceId: Self.appleHealthSource, + from: night.day, to: night.day)) ?? [] + return NightHRV(day: night.day, rmssd: night.avgHrv, sdnn: night.avgSdnn, + appleSdnn: appleRows.first?.avgSdnn) + } + + /// Full-history nightly HRV tagged by its WINNING source id, for the cross-device HRV trend + /// (`CrossDeviceHRVTrend`). One winner per day: the real wearable of each era (Oura under "oura-api", + /// WHOOP under the strap/import/computed ids) — the real id strings are preserved so the engine's + /// `brandBucket` segments the Oura era from the WHOOP era. `apple-health` is deliberately EXCLUDED: + /// Apple's HRV is SDNN, method-incompatible with the strap/Oura RMSSD this trend plots (the #696 + /// method-mixing hazard), and its `brandBucket` "whoop" tag would fragment a wearable era on any day it + /// backfilled. Oldest→newest; only nights with a positive HRV. + func crossDeviceHrvHistory(from: String = "2018-01-01", + to: String = "9999-12-31") async + -> [(day: String, sourceId: String, hrv: Double)] { + guard let store = await ensureStore() else { return [] } + // Priority: live strap + canonical import first, then wearable imports (Oura/Fitbit/Garmin), then + // the computed "-noop" sibling as a fallback. All whoop-family ids bucket to "whoop", so intra-era + // precedence only picks which value represents a shared WHOOP-era day; Oura owns its own era. + var ordered = importedReadIds + ordered.append(contentsOf: Self.wearableImportSources) + ordered.append(contentsOf: computedReadIds) + var seen = Set() + let ids = ordered.filter { seen.insert($0).inserted } + + var winnerByDay: [String: (rank: Int, sourceId: String, hrv: Double)] = [:] + for (rank, id) in ids.enumerated() { + let rows = (try? await store.dailyMetrics(deviceId: id, from: from, to: to)) ?? [] + for r in rows { + guard let hrv = r.avgHrv, hrv > 0 else { continue } + if let cur = winnerByDay[r.day], cur.rank <= rank { continue } + winnerByDay[r.day] = (rank, id, hrv) + } + } + return winnerByDay + .map { (day: $0.key, sourceId: $0.value.sourceId, hrv: $0.value.hrv) } + .sorted { $0.day < $1.day } + } + /// #856: the same dedup over an EXPLICIT id list, so a workout's zone minutes bin the rows its own /// recording strap banked rather than the day-level active ∪ canonical. Order is precedence. func hrSamples(deviceIds: [String], from: Int, to: Int, limit: Int = 8000) async -> [HRSample] { @@ -1403,6 +1474,12 @@ final class Repository: ObservableObject { guard !edited.isEmpty else { return [] } var healed = false for row in edited { + // Stage-locked rows carry an AUTHORITATIVE stage correction (cloud edit_sleep_stages — + // see CloudEditApplier's lock write). Re-deriving from raw would revert the correction + // on every analyze pass, since a deliberate edit always differs from the stager's output. + if ((try? await store.cursor("stagelock:\(computedDeviceId):\(row.startTs)")) ?? 0) == 1 { + continue + } // Re-derive over the LOCKED corrected window (effective onset → wake). Skip when the raw // isn't dense yet, or when the result already matches what's stored (steady state , no write). guard let newJSON = await restageFromRaw(start: row.effectiveStartTs, end: row.endTs), @@ -2792,7 +2869,8 @@ private extension DailyMetric { // Raw SpO2 is on-device only (imports never carry it), so the imported row's nil is // backfilled from the computed fallback — otherwise the nightly means would be lost. (#93) spo2Red: spo2Red ?? fallback.spo2Red, - spo2Ir: spo2Ir ?? fallback.spo2Ir + spo2Ir: spo2Ir ?? fallback.spo2Ir, + avgSdnn: avgSdnn ?? fallback.avgSdnn ) } @@ -2820,7 +2898,8 @@ private extension DailyMetric { steps: steps, activeKcalEst: activeKcalEst, spo2Red: spo2Red, // non-sleep field: preserved as-is (#93) - spo2Ir: spo2Ir + spo2Ir: spo2Ir, + avgSdnn: avgSdnn // non-sleep (HRV) field: preserved as-is ) } } diff --git a/Strand/Liquid/LiquidTodayView.swift b/Strand/Liquid/LiquidTodayView.swift index 3195966903..4f8524c3b7 100644 --- a/Strand/Liquid/LiquidTodayView.swift +++ b/Strand/Liquid/LiquidTodayView.swift @@ -91,6 +91,13 @@ struct LiquidTodayView: View { /// today's row has no vitals yet, so these fall back to the last night that recorded them. Never /// resolved in body — body rescans repo.days ~23× per pass, and this cache keeps that read O(1). @State private var cachedVitalsDay: DailyMetric? + /// PER-FIELD SpO₂ / skin-temp carries — the twins of `cachedVitalsDay` for the two fields its predicate + /// does NOT check (it gates on HRV / RHR / respiratory). The engine writes `spo2Pct` / `skinTempDevC = + /// nil` on every computed "-noop" row, so the whole-row carry lands on a null value while an older imported + /// row holds a real one; these resolve each field from its own freshest strictly-prior row. Resolved ONCE + /// in load() (never in body) so the Blood-Oxygen / Skin-Temp tiles read O(1); non-nil only on today. + @State private var cachedSpo2Day: DailyMetric? + @State private var cachedSkinTempDay: DailyMetric? /// The Charge hero's resolved state (#543 carry + the honest label), resolved ONCE in load() alongside /// the other caches. It composes `TodayView.lastScoredRecoveryDay`, which is O(days) — exactly the scan /// this cache exists to keep out of body. Never resolved in body. @@ -144,9 +151,25 @@ struct LiquidTodayView: View { /// The prior-day vitals carry (see `cachedVitalsDay`), read O(1) from the cache. Non-nil only at /// offset 0 (today); a navigated past day carries nothing (its own row is the whole story). private var vitalsDay: DailyMetric? { cachedVitalsDay } + /// The per-field SpO₂ carry (see `cachedSpo2Day`), read O(1) from the cache. + private var spo2Day: DailyMetric? { cachedSpo2Day } + /// The per-field skin-temperature carry (see `cachedSkinTempDay`), read O(1) from the cache. + private var skinTempDay: DailyMetric? { cachedSkinTempDay } /// The Charge hero's resolved state (see `cachedChargeDisplay`), read O(1) from the cache. private var chargeDisplay: ChargeDisplay { cachedChargeDisplay } + /// The value a per-field vitals tile (Blood Oxygen, Skin Temp) draws: today's own reading wins, else the + /// whole-row vitals carry's copy of the field, else the PER-FIELD carry — the freshest strictly-prior row + /// that actually HAS the field. The whole-row carry (`vitalsDay`) gates on HRV / resting-HR / respiratory, + /// so it lands on a computed "-noop" row whose `spo2Pct` / `skinTempDevC` is engine-written nil while an + /// older imported row holds a real reading; the per-field carry (`spo2Day` / `skinTempDay`) resolves that. + /// Pure so the today-first → whole-row → per-field ordering is pinned without a live view, mirroring + /// classic `TodayView.carriedVital`'s fallthrough and the Android per-field read. Named so all three tiles + /// (Key-Metrics SpO₂ + the two "Your cards") share one source of truth and can't drift. + static func perFieldVital(today: Double?, vitalsCarry: Double?, fieldCarry: Double?) -> Double? { + today ?? vitalsCarry ?? fieldCarry + } + /// The actual O(days) resolution. Offset 0 prefers live repo.today; past offsets look up. Run ONCE /// per data/day change from load(), never from body. private func resolveDisplayDay() -> DailyMetric? { @@ -245,6 +268,14 @@ struct LiquidTodayView: View { VStack(alignment: .leading, spacing: 12) { scene + // The illness early-warning banner, dropped in the liquid Home rewrite while Liquid became + // the default Today — so a raised/already-unwell signal had no surface on the screen the + // user actually opens (only the Health tab's HeadsUpCard and the once-a-day notification). + // Restored as the SAME leaf the classic TodayView renders directly under its top bar, so + // it sits above the reorderable block and can't be reordered out of sight. Renders nothing + // when there's no alert; it owns its own AppModel observation, so the ~1 Hz HR stream + // re-renders only the banner, not this body. + HealthAlertBanner() // #105: the live "workout in progress" card, dropped in the liquid Home rewrite. Restored // here as the SAME leaf the classic TodayView renders (and Android's WorkoutInProgressCard), // pinned above the reorderable block so an active manual workout is immediately visible @@ -643,12 +674,18 @@ struct LiquidTodayView: View { cardLink(.metricSourced(key: stepsDetailKey, source: stepsDetailSource), title: card.title, sub: card.subtitle, value: stepsText, tint: StrandPalette.metricCyan, frac: fracOver(stepCount, 10000)) case .bloodOxygen: - // Not wired to a real read yet — render EMPTY (not half-full) so it doesn't imply a reading. + // Per-field carry (today → whole-row vitals carry → the last row that HAS a %), matching the + // Key-Metrics tile + classic dashboardCardValue so the surfaces agree instead of a bare "–". + let spo2 = Self.perFieldVital(today: displayDay?.spo2Pct, vitalsCarry: vitalsDay?.spo2Pct, fieldCarry: spo2Day?.spo2Pct) cardLink(.metric("spo2"), title: card.title, sub: card.subtitle, - value: "–", tint: StrandPalette.metricCyan, frac: nil) + value: spo2.map { String(format: "%.0f%%", $0) } ?? "–", tint: StrandPalette.metricCyan, + frac: fracOver(spo2, 100)) case .skinTemp: + // Per-field carry, matching classic dashboardCardValue. A signed deviation from baseline (°C), + // shown signed; a deviation has no natural 0–1 fill, so the vessel stays empty (as classic does). + let skin = Self.perFieldVital(today: displayDay?.skinTempDevC, vitalsCarry: vitalsDay?.skinTempDevC, fieldCarry: skinTempDay?.skinTempDevC) cardLink(.metric("skin_temp"), title: card.title, sub: card.subtitle, - value: "–", tint: StrandPalette.metricAmber, frac: nil) + value: skin.map { String(format: "%+.1f°", $0) } ?? "–", tint: StrandPalette.metricAmber, frac: nil) case .calories: // #616: show the resolved imported-first value and route to the matching detail source, like // the Steps card — was a "–" placeholder wired to the imported-only detail. @@ -902,7 +939,9 @@ struct LiquidTodayView: View { case .restingHr: ktile(String(localized: "Rest HR"), intText(rhr), "bpm", StrandPalette.metricRose, fracOver(rhr, 100), key: "rhr") case .bloodOxygen: - let spo2 = displayDay?.spo2Pct ?? vitalsDay?.spo2Pct + // Per-field carry: today → whole-row vitals carry → the last row that actually HAS a % (computed + // "-noop" rows write spo2Pct = nil), so the tile doesn't blank at the rollover. Matches classic. + let spo2 = Self.perFieldVital(today: displayDay?.spo2Pct, vitalsCarry: vitalsDay?.spo2Pct, fieldCarry: spo2Day?.spo2Pct) ktile(String(localized: "Blood Oxygen"), intText(spo2), "%", StrandPalette.metricCyan, fracOver(spo2, 100), key: "spo2") case .respiratory: let resp = displayDay?.respRateBpm ?? vitalsDay?.respRateBpm @@ -1029,6 +1068,7 @@ struct LiquidTodayView: View { } LiquidStrapBatteryRow() LiquidSyncStatusRow() + LiquidBackfillProgressRow() } } } @@ -1073,6 +1113,11 @@ struct LiquidTodayView: View { // echo today's still-forming row; only on today (a past day's own row is the whole story). let tkey = cachedDisplayDay?.day ?? selectedDayKey cachedVitalsDay = (selectedDayOffset == 0) ? Repository.lastVitalsDay(days: repo.days, todayKey: tkey) : nil + // PER-FIELD SpO₂ / skin-temp carries — the two fields lastVitalsDay's predicate doesn't check, so the + // whole-row carry above can land on a null value while an older imported row holds a real one. Same + // today-key bound; only on today. Resolved here (never in body) so the tiles stay O(1). + cachedSpo2Day = (selectedDayOffset == 0) ? Repository.lastSpo2Day(days: repo.days, todayKey: tkey) : nil + cachedSkinTempDay = (selectedDayOffset == 0) ? Repository.lastSkinTempDay(days: repo.days, todayKey: tkey) : nil // Charge carry (#543) + the honest label, resolved here for the same reason as the two above: the // selector below scans repo.days. Calibration nights come from the SAME `RecoveryScorer` helper the // classic Today reads, so the two screens agree on when a wearer is genuinely mid-calibration @@ -1831,7 +1876,9 @@ private struct LiquidBatteryButton: View { @EnvironmentObject var live: LiveState @EnvironmentObject var router: NavRouter private var display: LiquidTodayView.StrapBatteryDisplay { - .resolve(connected: live.connected, batteryPct: live.batteryPct, charging: live.charging) + // `chargingEffective`, not the raw flag — the 5.0 charge bit is unverified and was observed reading + // 0 on a charging strap; a rising SoC is the witness that can't lie. See StrapChargeInference. + .resolve(connected: live.connected, batteryPct: live.batteryPct, charging: live.chargingEffective) } var body: some View { Button { router.openDevices() } label: { @@ -1947,10 +1994,11 @@ private struct LiquidSyncChip: View { /// dropped the "~X days left" runtime estimate from the row directly above this one. /// /// Deliberately scoped to what LiveState can honestly answer: THAT a drain is running, how many chunks -/// it has pulled, and when one last completed. It does NOT yet say "~15h behind" — that needs the -/// persisted data frontier (max HR ts) compared against `strapRange.newestUnix`, and the frontier is a -/// Repository read that LiveState does not carry. That remains open in B1. Kept here in the Data Sources -/// card as the detailed view; `LiquidSyncChip` above is the header's ambient at-a-glance signal. +/// it has pulled, and when one last completed. The remaining half of B1 — "~15h behind", which needs the +/// persisted data frontier (max HR ts) compared against `strapRange.newestUnix` — is answered by +/// `LiquidBackfillProgressRow` directly below, which BLEManager now feeds via `LiveState.setDataFrontier`. +/// Kept here in the Data Sources card as the detailed view; `LiquidSyncChip` above is the header's +/// ambient at-a-glance signal. private struct LiquidSyncStatusRow: View { @EnvironmentObject var live: LiveState var body: some View { @@ -1981,6 +2029,50 @@ private struct LiquidSyncStatusRow: View { } } +/// Strap-history recovery, inside the Data Sources card. Owns LiveState; display-only. +/// +/// The gap this closes: the app already said "Syncing strap history…" WHILE a session ran, but said nothing +/// about how far behind the strap was — and nothing at all between sessions, so an idle app looked finished +/// when it was actually 18 hours behind and waiting out a 15-minute floor. A user reasonably read that +/// silence as "my data is gone". Both states are now stated outright, and the numbers come from the pure +/// `BackfillProgress` so this row can never disagree with the drain's own idea of "behind". +private struct LiquidBackfillProgressRow: View { + @EnvironmentObject var live: LiveState + var body: some View { + let progress = live.backfillProgress + if let behind = progress.behindLabel, let frontier = progress.frontierUnix { + HStack { + VStack(alignment: .leading, spacing: 2) { + // `String(localized:)`, matching `LiquidSyncStatusRow` directly above — a bare + // `Text("literal")` here relies on Xcode's build-time extraction and left these three + // strings out of the String Catalog entirely (caught by Tools/i18n_audit.py). + Text(isRecovering(progress) + ? String(localized: "Recovering strap history") + : String(localized: "Strap history pending")) + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + // The anchor that answers "is my data lost?" — we HAVE your data up to this instant. + Text(String(localized: "Have it through \(recoveredThrough(frontier))")) + .font(StrandFont.caption).foregroundStyle(StrandPalette.textTertiary) + } + Spacer() + // Same shape as the Devices card's pill: the pure resolver hands over a plain label, the + // view localizes it. Pulsing IS the active/idle indicator — draining vs merely behind. + StatePill(LocalizedStringKey(behind), tone: .accent, pulsing: isRecovering(progress)) + } + } + } + private func isRecovering(_ p: BackfillProgress) -> Bool { + if case .recovering = p { return true } + return false + } + /// "July 14, 14:31" — a wall-clock instant the user can place against their own day. + private func recoveredThrough(_ unix: Int) -> String { + let f = DateFormatter() + f.dateFormat = "MMM d, HH:mm" + return f.string(from: Date(timeIntervalSince1970: TimeInterval(unix))) + } +} + /// The strap-battery readout inside the Data Sources card. Owns LiveState; display-only. private struct LiquidStrapBatteryRow: View { @EnvironmentObject var live: LiveState @@ -2000,7 +2092,10 @@ private struct LiquidStrapBatteryRow: View { /// Mac / Android pill and the classic Today badge. private func batteryText(pct: Double) -> String { let base = "\(Int(pct.rounded()))%" - if live.charging == true { return "\(base) · Charging" } + // `chargingEffective`, not the raw flag: the 5.0 charge bit is unverified and has been seen reading + // 0 on a strap that was demonstrably on its puck, which left this row showing a DISCHARGE estimate + // ("31% · ~2 days left") to a user who was watching it charge. A rising SoC overrides the bit. + if live.chargingEffective == true { return "\(base) · Charging" } if let est = estimateText { return "\(base) · \(est)" } return base } @@ -2009,7 +2104,7 @@ private struct LiquidStrapBatteryRow: View { /// Reproduced verbatim from `TodayView.estimateText`: under 48 h show hours, at two days or more round to /// days; nil (no banked discharge yet, or charging) hides it, so the row only ever shows an estimate we trust. private var estimateText: String? { - guard live.charging != true, let est = live.batteryEstimate else { return nil } + guard live.chargingEffective != true, let est = live.batteryEstimate else { return nil } let hours = est.hoursRemaining guard hours.isFinite, hours > 0 else { return nil } if hours < 48 { diff --git a/Strand/Oura/OuraSecrets.example.xcconfig b/Strand/Oura/OuraSecrets.example.xcconfig index 4151f469e7..b87ed68cdb 100644 --- a/Strand/Oura/OuraSecrets.example.xcconfig +++ b/Strand/Oura/OuraSecrets.example.xcconfig @@ -12,3 +12,16 @@ OURA_CLIENT_SECRET = your_client_secret_here // expansion $() below breaks up the "//" token without changing the resolved string, which is // still exactly noop://oura/callback at build time. Keep this trick if you edit the value. OURA_REDIRECT_URI = noop:/$()/oura/callback + +// Optional second personal-fork lane: add CLOUD_SYNC to the same line to compile the +// Strand/CloudSync module (upload this device's own backup, pull confirmed noop-cloud edits, apply +// them, and re-export into Apple Health). +// SWIFT_ACTIVE_COMPILATION_CONDITIONS = $(inherited) OURA_CLOUD_IMPORT CLOUD_SYNC +// +// Zero-touch (Phase 3.5): with CLOUD_SYNC set, these two keys pre-configure the lane at build time — +// no manual URL/token entry in the app, no manual tap to start syncing (an on-launch catch-up fires +// automatically at most once per ~20h). Read at runtime via CloudSyncSettings.effectiveURL/ +// effectiveToken; absent/blank leaves the lane on manual Keychain configuration only (Data Sources → +// Cloud Sync → Override…). A Keychain value saved through that manual override always wins over these. +// CLOUDSYNC_URL = https://your-noop-cloud-server.example.com +// CLOUDSYNC_TOKEN = your_read_write_token_here diff --git a/Strand/Resources/Info.plist b/Strand/Resources/Info.plist index 8a2f2a392f..d3e20eae45 100644 --- a/Strand/Resources/Info.plist +++ b/Strand/Resources/Info.plist @@ -29,6 +29,10 @@ CFBundleVersion 198 + CLOUDSYNC_TOKEN + $(CLOUDSYNC_TOKEN) + CLOUDSYNC_URL + $(CLOUDSYNC_URL) LSApplicationCategoryType public.app-category.healthcare-fitness LSMinimumSystemVersion diff --git a/Strand/Resources/Localizable.xcstrings b/Strand/Resources/Localizable.xcstrings index e3f2f9f375..58112f1ffc 100644 --- a/Strand/Resources/Localizable.xcstrings +++ b/Strand/Resources/Localizable.xcstrings @@ -173360,6 +173360,838 @@ } } } + }, + "Recovering strap history": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Strap-Verlauf wird wiederhergestellt" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Recuperando el historial de la pulsera" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Récupération de l'historique du bracelet" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Recupero della cronologia della fascia" + } + }, + "pt-PT": { + "stringUnit": { + "state": "needs_review", + "value": "A recuperar o histórico da pulseira" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Восстановление истории браслета" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "正在恢复手环历史数据" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "正在復原手環歷史資料" + } + } + } + }, + "Strap history pending": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Strap-Verlauf ausstehend" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Historial de la pulsera pendiente" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Historique du bracelet en attente" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Cronologia della fascia in sospeso" + } + }, + "pt-PT": { + "stringUnit": { + "state": "needs_review", + "value": "Histórico da pulseira pendente" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "История браслета ожидает" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "手环历史数据待同步" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "手環歷史資料待同步" + } + } + } + }, + "Have it through %@": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Vorhanden bis %@" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Datos hasta %@" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Données jusqu'à %@" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Dati fino a %@" + } + }, + "pt-PT": { + "stringUnit": { + "state": "needs_review", + "value": "Dados até %@" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Данные по %@" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "已获取至 %@" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "已取得至 %@" + } + } + } + }, + "Reduce motion in NOOP": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Bewegung in NOOP reduzieren" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Reducir movimiento en NOOP" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Réduire les animations dans NOOP" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Riduci movimento in NOOP" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Reduzir movimento no NOOP" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Уменьшить анимацию в NOOP" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "减少 NOOP 中的动态效果" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "減少 NOOP 中的動態效果" + } + } + } + }, + "Holds the liquid gauges, the sky and the tilt response still, and turns off the motion sensor that drives them. Saves battery. Low Power Mode and the system Reduce Motion setting already do this.": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Hält die Flüssigkeitsanzeigen, den Himmel und die Neigungsreaktion still und schaltet den Bewegungssensor ab, der sie antreibt. Spart Akku. Der Stromsparmodus und die Systemeinstellung „Bewegung reduzieren“ tun dies bereits." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Mantiene quietos los medidores líquidos, el cielo y la respuesta a la inclinación, y apaga el sensor de movimiento que los impulsa. Ahorra batería. El modo de bajo consumo y el ajuste del sistema «Reducir movimiento» ya hacen esto." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Fige les jauges liquides, le ciel et la réaction à l’inclinaison, et coupe le capteur de mouvement qui les anime. Économise la batterie. Le mode économie d’énergie et le réglage système « Réduire les animations » font déjà cela." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Blocca gli indicatori liquidi, il cielo e la risposta all’inclinazione e spegne il sensore di movimento che li anima. Risparmia batteria. La modalità Risparmio energetico e l’impostazione di sistema «Riduci movimento» lo fanno già." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Mantém parados os medidores líquidos, o céu e a resposta à inclinação, e desliga o sensor de movimento que os anima. Poupa bateria. O Modo de Baixo Consumo e a definição de sistema «Reduzir movimento» já fazem isto." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Останавливает жидкостные шкалы, небо и отклик на наклон и отключает датчик движения, который их питает. Экономит заряд батареи. Режим энергосбережения и системная настройка «Уменьшение движения» уже делают это." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "让液态仪表、天空和倾斜响应保持静止，并关闭驱动它们的运动传感器。省电。低电量模式和系统的“减弱动态效果”设置已经会这样做。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "讓液態儀表、天空和傾斜回應保持靜止，並關閉驅動它們的動作感測器。省電。低耗電模式和系統的「減少動態效果」設定已經會這樣做。" + } + } + } + }, + "Clear the R22 flags on your strap?": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "R22-Flags auf deinem Strap löschen?" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "¿Borrar las marcas R22 de tu pulsera?" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Effacer les indicateurs R22 sur votre bracelet ?" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Cancellare i flag R22 sulla fascia?" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Limpar as marcas R22 na sua bracelete?" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Очистить флаги R22 на браслете?" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "要清除手环上的 R22 标记吗？" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "要清除手環上的 R22 標記嗎？" + } + } + } + }, + "Clear flags on strap": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Flags auf dem Strap löschen" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Borrar marcas en la pulsera" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Effacer les indicateurs" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Cancella i flag sulla fascia" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Limpar marcas na bracelete" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Очистить флаги на браслете" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "清除手环上的标记" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "清除手環上的標記" + } + } + } + }, + "Just stop sending": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Nur nicht mehr senden" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Solo dejar de enviar" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Cesser seulement d’envoyer" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Smetti solo di inviare" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Apenas parar de enviar" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Просто перестать отправлять" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "仅停止发送" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "僅停止傳送" + } + } + } + }, + "Turning this switch off only stops NOOP sending the unlock. The flags it already wrote stay on the strap until something clears them. NOOP can write the off value to all 16 now and read each one back so you can see what the strap actually stores. Needs the strap connected and bonded.": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Diesen Schalter auszuschalten stoppt nur, dass NOOP die Freischaltung sendet. Die bereits geschriebenen Flags bleiben auf dem Strap, bis sie etwas löscht. NOOP kann jetzt den Aus-Wert in alle 16 schreiben und jedes zurücklesen, damit du siehst, was der Strap tatsächlich speichert. Der Strap muss verbunden und gekoppelt sein." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Apagar este interruptor solo impide que NOOP envíe el desbloqueo. Las marcas ya escritas siguen en la pulsera hasta que algo las borre. NOOP puede escribir ahora el valor de apagado en las 16 y releer cada una para que veas lo que la pulsera guarda realmente. Requiere la pulsera conectada y vinculada." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Désactiver cet interrupteur empêche seulement NOOP d’envoyer le déverrouillage. Les indicateurs déjà écrits restent sur le bracelet jusqu’à ce que quelque chose les efface. NOOP peut maintenant écrire la valeur d’arrêt dans les 16 et relire chacun d’eux pour que vous voyiez ce que le bracelet enregistre réellement. Nécessite le bracelet connecté et appairé." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Disattivare questo interruttore impedisce solo a NOOP di inviare lo sblocco. I flag già scritti restano sulla fascia finché qualcosa non li cancella. NOOP può scrivere ora il valore di spegnimento su tutti e 16 e rileggerli uno a uno, così vedi che cosa la fascia memorizza davvero. Richiede la fascia connessa e associata." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Desligar este interruptor apenas impede o NOOP de enviar o desbloqueio. As marcas já escritas permanecem na bracelete até que algo as limpe. O NOOP pode agora escrever o valor de desativação nas 16 e reler cada uma para que veja o que a bracelete guarda realmente. Necessita da bracelete ligada e emparelhada." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Выключение этого переключателя лишь останавливает отправку разблокировки из NOOP. Уже записанные флаги остаются на браслете, пока что-нибудь их не очистит. NOOP может сейчас записать значение выключения во все 16 и прочитать каждый обратно, чтобы вы увидели, что браслет действительно хранит. Требуется подключённый и сопряжённый браслет." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "关闭此开关只会让 NOOP 停止发送解锁序列。已写入的标记会留在手环上，直到有东西清除它们。NOOP 现在可以向全部 16 个标记写入关闭值并逐一读回，让你看到手环实际存储的内容。需要手环已连接并完成配对。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "關閉此開關只會讓 NOOP 停止傳送解鎖序列。已寫入的標記會留在手環上，直到有東西清除它們。NOOP 現在可以向全部 16 個標記寫入關閉值並逐一讀回，讓你看到手環實際儲存的內容。需要手環已連線並完成配對。" + } + } + } + }, + "WHOOP 5/MG straps hand a fresh app only live heart rate. The official app switches on the deeper streams (high-rate HR + motion + history) by writing a set of feature flags, a sequence two independent projects have documented. With this on, the button below sends that exact sequence to your strap. Unlike everything else here it does write to the strap — and it is reversible: “Turn deep data back off” writes the off value to the same flags and then reads every one of them back, so you see what the strap actually stores rather than just that it acked. Experimental: it may do nothing on your firmware. iPhone/Android only. A Mac can't write to a 5/MG.": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "WHOOP 5/MG-Straps geben einer neuen App nur den Live-Puls. Die offizielle App schaltet die tieferen Datenströme (hochfrequenter Puls + Bewegung + Verlauf) frei, indem sie eine Reihe von Feature-Flags schreibt — eine Sequenz, die zwei unabhängige Projekte dokumentiert haben. Wenn dies aktiv ist, sendet die Schaltfläche unten genau diese Sequenz an deinen Strap. Anders als alles andere hier schreibt sie auf den Strap — und sie ist umkehrbar: „Tiefe Daten wieder ausschalten“ schreibt den Aus-Wert in dieselben Flags und liest anschließend jedes einzeln zurück, sodass du siehst, was der Strap tatsächlich speichert, statt nur, dass er bestätigt hat. Experimentell: Auf deiner Firmware bewirkt es womöglich nichts. Nur iPhone/Android. Ein Mac kann nicht auf einen 5/MG schreiben." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Las pulseras WHOOP 5/MG solo entregan la frecuencia cardiaca en vivo a una app nueva. La app oficial activa los flujos más profundos (FC de alta frecuencia + movimiento + historial) escribiendo un conjunto de marcas de función, una secuencia documentada por dos proyectos independientes. Con esto activado, el botón de abajo envía esa secuencia exacta a tu pulsera. A diferencia de todo lo demás aquí, sí escribe en la pulsera, y es reversible: «Volver a desactivar los datos profundos» escribe el valor de apagado en esas mismas marcas y luego relee cada una, de modo que veas lo que la pulsera guarda realmente y no solo que confirmó. Experimental: puede que no haga nada en tu firmware. Solo iPhone/Android. Un Mac no puede escribir en una 5/MG." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Les bracelets WHOOP 5/MG ne transmettent à une nouvelle app que la fréquence cardiaque en direct. L’app officielle active les flux plus détaillés (FC haute fréquence + mouvement + historique) en écrivant un ensemble d’indicateurs de fonctionnalité, une séquence documentée par deux projets indépendants. Avec cette option activée, le bouton ci-dessous envoie exactement cette séquence à votre bracelet. Contrairement à tout le reste ici, elle écrit bien sur le bracelet — et elle est réversible : « Désactiver à nouveau les données détaillées » écrit la valeur d’arrêt dans ces mêmes indicateurs puis relit chacun d’eux, pour que vous voyiez ce que le bracelet enregistre réellement et pas seulement qu’il a accusé réception. Expérimental : cela peut ne rien faire sur votre firmware. iPhone/Android uniquement. Un Mac ne peut pas écrire sur un 5/MG." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Le fasce WHOOP 5/MG offrono a una nuova app solo la frequenza cardiaca in tempo reale. L’app ufficiale attiva i flussi più profondi (FC ad alta frequenza + movimento + cronologia) scrivendo una serie di flag di funzionalità, una sequenza documentata da due progetti indipendenti. Con questa opzione attiva, il pulsante qui sotto invia esattamente quella sequenza alla tua fascia. A differenza di tutto il resto qui, scrive davvero sulla fascia — ed è reversibile: «Disattiva di nuovo i dati approfonditi» scrive il valore di spegnimento sugli stessi flag e poi li rilegge tutti, così vedi che cosa la fascia memorizza davvero e non solo che ha confermato. Sperimentale: sul tuo firmware potrebbe non fare nulla. Solo iPhone/Android. Un Mac non può scrivere su un 5/MG." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "As braceletes WHOOP 5/MG entregam a uma app nova apenas a frequência cardíaca em direto. A app oficial ativa os fluxos mais profundos (FC de alta frequência + movimento + histórico) escrevendo um conjunto de marcas de funcionalidade, uma sequência documentada por dois projetos independentes. Com isto ativado, o botão abaixo envia essa sequência exata para a sua bracelete. Ao contrário de tudo o resto aqui, escreve mesmo na bracelete — e é reversível: «Voltar a desativar os dados profundos» escreve o valor de desativação nessas mesmas marcas e depois relê cada uma, para que veja o que a bracelete guarda realmente e não apenas que confirmou. Experimental: pode não fazer nada no seu firmware. Apenas iPhone/Android. Um Mac não consegue escrever numa 5/MG." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Браслеты WHOOP 5/MG передают новому приложению только пульс в реальном времени. Официальное приложение включает более подробные потоки (частый пульс + движение + история), записывая набор функциональных флагов — последовательность, задокументированную двумя независимыми проектами. При включённой опции кнопка ниже отправляет браслету именно эту последовательность. В отличие от всего остального здесь, она действительно пишет на браслет — и это обратимо: «Снова отключить подробные данные» записывает значение выключения в те же флаги, а затем читает каждый обратно, чтобы вы увидели, что браслет действительно хранит, а не только что он подтвердил. Экспериментально: на вашей прошивке может не дать эффекта. Только iPhone/Android. Mac не может писать на 5/MG." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "WHOOP 5/MG 手环只向新应用提供实时心率。官方应用通过写入一组功能标记来开启更深层的数据流（高频心率 + 运动 + 历史），这一序列已由两个独立项目记录。启用后，下方按钮会向你的手环发送完全相同的序列。与这里其他项不同，它确实会写入手环——而且可以撤销：“重新关闭深层数据”会向同一批标记写入关闭值，然后逐一读回，让你看到手环实际存储的内容，而不仅仅是它已确认。实验性功能：在你的固件上可能毫无作用。仅限 iPhone/Android。Mac 无法写入 5/MG。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "WHOOP 5/MG 手環只向新應用程式提供即時心率。官方應用程式透過寫入一組功能標記來開啟更深層的資料流（高頻心率 + 動作 + 歷史），這一序列已由兩個獨立專案記錄。啟用後，下方按鈕會向你的手環傳送完全相同的序列。與這裡其他項不同，它確實會寫入手環——而且可以復原：「重新關閉深層資料」會向同一批標記寫入關閉值，然後逐一讀回，讓你看到手環實際儲存的內容，而不僅僅是它已確認。實驗性功能：在你的韌體上可能毫無作用。僅限 iPhone/Android。Mac 無法寫入 5/MG。" + } + } + } + }, + "Strap accepted all %lld R22 flags": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Strap hat alle %lld R22-Flags akzeptiert" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "La pulsera aceptó las %lld marcas R22" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Le bracelet a accepté les %lld indicateurs R22" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "La fascia ha accettato tutti i %lld flag R22" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "A bracelete aceitou as %lld marcas R22" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Браслет принял все %lld флагов R22" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "手环已接受全部 %lld 个 R22 标记" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "手環已接受全部 %lld 個 R22 標記" + } + } + } + }, + "Strap accepted %lld/%lld R22 flags…": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Strap hat %lld/%lld R22-Flags akzeptiert…" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "La pulsera aceptó %lld/%lld marcas R22…" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Le bracelet a accepté %lld/%lld indicateurs R22…" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "La fascia ha accettato %lld/%lld flag R22…" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "A bracelete aceitou %lld/%lld marcas R22…" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Браслет принял %lld/%lld флагов R22…" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "手环已接受 %lld/%lld 个 R22 标记…" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "手環已接受 %lld/%lld 個 R22 標記…" + } + } + } + }, + "Clearing R22 flags and reading each one back…": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "R22-Flags werden gelöscht und einzeln zurückgelesen…" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Borrando las marcas R22 y releyendo cada una…" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Effacement des indicateurs R22 et relecture de chacun…" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Cancellazione dei flag R22 e rilettura di ciascuno…" + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "A limpar as marcas R22 e a reler cada uma…" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Очистка флагов R22 и обратное чтение каждого…" + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "正在清除 R22 标记并逐一读回…" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "正在清除 R22 標記並逐一讀回…" + } + } + } + }, + "Turning deep data back off needs an iPhone or Android. A Mac can't form the encrypted bond a 5/MG requires.": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Tiefe Daten wieder auszuschalten benötigt ein iPhone oder Android. Ein Mac kann die verschlüsselte Kopplung, die ein 5/MG verlangt, nicht herstellen." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Volver a desactivar los datos profundos requiere un iPhone o Android. Un Mac no puede formar el vínculo cifrado que exige una 5/MG." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Désactiver à nouveau les données détaillées nécessite un iPhone ou un Android. Un Mac ne peut pas former l’appairage chiffré qu’exige un 5/MG." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Riattivare lo spegnimento dei dati approfonditi richiede un iPhone o un Android. Un Mac non può creare l’associazione cifrata richiesta da un 5/MG." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Voltar a desativar os dados profundos requer um iPhone ou Android. Um Mac não consegue formar a ligação encriptada que uma 5/MG exige." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Чтобы снова отключить подробные данные, нужен iPhone или Android. Mac не может установить шифрованное сопряжение, которого требует 5/MG." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "重新关闭深层数据需要 iPhone 或 Android。Mac 无法建立 5/MG 所需的加密配对。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "重新關閉深層資料需要 iPhone 或 Android。Mac 無法建立 5/MG 所需的加密配對。" + } + } + } + }, + "Needs the full encrypted bond: close the official WHOOP app and pair the strap to NOOP first (a live-HR-only link can't carry the write).": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Benötigt die vollständig verschlüsselte Kopplung: Schließe zuerst die offizielle WHOOP-App und koppele den Strap mit NOOP (eine Verbindung nur für den Live-Puls kann den Schreibvorgang nicht übertragen)." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Necesita el vínculo cifrado completo: cierra primero la app oficial de WHOOP y empareja la pulsera con NOOP (un enlace solo de FC en vivo no puede transportar la escritura)." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Nécessite l’appairage chiffré complet : fermez d’abord l’app WHOOP officielle et appairez le bracelet à NOOP (une liaison limitée à la FC en direct ne peut pas transporter l’écriture)." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Richiede l’associazione cifrata completa: chiudi prima l’app WHOOP ufficiale e associa la fascia a NOOP (un collegamento solo per la FC in tempo reale non può trasportare la scrittura)." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Necessita da ligação encriptada completa: feche primeiro a app oficial da WHOOP e emparelhe a bracelete com o NOOP (uma ligação apenas de FC em direto não consegue transportar a escrita)." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Требуется полное шифрованное сопряжение: сначала закройте официальное приложение WHOOP и сопрягите браслет с NOOP (канал только для пульса в реальном времени не может передать запись)." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "需要完整的加密配对：请先关闭官方 WHOOP 应用并将手环与 NOOP 配对（仅传输实时心率的连接无法承载写入）。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "需要完整的加密配對：請先關閉官方 WHOOP 應用程式並將手環與 NOOP 配對（僅傳輸即時心率的連線無法承載寫入）。" + } + } + } + }, + "Writes the off value to all 16 flags and reads each one back, so you see what the strap stores rather than just that it acked. The off value is inferred from the flag NOOP already turns off this way for Garmin broadcast — it has never been seen on an R22 flag, so NOOP tries one flag first and stops if the strap refuses it. Clearing the flags is not the same as watching the deep records stop: check that by syncing afterwards.": { + "localizations": { + "de": { + "stringUnit": { + "state": "translated", + "value": "Schreibt den Aus-Wert in alle 16 Flags und liest jedes einzeln zurück, sodass du siehst, was der Strap tatsächlich speichert — nicht nur, dass er bestätigt hat. Der Aus-Wert ist von dem Flag abgeleitet, das NOOP für die Garmin-Übertragung bereits so ausschaltet; auf einem R22-Flag wurde er noch nie beobachtet. Deshalb testet NOOP zuerst ein einzelnes Flag und bricht ab, wenn der Strap es ablehnt. Gelöschte Flags sind nicht dasselbe wie tatsächlich versiegende Tiefendaten: prüfe das mit einer anschließenden Synchronisierung." + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Escribe el valor de apagado en las 16 marcas y vuelve a leer cada una, para que veas lo que la pulsera guarda en vez de solo que confirmó. El valor de apagado se deduce de la marca que NOOP ya desactiva así para la difusión Garmin; nunca se ha visto en una marca R22, por eso NOOP prueba primero con una sola y se detiene si la pulsera la rechaza. Borrar las marcas no es lo mismo que ver parar los registros profundos: compruébalo sincronizando después." + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Écrit la valeur d’arrêt dans les 16 indicateurs et relit chacun d’eux, pour que vous voyiez ce que le bracelet enregistre réellement et pas seulement qu’il a accusé réception. La valeur d’arrêt est déduite de l’indicateur que NOOP désactive déjà ainsi pour la diffusion Garmin ; elle n’a jamais été observée sur un indicateur R22, c’est pourquoi NOOP en teste un seul d’abord et s’arrête si le bracelet le refuse. Effacer les indicateurs n’équivaut pas à voir cesser les enregistrements détaillés : vérifiez-le en synchronisant ensuite." + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Scrive il valore di spegnimento su tutti e 16 i flag e li rilegge uno a uno, così vedi che cosa la fascia memorizza e non solo che ha confermato. Il valore di spegnimento è dedotto dal flag che NOOP già disattiva così per la trasmissione Garmin: non è mai stato osservato su un flag R22, perciò NOOP ne prova prima uno solo e si ferma se la fascia lo rifiuta. Cancellare i flag non equivale a vedere fermarsi le registrazioni approfondite: verificalo sincronizzando dopo." + } + }, + "pt-PT": { + "stringUnit": { + "state": "translated", + "value": "Escreve o valor de desativação nas 16 marcas e lê cada uma de volta, para que veja o que a pulseira guarda e não apenas que confirmou. O valor de desativação é inferido da marca que o NOOP já desliga assim para a transmissão Garmin — nunca foi visto numa marca R22, por isso o NOOP testa primeiro uma só e pára se a pulseira a recusar. Limpar as marcas não é o mesmo que ver os registos profundos pararem: confirme isso sincronizando depois." + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Записывает значение выключения во все 16 флагов и читает каждый обратно, чтобы вы видели, что браслет хранит, а не только что он подтвердил. Значение выключения выведено из флага, который NOOP уже так отключает для трансляции на Garmin: на флаге R22 оно никогда не наблюдалось, поэтому NOOP сначала пробует один флаг и останавливается, если браслет его отвергнет. Очистка флагов — не то же самое, что увидеть остановку подробных записей: проверьте это последующей синхронизацией." + } + }, + "zh-Hans": { + "stringUnit": { + "state": "translated", + "value": "向全部 16 个标记写入关闭值，并逐一读回，让你看到手环实际存储的内容，而不仅仅是它已确认。该关闭值是从 NOOP 已用同样方式关闭的 Garmin 广播标记推断出来的——在 R22 标记上从未被观察到，因此 NOOP 会先试一个标记，若手环拒绝就停止。清除标记与看到深层记录停止并不相同：之后同步一次来确认。" + } + }, + "zh-Hant": { + "stringUnit": { + "state": "translated", + "value": "向全部 16 個標記寫入關閉值，並逐一讀回，讓你看到手環實際儲存的內容，而不僅僅是它已確認。該關閉值是從 NOOP 已用同樣方式關閉的 Garmin 廣播標記推斷出來的——在 R22 標記上從未被觀察到，因此 NOOP 會先試一個標記，若手環拒絕就停止。清除標記與看到深層記錄停止並不相同：之後同步一次來確認。" + } + } + } } }, "version": "1.0" diff --git a/Strand/Screens/CrossDeviceHRVView.swift b/Strand/Screens/CrossDeviceHRVView.swift new file mode 100644 index 0000000000..8cc6ab27ae --- /dev/null +++ b/Strand/Screens/CrossDeviceHRVView.swift @@ -0,0 +1,438 @@ +import SwiftUI +import Charts +import Foundation +import StrandDesign +import StrandAnalytics +import WhoopStore + +// MARK: - Cross-Device HRV +// +// A wearer who has worn more than one HRV sensor over time (e.g. years of Oura ring history +// continuing onto a WHOOP strap) has two INCOMPATIBLE RMSSD scales with no overlap night to +// calibrate against — Oura reads ~120-155 ms, WHOOP ~72-112 ms. Averaging them is dishonest; +// showing only the current strap throws away the long trend the wearer cares about. +// +// This screen resolves that for a DISPLAY/trend lens (daily 0-3 scores stay strictly per-device +// via deviceEraEpoch): every night is measured against its OWN device-era's robust log-scale +// centre, so the era-relative trend flows continuously across the switch — you can see whether +// HRV was trending up or down THROUGH the hardware change — while a second, raw chart keeps each +// sensor's true milliseconds. All math is the pure, DB-free `CrossDeviceHRVTrend` engine. + +struct CrossDeviceHRVView: View { + @EnvironmentObject var repo: Repository + + /// One night, parsed to a `Date` once so the chart never re-parses "yyyy-MM-dd" on body re-eval. + private struct Row: Identifiable { + let id: String // the day key — stable + unique per night + let date: Date + let raw: Double // absolute ms as the device reported it + let pct: Double // % above/below this device-era's own centre + let brand: String // "oura" / "whoop" / … + let isEraStart: Bool // first night of a non-first era = a device switch + } + + @State private var eras: [CrossDeviceHRVTrend.Era] = [] + @State private var rows: [Row] = [] // all nights, one continuous series (hero) + @State private var segments: [[Row]] = [] // nights grouped per era (raw, scale-honest) + @State private var latestNight: Repository.NightHRV? // dual-metric header + Apple cross-check + @State private var loaded = false + + /// One point on a smoothed trend line. + private struct Smoothed: Identifiable { + let id: String + let date: Date + let value: Double + } + /// The hero's readable trend (rolling median of the era-relative %), and the raw chart's per-era + /// equivalent. Precomputed in `load()` — never per body eval. + /// Smoothed trend split into CONTIGUOUS runs (see `breakAtGaps`) — one drawn line per run. + @State private var heroRuns: [[Smoothed]] = [] + @State private var rawRuns: [(brand: String, pts: [Smoothed])] = [] + + /// A wearer takes the band off. Charts joins consecutive points, so without an explicit break a + /// months-long non-wear hole renders as a straight line — a trend that never happened (this history + /// has real holes: 2025-06-27→09-25, 2025-10-13→11-19). Any gap longer than this opens a new run, so + /// the line simply stops and restarts and the hole reads as absence, which is the truth. + private static let gapBreakDays = 14.0 + + private func breakAtGaps(_ pts: [Smoothed]) -> [[Smoothed]] { + guard let first = pts.first else { return [] } + var out: [[Smoothed]] = [] + var cur: [Smoothed] = [first] + for p in pts.dropFirst() { + if let last = cur.last, p.date.timeIntervalSince(last.date) > Self.gapBreakDays * 86_400 { + out.append(cur); cur = [] + } + cur.append(p) + } + if !cur.isEmpty { out.append(cur) } + return out + } + + /// Nights per side of the rolling window. 420 raw nights across ~700 pt is ~1.7 pt/night — drawn as a + /// line that is a solid block of vertical strokes, not a trend. A 7-night CENTERED rolling median is + /// the standard fix: it strips night-to-night noise and (being a median, not a mean) refuses to let a + /// nap-fragment artifact drag the curve, leaving ~60 readable points across two years. + private static let smoothWindow = 7 + + /// Centered rolling median over (date, value) pairs, oldest→newest. Returns [] when there is less + /// history than the window — the caller then just shows the raw nights, which at that size are legible. + private func rollingMedian(_ pts: [(date: Date, value: Double)], window: Int = smoothWindow) -> [Smoothed] { + guard window > 1, pts.count >= window else { return [] } + let half = window / 2 + return pts.indices.map { i in + let lo = max(0, i - half), hi = min(pts.count - 1, i + half) + let s = pts[lo...hi].map(\.value).sorted() + let mid = s.count / 2 + let med = s.count % 2 == 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2 + return Smoothed(id: "\(i)", date: pts[i].date, value: med) + } + } + + var body: some View { + ScreenScaffold(title: "Cross-Device HRV", + subtitle: "Your full HRV history, stitched across a device switch") { + VStack(alignment: .leading, spacing: NoopMetrics.sectionSpacing) { + if !loaded { + ProgressView() + .frame(maxWidth: .infinity, minHeight: 220, alignment: .center) + } else { + latestNightSection + if rows.count >= 2 { + heroTrend + rawTrend + eraLegend + methodology + } else if latestNight == nil { + emptyCard + } + } + } + } + .task { await load() } + } + + // MARK: 0 · Latest night — both HRV metrics, labeled (+ Apple cross-check) + + @ViewBuilder private var latestNightSection: some View { + if let n = latestNight, n.rmssd != nil || n.sdnn != nil { + VStack(alignment: .leading, spacing: NoopMetrics.gap) { + SectionHeader("Latest Night", overline: "Both HRV metrics, labeled", trailing: prettyDay(n.day)) + NoopCard(tint: StrandPalette.metricPurple) { + VStack(alignment: .leading, spacing: NoopMetrics.cardInnerSpacing) { + HStack(alignment: .top, spacing: 0) { + metricColumn("RMSSD", n.rmssd, "beat-to-beat · vagal", StrandPalette.metricPurple) + Divider().frame(height: 46).overlay(StrandPalette.hairline) + metricColumn("SDNN", n.sdnn, "5-min index · overall", StrandPalette.metricCyan) + } + if let local = n.sdnn, let apple = n.appleSdnn, apple > 0 { + Divider().overlay(StrandPalette.hairline) + appleCrossCheck(local: local, apple: apple) + } + Text("RMSSD tracks fast vagal (parasympathetic) tone; SDNN captures overall variability. Both are computed from the same night's beats — different lenses, not different data — so they're no longer collapsed into one ambiguous \u{201C}HRV\u{201D} number.") + .font(StrandFont.footnote).foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + } + } + + private func metricColumn(_ label: LocalizedStringKey, _ value: Double?, _ caption: String, + _ accent: Color) -> some View { + VStack(alignment: .leading, spacing: 2) { + Text(label).strandOverline() + Text(value.map { "\(Int($0.rounded())) ms" } ?? "—") + .font(StrandFont.title2) + .foregroundStyle(value == nil ? StrandPalette.textTertiary : accent) + Text(caption).font(StrandFont.footnote).foregroundStyle(StrandPalette.textTertiary) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 2) + } + + private func appleCrossCheck(local: Double, apple: Double) -> some View { + let deltaPct = Int(((local - apple) / apple * 100).rounded()) + return VStack(alignment: .leading, spacing: 3) { + Text("Apple Watch cross-check").strandOverline() + HStack { + Text("NOOP 5-min SDNN \(Int(local.rounded())) ms") + Spacer() + Text("Apple \(Int(apple.rounded())) ms") + } + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + Text("\(deltaPct >= 0 ? "+" : "")\(deltaPct)% vs your watch — both are short-window SDNN (NOOP over 5-min segments, Apple ~1-min), so NOOP's usually reads a little higher but tracks the same nightly moves.") + .font(StrandFont.footnote).foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + + // MARK: 1 · Era-relative long-term trend (the continuous shape across the switch) + + private var heroTrend: some View { + let boundaries = rows.filter { $0.isEraStart } + // Robust symmetric Y-bound: the era-relative % is a log-ratio, so a known Oura nap-fragment + // artifact (an overnight "HRV" of ~12 ms against a ~135 ms normal) reads as −240%+ and would + // auto-stretch the axis until the real ±20-40% trend collapses to a flat line. Scale to the 90th + // percentile of |deviation| (padded), clamped to a readable band; genuine outliers clip cleanly. + let absPcts = rows.map { abs($0.pct) }.sorted() + let p90 = absPcts.isEmpty ? 40 : absPcts[min(absPcts.count - 1, Int(Double(absPcts.count) * 0.90))] + let heroBound = min(75, max(25, (p90 * 1.3).rounded())) + return VStack(alignment: .leading, spacing: NoopMetrics.gap) { + SectionHeader("Long-Term Trend", overline: "Stitched across devices", + trailing: "\(rows.count) nights") + NoopCard(tint: StrandPalette.metricPurple) { + VStack(alignment: .leading, spacing: NoopMetrics.cardInnerSpacing) { + Text("HRV vs each device-era's own normal").strandOverline() + Chart { + RuleMark(y: .value("Normal", 0)) + .lineStyle(StrokeStyle(lineWidth: 1, dash: [4, 4])) + .foregroundStyle(StrandPalette.textTertiary.opacity(0.6)) + // Every real night, as a faint cloud — the honest spread stays visible… + ForEach(rows) { row in + PointMark(x: .value("Date", row.date), + y: .value("Night", row.pct)) + .symbolSize(4) + .foregroundStyle(StrandPalette.metricPurple.opacity(0.16)) + } + // …and the TREND is the rolling median riding through it, which is the only thing + // legible at two years per screen. + ForEach(Array(heroRuns.enumerated()), id: \.offset) { idx, run in + ForEach(run) { s in + LineMark(x: .value("Date", s.date), + y: .value("Trend", s.value), + series: .value("Run", idx)) + .interpolationMethod(.monotone) + .lineStyle(StrokeStyle(lineWidth: 2.5, lineCap: .round, lineJoin: .round)) + .foregroundStyle(StrandPalette.metricPurple) + } + } + ForEach(boundaries) { b in + RuleMark(x: .value("Switch", b.date)) + .lineStyle(StrokeStyle(lineWidth: 1, dash: [3, 3])) + .foregroundStyle(brandColor(b.brand).opacity(0.7)) + .annotation(position: .top, alignment: .center, spacing: 2) { + Text("→ \(brandLabel(b.brand))") + .font(StrandFont.footnote) + .foregroundStyle(brandColor(b.brand)) + } + } + } + .frame(height: NoopMetrics.chartHeight) + .chartYScale(domain: -heroBound...heroBound) + .chartPlotStyle { $0.clipped() } + .chartXAxis { dateAxis } + .chartYAxis { + AxisMarks(position: .leading, values: .automatic(desiredCount: 5)) { value in + AxisGridLine().foregroundStyle(StrandPalette.hairline.opacity(0.4)) + AxisValueLabel { + if let pct = value.as(Double.self) { + Text("\(pct > 0 ? "+" : "")\(Int(pct))%") + } + } + .foregroundStyle(StrandPalette.textTertiary) + .font(StrandFont.footnote) + } + } + Text("Dots are your actual nights; the line is their 7-night rolling median — the trend, without the night-to-night noise. It stops wherever you weren't wearing anything for a couple of weeks, rather than drawing across the gap. Each sensor is compared to its own baseline, so the shape carries across your switch even though the raw numbers differ. The dashed line is each era's normal.") + .font(StrandFont.footnote) + .foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + } + + // MARK: 2 · Raw nightly HRV (scale-honest, per device) + + private var rawTrend: some View { + VStack(alignment: .leading, spacing: NoopMetrics.gap) { + SectionHeader("Raw Nightly HRV", overline: "Scale-honest per device") + NoopCard(tint: StrandPalette.metricCyan) { + VStack(alignment: .leading, spacing: NoopMetrics.cardInnerSpacing) { + Text("Actual milliseconds, never averaged").strandOverline() + Chart { + // Faint per-night cloud, coloured by its era… + ForEach(Array(segments.enumerated()), id: \.offset) { _, seg in + ForEach(seg) { row in + PointMark(x: .value("Date", row.date), + y: .value("HRV", row.raw)) + .symbolSize(4) + .foregroundStyle(brandColor(row.brand).opacity(0.16)) + } + } + // …with each era's own smoothed trend. Smoothed PER ERA and drawn as separate + // series, so the median never blends the two incompatible scales and no line is + // drawn across the switch — the step between eras stays visible, which is the point. + ForEach(Array(rawRuns.enumerated()), id: \.offset) { idx, run in + ForEach(run.pts) { s in + LineMark(x: .value("Date", s.date), + y: .value("Trend", s.value), + series: .value("Run", idx)) + .interpolationMethod(.monotone) + .lineStyle(StrokeStyle(lineWidth: 2.5, lineCap: .round, lineJoin: .round)) + .foregroundStyle(brandColor(run.brand)) + } + } + } + .frame(height: NoopMetrics.chartHeight) + .chartPlotStyle { $0.clipped() } + .chartXAxis { dateAxis } + .chartYAxis { + AxisMarks(position: .leading, values: .automatic(desiredCount: 4)) { _ in + AxisGridLine().foregroundStyle(StrandPalette.hairline.opacity(0.4)) + AxisValueLabel().foregroundStyle(StrandPalette.textTertiary).font(StrandFont.footnote) + } + } + Text("Oura and WHOOP measure HRV on different scales, so the two eras sit at different heights — that gap is real, not a change in you. Each era is smoothed on its own and never joined across the switch. We keep each sensor's true numbers instead of blending them.") + .font(StrandFont.footnote) + .foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + } + + // MARK: 3 · Era legend + + private var eraLegend: some View { + VStack(alignment: .leading, spacing: NoopMetrics.gap) { + SectionHeader("Devices", overline: "Eras") + NoopCard(tint: StrandPalette.accent) { + VStack(spacing: NoopMetrics.cardInnerSpacing) { + ForEach(Array(eras.enumerated()), id: \.offset) { idx, era in + HStack(spacing: 10) { + Circle().fill(brandColor(era.brand)).frame(width: 10, height: 10) + VStack(alignment: .leading, spacing: 1) { + Text(brandLabel(era.brand)) + .font(StrandFont.body).foregroundStyle(StrandPalette.textPrimary) + Text("\(monthYear(era.startDay)) – \(monthYear(era.endDay)) · \(era.nights) nights") + .font(StrandFont.footnote).foregroundStyle(StrandPalette.textTertiary) + } + Spacer() + VStack(alignment: .trailing, spacing: 1) { + Text("\(Int(era.centerMs.rounded())) ms") + .font(StrandFont.body).foregroundStyle(brandColor(era.brand)) + Text("typical").font(StrandFont.footnote).foregroundStyle(StrandPalette.textTertiary) + } + } + if idx < eras.count - 1 { Divider().overlay(StrandPalette.hairline) } + } + } + } + } + } + + // MARK: 4 · Methodology + + private var methodology: some View { + NoopCard(tint: StrandPalette.metricPurple) { + VStack(alignment: .leading, spacing: NoopMetrics.cardInnerSpacing) { + Text("How this is stitched").strandOverline() + Text("You've worn more than one HRV sensor. Different sensors report HRV on different scales, and there's no night where both were worn to line them up — so any single blended number would be fiction.") + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + .fixedSize(horizontal: false, vertical: true) + Text("Instead, each night is measured against its own device-era's normal (a robust log-scale centre). Plotting that relative view lets the long-term trend flow continuously across the switch, while the raw chart keeps each sensor's true milliseconds. Your daily readiness and stress scores stay strictly per-device.") + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + + private var emptyCard: some View { + NoopCard(tint: StrandPalette.metricPurple) { + Text("Not enough HRV history yet to stitch a cross-device trend. Keep wearing your strap, or import a prior device's history.") + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textTertiary) + .frame(maxWidth: .infinity, minHeight: 120, alignment: .center) + .multilineTextAlignment(.center) + } + } + + // MARK: - Shared axis / brand helpers + + private var dateAxis: some AxisContent { + AxisMarks(values: .automatic(desiredCount: 4)) { _ in + AxisGridLine().foregroundStyle(StrandPalette.hairline.opacity(0.4)) + AxisValueLabel(format: .dateTime.month(.abbreviated).year(.twoDigits)) + .foregroundStyle(StrandPalette.textTertiary).font(StrandFont.footnote) + } + } + + private func brandColor(_ brand: String) -> Color { + switch brand { + case "oura": return StrandPalette.metricCyan + case "whoop": return StrandPalette.metricPurple + case "fitbit": return StrandPalette.metricAmber + case "garmin": return StrandPalette.chargeColor + default: return StrandPalette.accent + } + } + + private func brandLabel(_ brand: String) -> String { + switch brand { + case "oura": return "Oura" + case "whoop": return "WHOOP" + case "fitbit": return "Fitbit" + case "garmin": return "Garmin" + default: return brand.capitalized + } + } + + private static let dayParser: DateFormatter = { + let f = DateFormatter(); f.dateFormat = "yyyy-MM-dd" + f.locale = Locale(identifier: "en_US_POSIX"); f.timeZone = TimeZone(identifier: "UTC") + return f + }() + private static let monthYearFmt: DateFormatter = { + let f = DateFormatter(); f.dateFormat = "MMM yyyy"; f.locale = Locale(identifier: "en_US_POSIX") + return f + }() + private func monthYear(_ day: String) -> String { + guard let d = Self.dayParser.date(from: day) else { return day } + return Self.monthYearFmt.string(from: d) + } + private static let dayFmt: DateFormatter = { + let f = DateFormatter(); f.dateFormat = "MMM d"; f.locale = Locale(identifier: "en_US_POSIX") + return f + }() + private func prettyDay(_ day: String) -> String { + guard let d = Self.dayParser.date(from: day) else { return day } + return Self.dayFmt.string(from: d) + } + + // MARK: - Load + + private func load() async { + let history = await repo.crossDeviceHrvHistory() + let result = CrossDeviceHRVTrend.build(history) + let parsed: [Row] = result.points.map { p in + Row(id: p.day, date: Self.dayParser.date(from: p.day) ?? Date(timeIntervalSince1970: 0), + raw: p.raw, pct: p.eraRelativePct, brand: p.brand, isEraStart: p.isEraStart) + } + // Group into per-era segments (a device switch opens a new segment). + var segs: [[Row]] = []; var cur: [Row] = [] + for row in parsed { + if row.isEraStart && !cur.isEmpty { segs.append(cur); cur = [] } + cur.append(row) + } + if !cur.isEmpty { segs.append(cur) } + + let night = await repo.latestNightHRV() + + self.eras = result.eras + self.rows = parsed + self.segments = segs + // The era-relative % is cross-era comparable, so one median over the whole history is correct — + // then split at non-wear gaps so no line is drawn across a hole. + self.heroRuns = breakAtGaps(rollingMedian(parsed.map { (date: $0.date, value: $0.pct) })) + // The raw ms is NOT cross-era comparable — smooth each era separately, then gap-split within it, + // carrying the era's brand so each run keeps its colour. + self.rawRuns = segs.flatMap { seg -> [(brand: String, pts: [Smoothed])] in + let brand = seg.first?.brand ?? "whoop" + return breakAtGaps(rollingMedian(seg.map { (date: $0.date, value: $0.raw) })) + .map { (brand: brand, pts: $0) } + } + self.latestNight = night + self.loaded = true + } +} diff --git a/Strand/Screens/DataSourcesView.swift b/Strand/Screens/DataSourcesView.swift index fe58eb63cc..a80ac5f672 100644 --- a/Strand/Screens/DataSourcesView.swift +++ b/Strand/Screens/DataSourcesView.swift @@ -43,6 +43,20 @@ struct DataSourcesView: View { // `@StateObject` construction time would either fail to compile or crash at runtime. @StateObject private var oura = OuraConnectModel() #endif + #if CLOUD_SYNC + // Cloud Sync (compiled in ONLY with CLOUD_SYNC): uploads this device's own backup and pulls edits + // confirmed on the user's own noop-cloud server, applying them locally and re-exporting into Apple + // Health on iOS. Same call-time-`repo:` shape as `OuraConnectModel` above, for the same + // environment-object-timing reason. The URL/token fields are local drafts — `CloudSyncModel` only + // holds the SAVED values (in the Keychain via `CloudSyncSettings`), never the in-progress text. + @StateObject private var cloudSync = CloudSyncModel() + @State private var cloudSyncURLDraft = CloudSyncSettings.serverURL ?? "" + @State private var cloudSyncTokenDraft = "" + // Phase 3.5 (zero-touch): a build carrying bundle-injected CLOUDSYNC_URL/CLOUDSYNC_TOKEN collapses + // the card to a one-line summary by default; this reveals the manual fields for a per-device + // override. Starts false even on a bundle-configured build — there is nothing to override yet. + @State private var cloudSyncShowOverride = false + #endif // "Remove Apple Health imported data" (ah-delete #616): a destructive escape hatch that purges every // row stored under the "apple-health" source via DeviceRegistryStore.deleteAllData. Two-step (a // confirmation alert) since it can't be undone. Local to this screen; no live strap data is touched. @@ -100,8 +114,11 @@ struct DataSourcesView: View { #if OURA_CLOUD_IMPORT ouraCloudCard.staggeredAppear(index: 7) #endif - broadcastHrCard.staggeredAppear(index: 8) - liveCard.staggeredAppear(index: 9) + #if CLOUD_SYNC + cloudSyncCard.staggeredAppear(index: 8) + #endif + broadcastHrCard.staggeredAppear(index: 9) + liveCard.staggeredAppear(index: 10) } } .onAppear { @@ -329,6 +346,136 @@ struct DataSourcesView: View { } #endif // OURA_CLOUD_IMPORT + #if CLOUD_SYNC + /// Cloud Sync: uploads this device's own `.noopbak` and pulls + applies edit-journal rows YOU + /// confirmed on your own noop-cloud server (a personal-fork companion, not a shipped/supported + /// service), re-exporting into Apple Health on iOS when the pull touched sleep/workouts/HR. + /// `cloudSync.syncNow(repo:)`/`saveSettings(url:token:)` take `repo`/args at call time — see the + /// `@StateObject` declaration's note. Phase 3.5 (zero-touch): a build carrying bundle-injected + /// `CLOUDSYNC_URL`/`CLOUDSYNC_TOKEN` with no Keychain override collapses to a one-line "Configured + /// from build" summary instead of the URL/token fields; "Override…" reveals them for a manual, + /// per-device server. + private var cloudSyncCard: some View { + card(title: String(localized: "Cloud Sync"), icon: "arrow.triangle.2.circlepath.circle", + tint: StrandPalette.metricRose, + subtitle: String(localized: "Upload this device's data to your own noop-cloud server and pull back edits you confirmed elsewhere. A personal-fork lane — off unless you've set up your own server.")) { + if CloudSyncSettings.isBundleConfigured && !cloudSyncShowOverride { + bundleConfiguredCloudSync + } else { + manualCloudSyncFields + } + } + } + + /// Collapsed state for a build with bundle-injected credentials and no Keychain override: nothing + /// to fill in, just a confirmation + the primary action. "Override…" reveals `manualCloudSyncFields` + /// for a rare per-device custom server — the Keychain value it saves takes over on every future + /// launch (`CloudSyncSettings.effectiveURL` prefers the Keychain), and `isBundleConfigured` reads + /// false as soon as it exists, so the card never falls back to this collapsed view on its own + /// after that. + private var bundleConfiguredCloudSync: some View { + VStack(alignment: .leading, spacing: NoopMetrics.space3) { + HStack(spacing: 6) { + Image(systemName: "checkmark.circle.fill").foregroundStyle(StrandPalette.statusPositive) + Text("Configured from build").font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + } + cloudSyncSyncRow + Button("Override…") { cloudSyncShowOverride = true } + .buttonStyle(.plain) + .font(StrandFont.caption) + .foregroundStyle(StrandPalette.textSecondary) + } + } + + /// Manual entry — the Server URL / Token fields + Save, also reachable via "Override…" on a + /// bundle-configured build. + private var manualCloudSyncFields: some View { + VStack(alignment: .leading, spacing: NoopMetrics.space3) { + VStack(alignment: .leading, spacing: 6) { + Text("Server URL").strandOverline() + TextField("https://your-server.example.com", text: $cloudSyncURLDraft) + .textFieldStyle(.plain) + .font(StrandFont.body) + .foregroundStyle(StrandPalette.textPrimary) + .padding(.horizontal, 12) + .padding(.vertical, 9) + .background(StrandPalette.surfaceInset, in: RoundedRectangle(cornerRadius: 10, style: .continuous)) + .overlay(RoundedRectangle(cornerRadius: 10, style: .continuous) + .strokeBorder(StrandPalette.hairline, lineWidth: 1)) + .disableAutocorrection(true) + #if os(iOS) + .textInputAutocapitalization(.never) + .keyboardType(.URL) + #endif + .accessibilityLabel("noop-cloud server URL") + } + VStack(alignment: .leading, spacing: 6) { + Text("Token").strandOverline() + SecureField("Paste your server's read-write token", text: $cloudSyncTokenDraft) + .textFieldStyle(.plain) + .font(StrandFont.body) + .foregroundStyle(StrandPalette.textPrimary) + .padding(.horizontal, 12) + .padding(.vertical, 9) + .background(StrandPalette.surfaceInset, in: RoundedRectangle(cornerRadius: 10, style: .continuous)) + .overlay(RoundedRectangle(cornerRadius: 10, style: .continuous) + .strokeBorder(StrandPalette.hairline, lineWidth: 1)) + .accessibilityLabel("noop-cloud token") + } + Button { + cloudSync.saveSettings(url: cloudSyncURLDraft, token: cloudSyncTokenDraft) + } label: { + Label("Save", systemImage: "checkmark.circle") + } + .buttonStyle(NoopButtonStyle(.secondary)) + .disabled(cloudSyncURLDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + || cloudSyncTokenDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + cloudSyncSyncRow + if CloudSyncSettings.isBundleConfigured { + Button("Use build configuration instead") { cloudSyncShowOverride = false } + .buttonStyle(.plain) + .font(StrandFont.caption) + .foregroundStyle(StrandPalette.textSecondary) + } + } + } + + /// The "Sync now" button (+ busy spinner) and status line — identical whichever fields are + /// showing. Its own `VStack` (not a bare `Group`) so it's always a full, independent sibling in + /// whichever outer `VStack` hosts it, never flattened sideways into a parent `HStack`. + /// + /// Falls back to the PERSISTED status (Finding 2) when this card's own `cloudSync.statusText` is + /// nil — e.g. right after launch, when the zero-touch auto-sync ran on its own throwaway + /// `CloudSyncModel` instance (see `CloudSyncGate`'s doc comment) and this screen's `@StateObject` + /// never saw it happen. A plain `UserDefaults` read on render, no cross-instance observation. + private var cloudSyncSyncRow: some View { + VStack(alignment: .leading, spacing: NoopMetrics.space3) { + HStack(spacing: NoopMetrics.space3) { + Button { + // Recompute rollups after applied edits — see CloudSyncModel.postApplyRefresh. + let intelligence = model.intelligence + let repository = repo + cloudSync.postApplyRefresh = { + await intelligence.analyzeRecent() + await repository.refresh() + } + cloudSync.syncNow(repo: repo) + } label: { + Label(cloudSync.busy ? "Syncing…" : "Sync now", systemImage: "arrow.triangle.2.circlepath") + } + .buttonStyle(NoopButtonStyle(.primary)) + .disabled(!cloudSync.isConfigured || cloudSync.busy) + if cloudSync.busy { ProgressView().controlSize(.small) } + } + if let s = cloudSync.statusText { + Text(s).font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + } else if let last = CloudSyncModel.lastPersistedStatus { + Text("Last sync: \(last)").font(StrandFont.subhead).foregroundStyle(StrandPalette.textSecondary) + } + } + } + #endif // CLOUD_SYNC + private func presentImporter(_ target: ImportTarget) { importTarget = target #if os(iOS) diff --git a/Strand/Screens/DevicesView.swift b/Strand/Screens/DevicesView.swift index 8c4a45c12e..c8b827abba 100644 --- a/Strand/Screens/DevicesView.swift +++ b/Strand/Screens/DevicesView.swift @@ -159,6 +159,11 @@ private struct DevicesContent: View { device: device, isActive: device.status == .active, isLiveConnected: device.status == .active && live.connected, + // Has this link actually proven itself (any data, or a completed handshake)? `connected` + // alone is just "CoreBluetooth handed us an object" — true before service discovery, and + // true for a state-restored peripheral that is already dead. Without this the card + // pulsed "Active · Live" for a strap that had flat-lined hours earlier. + linkProven: live.linkProven, // #221: a WHOOP 5/MG can be BLE-connected yet have its ENCRYPTED bond refused (the // WHOOP app, or a stale iOS pairing, holds the single-app bond) — no HR/biometric data // flows even though the link is up, so "Active · Live" overstates it. pairingHint is @@ -457,13 +462,29 @@ struct DevicePillState: Equatable { var pulsing: Bool = false var showsDot: Bool = true + /// `linkProven` (see `LiveState.linkProven`): has this link actually carried a byte, or completed its + /// connect handshake? `isLiveConnected` is built from `live.connected`, which BLEManager publishes the + /// instant CoreBluetooth's `didConnect` fires — BEFORE service discovery, so before any characteristic + /// exists or any data flows. Layer iOS state restoration on top (it hands back a peripheral that reads + /// `.connected` and re-seeds the bond flags from a PREVIOUS process) and the pill confidently pulsed + /// "Active · Live" for a strap that had died on the user's wrist hours earlier. + /// + /// Defaults `true` so every existing caller — and the whole non-WHOOP source family (Oura, Huami, FTMS, + /// standard-HR), none of which touch the flag — keeps its exact current behaviour. An unproven link + /// falls back to plain "Active", which is the honest statement: this IS your active device, we just + /// can't claim data is flowing. It deliberately does NOT invent a new pill state — the strap may be a + /// second away from streaming, and the stalled-handshake watchdog cancels a genuinely dead link inside + /// 90 s anyway. static func resolve(isArchived: Bool, isActive: Bool, isReconnecting: Bool, - bondRefused: Bool, isLiveConnected: Bool) -> DevicePillState { + bondRefused: Bool, isLiveConnected: Bool, + linkProven: Bool = true) -> DevicePillState { if isArchived { return DevicePillState(label: "Removed", tone: .neutral, showsDot: false) } guard isActive else { return DevicePillState(label: "Paired", tone: .neutral) } if isReconnecting { return DevicePillState(label: "Reconnecting…", tone: .warning, pulsing: true) } if bondRefused { return DevicePillState(label: "Connected · not paired", tone: .warning) } - if isLiveConnected { return DevicePillState(label: "Active · Live", tone: .positive, pulsing: true) } + if isLiveConnected && linkProven { + return DevicePillState(label: "Active · Live", tone: .positive, pulsing: true) + } return DevicePillState(label: "Active", tone: .positive) } } @@ -476,6 +497,10 @@ private struct DeviceCard: View { let device: PairedDevice let isActive: Bool let isLiveConnected: Bool + /// Has the live link actually carried data / completed its handshake (`LiveState.linkProven`)? Gates the + /// "· Live" half of the pill so a raw-but-dead CBPeripheral — the restored-zombie case — reads "Active" + /// instead of pulsing "Active · Live". Defaults true: every non-WHOOP source keeps today's behaviour. + var linkProven: Bool = true /// #221: the active+connected strap is BLE-linked but its encrypted bond was refused (#78 state) — /// no HR/biometric data flows despite the link being up. Drives the "Connected · not paired" pill /// (which takes priority over "Active · Live") and the honest subtitle/footnote. False for every @@ -731,7 +756,7 @@ private struct DeviceCard: View { private var pillState: DevicePillState { DevicePillState.resolve(isArchived: device.status == .archived, isActive: isActive, isReconnecting: isReconnecting, bondRefused: bondRefused, - isLiveConnected: isLiveConnected) + isLiveConnected: isLiveConnected, linkProven: linkProven) } private var statePill: some View { @@ -1190,8 +1215,17 @@ private struct EcgProbeSheets: ViewModifier { titleVisibility: .visible, presenting: target) { device in Button("Start ECG capture") { model.ecgStartCapture(); target = nil } + // Deliberately worded so it cannot be mistaken for the normal path: it sends the same + // three commands with ONE byte changed (mainControlECGDataGeneration restart=2 rather + // than start=1), which is the only argument in that enum no build here has ever sent. + // Same gates as every other entry in this dialog; "Stop" reverses it either way. + Button("Start ECG capture (restart variant, experimental)") { + model.ecgRestartCapture(); target = nil + } Button("Stop ECG capture") { model.ecgStopCapture(); target = nil } Button("Set which wrist you wear it on…") { target = nil; wristTarget = device } + // Local file read, no strap traffic: did the record actually land in the raw archive? + Button("Check archived raw records") { model.reportRawHistoryArchive(); target = nil } Button("Cancel", role: .cancel) { target = nil } } message: { _ in Text("NOOP is not a medical device and this is not an ECG test. It asks your MG to start its ECG subsystem and logs whatever comes back — unvalidated instrumentation for protocol research, never a measurement or a diagnosis, including any heart-rhythm classification the strap happens to send. Don't use it to make a health decision; see a doctor if you have symptoms.\n\nHold the two indents on the clasp with the fingers of your other hand for the whole capture. The MG measures across your wrist AND that clasp, so until you hold it the circuit is open, the strap has nothing to record, and you would see zero packets whatever the firmware did.\n\nNobody has confirmed a strap honours these commands, so the likely outcome is that nothing happens. Everything here is reversible: “Stop” turns the streams back off. Results land in the strap log.") @@ -1212,6 +1246,46 @@ private struct EcgProbeSheets: ViewModifier { EcgProbeResultView(text: live.ecgProbe ?? "", onClose: { model.clearEcgProbe() }) } + .sheet(isPresented: Binding(get: { live.rawHistoryArchiveReport != nil }, + set: { if !$0 { model.clearRawHistoryArchiveReport() } })) { + RawHistoryArchiveReportView(text: live.rawHistoryArchiveReport ?? "", + onClose: { model.clearRawHistoryArchiveReport() }) + } + } +} + +/// What the on-disk raw-history reject archive currently holds, rendered by +/// `RawHistoryArchive.summaryText`. Structurally the #592/#690/ECG result views with the medical +/// disclaimer dropped: this reports the contents of a local file, not anything a strap said. +/// +/// The two lines to read after an experiment are the per-version split (an "all-zero payload" count in +/// the thousands is a placeholder flood, not data) and the newest capture timestamps, which is what you +/// match against the electrode-contact window. +private struct RawHistoryArchiveReportView: View { + let text: String + let onClose: () -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + Text("Archived raw records") + .font(StrandFont.title2) + .foregroundStyle(StrandPalette.textPrimary) + ScrollView { + Text(text) + .font(StrandFont.mono) + .foregroundStyle(StrandPalette.textSecondary) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + } + HStack { + Button("Copy") { PlatformPasteboard.copy(text) } + Spacer() + Button("Close") { onClose() } + } + } + .padding(20) + .frame(minWidth: 340, minHeight: 260) + .background(StrandPalette.surfaceOverlay) } } diff --git a/Strand/Screens/SettingsView.swift b/Strand/Screens/SettingsView.swift index 7e6125690b..32e39f95f4 100644 --- a/Strand/Screens/SettingsView.swift +++ b/Strand/Screens/SettingsView.swift @@ -90,6 +90,7 @@ struct SettingsView: View { /// minutes. Self-gates on OBSERVED gravity + step density (#345) — a no-op on a sparse night (e.g. /// WHOOP 4.0) regardless of this switch. See [PuffinExperiment.motionAwareWakeKey]. @AppStorage(PuffinExperiment.motionAwareWakeKey) private var motionAwareWakeEnabled = false + @AppStorage(PuffinExperiment.wakeBridgeKey) private var wakeBridgeEnabled = false // Imperial/Metric display preference (D#103). Stored data is always SI; this only changes how // distances/weights/heights/temperatures are SHOWN — and lets the profile fields below take @@ -1431,6 +1432,21 @@ struct SettingsView: View { .font(StrandFont.caption) .foregroundStyle(StrandPalette.textTertiary) .fixedSize(horizontal: false, vertical: true) + + Divider().overlay(StrandPalette.hairline) + + // MARK: Bridge brief wakes — default OFF. + Toggle(isOn: $wakeBridgeEnabled) { + Text("Bridge brief wakes") + .font(StrandFont.subhead) + .foregroundStyle(StrandPalette.textPrimary) + } + .toggleStyle(.switch) + .tint(StrandPalette.accent) + Text("Keeps a night interrupted by a brief get-up (under ~45 minutes) as one sleep, with the wake scored inside it, instead of splitting it into separate sleeps — and short sleep stretches between such wakes are no longer discarded for being under an hour on their own. Worth trying if your nights show up split into two or three pieces around bathroom trips or brief stirs. Off by default; takes effect on the next nights scored.") + .font(StrandFont.caption) + .foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) } } } diff --git a/Strand/Screens/SleepView.swift b/Strand/Screens/SleepView.swift index a1d85e8455..fbb4869daa 100644 --- a/Strand/Screens/SleepView.swift +++ b/Strand/Screens/SleepView.swift @@ -1851,11 +1851,27 @@ struct SleepView: View { sleepDebtLedger: debtLedger(napSleepMinByDay: napSleepMinByDay)) } - /// The rolling 14-night sleep-debt ledger from the cached daily metrics. Uses the - /// SAME personal sleep need the tiles use (`sleepNeedMin`, ≥ 7.5 h, the per-user - /// override over the 8 h default), measured against each main night's `totalSleepMin` - /// plus actual asleep minutes from separately-recorded naps. - /// Skips nights with no sleep (the analytics function does the skip). (#242) + /// The defensible per-user sleep NEED (hours) BOTH debt surfaces (this ledger card and the + /// "Sleep Debt" tile) measure against — the population-anchored, age-floored, upper-quartile + /// `personalizedNeedHours`, the SAME estimator Intelligence uses. Deliberately NOT the + /// self-referential `sleepNeedMin` (mean total sleep): the mean drifts DOWN toward a chronic + /// under-sleeper's own deficit and quietly erases their debt, whereas the upper-quartile + /// estimate floored at the ~8 h adult target only adjusts UP for genuine long sleepers. Age + /// isn't plumbed to this screen, so it floors at the adult target (age: nil) — wiring the + /// profile age through would only raise it for under-18s. One need, both debt surfaces. (#242) + private var personalSleepNeedHours: Double { + AnalyticsEngine.Rest.personalizedNeedHours( + nightlyHours: repo.days.compactMap { $0.totalSleepMin.map { $0 / 60.0 } }, + age: nil) + } + + /// The rolling sleep-debt ledger from the cached daily metrics, measured against the + /// defensible `personalSleepNeedHours`. Each night's credited sleep is the main night's + /// `totalSleepMin` plus actual asleep minutes from separately-recorded naps (#1042). The + /// recency-weighted, capped, asymmetric running balance (recent nights weigh more, |balance| + /// bounded, oversleep repays less than a short night accrues) lives in `SleepDebt.ledger`; + /// here we only supply the series + need. Skips nights with no sleep (the analytics function + /// does the skip). (#242) private func debtLedger(napSleepMinByDay: [String: Double]) -> SleepDebtLedger { SleepDebt.ledger( series: repo.days.map { day in @@ -1863,7 +1879,7 @@ struct SleepView: View { mainSleepMin: day.totalSleepMin, napSleepMin: napSleepMinByDay[day.day] ?? 0)) }, - needHours: sleepNeedMin / 60.0) + needHours: personalSleepNeedHours) } // MARK: - Derived model @@ -2341,11 +2357,15 @@ struct SleepView: View { metric { $0.respRateBpm } } - /// Sleep debt (minutes): the imported sleep_debt_min when the export carried it; else - /// the APPROXIMATE per-night need − (main sleep + nap sleep), floored at 0. + /// Sleep debt (minutes): the imported sleep_debt_min when the export carried it; else the + /// APPROXIMATE per-night need − (main sleep + nap sleep), floored at 0 (no "credit"). + /// Measured against the SAME defensible `personalSleepNeedHours` the ledger card uses, so + /// this per-night tile and the running-balance card agree on what "need" means — one need + /// across both debt surfaces, rather than the self-referential `sleepNeedMin` mean this + /// used to use. (#242) private func sleepDebtSeries(napSleepMinByDay: [String: Double]) -> Metric { let imported = repo.importedSleep - let need = sleepNeedMin + let need = personalSleepNeedHours * 60.0 let series = repo.days.compactMap { d -> Double? in if let debt = imported[d.day]?.debtMin { return debt } // minutes, export-verbatim guard let asleep = SleepDebt.creditedSleepMin( diff --git a/Strand/Screens/StressView.swift b/Strand/Screens/StressView.swift index e795ebaf7d..07526f0812 100644 --- a/Strand/Screens/StressView.swift +++ b/Strand/Screens/StressView.swift @@ -40,6 +40,10 @@ struct StressView: View { /// from the day's banked HR + R-R via the SAME 0–3 proxy the daily score uses. Nil /// until the async read completes; `.empty` when the day has no usable intraday HR. @State private var daytime: DaytimeStress.Result? + /// Whether TODAY's intraday timeline is scored against the PERSONAL cross-day daytime baseline + /// (`.baselineRelative`, once enough worn history exists) instead of the day's own calm hours + /// (`.dayRelative`). Drives only the explanatory copy — the 0–3 scale + bands are identical either way. + @State private var daytimeUsesPersonalBaseline = false /// Drives the Breathe sheet presented from the sustained-stress suggestion. @State private var showBreathe = false @@ -111,8 +115,20 @@ struct StressView: View { } let rr = (try? await repo.storeHandle()?.rrIntervals( deviceId: repo.deviceId, from: from, to: to, limit: 200_000)) ?? [] + // Wrist accelerometer for the motion gate: an ambulatory hour is EXERTION, not stress, so it's + // masked rather than scored (DaytimeStress). Same store read as R-R; empty on hardware/imports + // with no gravity, which is exactly the "no masking, prior behaviour" degradation. + let gravity = (try? await repo.storeHandle()?.gravitySamples( + deviceId: repo.deviceId, from: from, to: to, limit: 200_000)) ?? [] - daytime = DaytimeStress.analyze(hr: hr, rr: rr, tzOffsetSeconds: tz) + // Score today's hours against the PERSONAL cross-day daytime baseline when enough worn history + // exists (Oura-style `.baselineRelative`, validated r≈0.6), else the day's own calm hours + // (`.dayRelative`, the unchanged default). The mode is resolved only AFTER the HR-count guard + // above, so the trailing-history reads are never paid on a day with no scorable timeline. + let mode = await daytimeScoringMode(startOfToday: startOfDay) + if case .baselineRelative = mode { daytimeUsesPersonalBaseline = true } + else { daytimeUsesPersonalBaseline = false } + daytime = DaytimeStress.analyze(hr: hr, rr: rr, gravity: gravity, tzOffsetSeconds: tz, mode: mode) // ADDITIVE advanced readouts, computed on-demand from the SAME `rr` (no extra fetch, no // DB / schema change, and no effect on the 0..3 score above). Each engine returns nil when @@ -122,6 +138,43 @@ struct StressView: View { freqHRV = HRVFreqDomain.freqDomain(rr: rr) } + /// Trailing local days folded into the personal daytime baselines the `.baselineRelative` mode + /// scores against. 30 mirrors the app's other rolling baselines (nightly resting-HR / HRV) and the + /// illness engine's ~28-day base. + private static let baselineHistoryDays = 30 + + /// Build the personal daytime baselines from the trailing `baselineHistoryDays` local days (TODAY + /// EXCLUDED — it's the day being scored, not part of its own baseline) and return the scoring mode + /// for today's intraday read: `.baselineRelative` once there's enough real worn daytime-HR history + /// for a usable baseline, else `.dayRelative` (the unchanged default). The only behavioural change + /// vs. before is that a user with a few worn days now scores today's hours against their own + /// cross-day floor instead of the day's own calm hours; a cold-start / sparse-history user is + /// byte-identical to before (`DaytimeStress.scoringMode` is the single degradation gate). + /// + /// PERF: reads each past day's raw HR (and, only when that day was worn, R-R) once, bounded per day, + /// off the main actor via `repo` — riding the same async `load()` the today-timeline already runs on. + /// Unworn days are skipped without an R-R read. The `DaytimeStress` analyze memo is untouched; the + /// fold itself is O(days). + private func daytimeScoringMode(startOfToday: Date) async -> DaytimeStress.ScoringMode { + let cal = Calendar.current + var days: [DaytimeStress.DaytimeDayStreams] = [] + days.reserveCapacity(Self.baselineHistoryDays) + // Oldest → newest so the EWMA fold replays the history in order. + for back in stride(from: Self.baselineHistoryDays, through: 1, by: -1) { + guard let dayStart = cal.date(byAdding: .day, value: -back, to: startOfToday), + let dayEnd = cal.date(byAdding: .day, value: 1, to: dayStart) else { continue } + let from = Int(dayStart.timeIntervalSince1970) + let to = Int(dayEnd.timeIntervalSince1970) - 1 + let dayTz = TimeZone.current.secondsFromGMT(for: dayStart) + let dayHR = await repo.hrSamples(from: from, to: to, limit: 200_000) + guard !dayHR.isEmpty else { continue } // unworn day — no floor to learn, skip the R-R read + let dayRR = (try? await repo.storeHandle()?.rrIntervals( + deviceId: repo.deviceId, from: from, to: to, limit: 200_000)) ?? [] + days.append(.init(hr: dayHR, rr: dayRR, tzOffsetSeconds: dayTz)) + } + return DaytimeStress.scoringMode(history: days) + } + /// Recompute the cached `StressModel` only when (repo.days, storedSeries) /// actually changed since the last build. Equality is an O(n) value compare, /// far cheaper than the model rebuild it guards. @@ -152,7 +205,7 @@ struct StressView: View { // 2. Today's numbers — uniform tiles in one grid. VStack(alignment: .leading, spacing: NoopMetrics.gap) { - SectionHeader("Today", overline: "Markers", trailing: String(localized: "vs 30-day baseline")) + SectionHeader("Today", overline: "Markers", trailing: String(localized: "vs your baseline")) tileGrid(model) } .staggeredAppear(index: 1) @@ -233,7 +286,7 @@ struct StressView: View { // bar split by how many waking hours sat in each band, with durations. StressTotalsBar(totals: StressTotals(hours: day.hours)) - Text("The line is each waking hour's 0-3 proxy, scored against your own calm hours today. The bar below splits your day into calm, moderate and high stress time.") + Text(daytimeTimelineCaption) .font(StrandFont.footnote) .foregroundStyle(StrandPalette.textTertiary) .fixedSize(horizontal: false, vertical: true) @@ -252,6 +305,16 @@ struct StressView: View { return String(localized: "avg \(String(format: "%.1f", mean)) · \(n)h") } + /// The timeline's explanatory line, honest about WHICH reference each hour was scored against — + /// the personal cross-day baseline (`.baselineRelative`) or the day's own calm hours (`.dayRelative`). + /// Explicit `LocalizedStringKey` so BOTH variants stay in the string catalog (a ternary inside + /// `Text(_:)` would resolve to the verbatim, non-localized `String` overload). + private var daytimeTimelineCaption: LocalizedStringKey { + daytimeUsesPersonalBaseline + ? "The line is each waking hour's 0-3 proxy, scored against your personal daytime baseline (how your own days usually run). The bar below splits your day into calm, moderate and high stress time." + : "The line is each waking hour's 0-3 proxy, scored against your own calm hours today. The bar below splits your day into calm, moderate and high stress time." + } + /// A passive, in-app nudge to run a Breathe session after a sustained high-stress run. /// No notification — just a card with a CTA that opens the existing trainer. private func sustainedBreatheCard(_ day: DaytimeStress.Result) -> some View { @@ -537,7 +600,7 @@ struct StressView: View { : "Stress is derived from two autonomic signals.") .font(StrandFont.body) .foregroundStyle(StrandPalette.textPrimary) - Text("We compare today's resting heart rate and HRV to your own 30-day baseline. A higher-than-usual resting HR and a lower-than-usual HRV both push the score up, classic signs the body is activated. The combined shift is mapped onto a 0-3 scale: 0 is calm, 1.5 sits at your baseline, 3 is highly activated.") + Text("We compare today's resting heart rate and HRV to your own recent baseline (HRV on a log scale, the way the science recommends). A higher-than-usual resting HR AND a lower-than-usual HRV together are the classic sign of activation — and we only read low HRV as stress when your resting HR is up too, so a deeply-relaxed low-HRV night isn't mistaken for stress. The combined shift is mapped onto a 0-3 scale: 0 is calm, 1.5 sits at your baseline, 3 is highly activated. A sustained shift in your baseline itself is called out separately.") .font(StrandFont.subhead) .foregroundStyle(StrandPalette.textSecondary) .fixedSize(horizontal: false, vertical: true) @@ -703,6 +766,12 @@ struct StressModel { let calmTimeValue: String // e.g. "58%" let calmTimeCaption: String // e.g. "of last 30 days" let usingStored: Bool // true when today's value came from the stored series + /// The Buchheit autonomic quadrant behind today's derived score (nil for a stored/cold-start day). + let quadrant: DailyStressEngine.Quadrant? + /// The sustained baseline-shift readout — surfaced SEPARATELY from the score (a sustained supplement / + /// training / illness load reads as a *shifted baseline*, not acute daily stress). Nil until there's + /// enough long history, or on a stored day. + let chronicShift: DailyStressEngine.ChronicShift? /// Last up-to-14 trend values, for the hero tile sparkline. var sparkValues: [Double] { Array(fullTrend.suffix(14)).map(\.value) } @@ -751,16 +820,26 @@ struct StressModel { let rhrT = today.restingHr.map(Double.init) let hrvT = today.avgHrv - // Resolve today's score: prefer a stored value, else derive. + // Resolve today's score: prefer a stored (imported) value; else the DailyStressEngine + // (dual-baseline, coupled, log-domain); else the legacy StressMath path for a cold-start day the + // engine can't yet score (< its minimum baseline). The engine also yields today's autonomic + // quadrant + the separate chronic-shift readout. + // One batch pass scores every day against its OWN prior baseline (sorts once) — reused for both + // today's headline and the whole trend below. + let engineByDay: [String: DailyStressEngine.DailyStress] = Dictionary( + DailyStressEngine.evaluateSeries(days: days).map { ($0.day, $0.stress) }, + uniquingKeysWith: { a, _ in a }) + let engineToday = engineByDay[today.day] let derivedAvailable = (rhrT != nil && meanRHR != nil) || (hrvT != nil && meanHRV != nil) let storedToday = storedByDay[today.day] - guard storedToday != nil || derivedAvailable else { return nil } + guard storedToday != nil || engineToday != nil || derivedAvailable else { return nil } - let derivedToday: Double? = derivedAvailable + let legacyDerived: Double? = derivedAvailable ? StressMath.squash(StressMath.rawScore( rhrToday: rhrT, meanRHR: meanRHR, sdRHR: sdRHR, hrvToday: hrvT, meanHRV: meanHRV, sdHRV: sdHRV)) : nil + let derivedToday: Double? = engineToday?.score ?? legacyDerived let s = storedToday ?? derivedToday ?? 1.5 self.usingStored = storedToday != nil @@ -770,22 +849,37 @@ struct StressModel { self.hrvToday = hrvT self.rhrDelta = (rhrT != nil && meanRHR != nil) ? (rhrT! - meanRHR!) : nil self.hrvDelta = (hrvT != nil && meanHRV != nil) ? (hrvT! - meanHRV!) : nil + // Quadrant + chronic-shift apply only to an engine-derived score, not a stored/import day. + self.quadrant = storedToday == nil ? engineToday?.quadrant : nil + self.chronicShift = storedToday == nil ? engineToday?.chronicShift : nil - self.explanation = StressMath.explanation( + var explanationText = StressMath.explanation( band: self.band, rhrDelta: self.rhrDelta, hrvDelta: self.hrvDelta, usingStored: self.usingStored ) + // Surface a SUSTAINED baseline shift as CONTEXT — never folded into the number. A low daily + // score sitting on top of a suppressed baseline is a shifted "normal", not calm; and a real + // sustained load stays visible instead of being defined away once the short baseline adapts. + if let cs = self.chronicShift, cs.isSustainedLoad { + explanationText += " Your resting HRV has held about \(Int(cs.hrvPctBelowLongTerm.rounded()))% " + + "below your 60-day normal — a sustained autonomic shift (a supplement, a heavy training " + + "block, or getting run down can all do this), not just today." + } + self.explanation = explanationText - // Full daily proxy history: stored value if present for the day, else the - // z-score derivation against the SAME baseline so the line is comparable. + // Full daily history: a stored value if present, else the engine's score for THAT day — each day + // vs its OWN contemporaneous dual baseline (fixing the old line, which re-scored all history + // against today's single baseline), else the legacy proxy for an early day the engine can't score. var pts: [TrendPoint] = [] for d in days { guard let date = Self.dayParser.date(from: d.day) else { continue } if let v = storedByDay[d.day] { - pts.append(TrendPoint(date: date, value: v)) - continue + pts.append(TrendPoint(date: date, value: v)); continue + } + if let e = engineByDay[d.day] { + pts.append(TrendPoint(date: date, value: e.score)); continue } let dRHR = d.restingHr.map(Double.init) let dHRV = d.avgHrv diff --git a/Strand/Screens/TestCentreView.swift b/Strand/Screens/TestCentreView.swift index 4e8b815f0c..4f208ea7aa 100644 --- a/Strand/Screens/TestCentreView.swift +++ b/Strand/Screens/TestCentreView.swift @@ -44,6 +44,13 @@ struct TestCentreView: View { @AppStorage(PuffinExperiment.ppgHrSubLagInterpKey) private var ppgHrSubLagInterpEnabled = false @AppStorage(PuffinExperiment.hrvReadinessKey) private var hrvReadinessEnabled = false + #if CLOUD_SYNC + // Section 5: the page-replication trial. Bound to the SAME UserDefaults key SyncReplicationTrial + // reads at launch, so this toggle IS the switch rather than a mirror of it. Default false, which is + // what a missing key reads as, so a fresh install and an explicit "off" are the same state. + @AppStorage(SyncReplicationTrial.enabledKey) private var replicationTrialEnabled = false + #endif + /// True when the connected strap is a 5/MG, so the 5/MG experimental block shows. Mirrors the /// SettingsView gate (#22): a confident 4.0 owner never sees controls that cannot touch their strap. private var is5MG: Bool { selectedWhoopModelRaw == WhoopModel.whoop5mg.rawValue } @@ -63,6 +70,9 @@ struct TestCentreView: View { diagnosticToolsCard.staggeredAppear(index: 1) exportCard.staggeredAppear(index: 2) experimentalAlgorithmsCard.staggeredAppear(index: 3) + #if CLOUD_SYNC + replicationTrialCard.staggeredAppear(index: 4) + #endif } } .id(refreshToken) @@ -345,6 +355,60 @@ struct TestCentreView: View { } } + // MARK: - Section 5: Page-replication trial (CLOUD_SYNC only, opt-in, off by default) + + #if CLOUD_SYNC + /// The one UI for `SyncReplicationTrial`. Until this existed the switch was a bare `UserDefaults` + /// key with no way to flip it short of attaching a debugger, which meant the measurement it exists + /// to produce could not be started on the device that has the data. + /// + /// Three things are shown, deliberately, because they are three different facts: + /// + /// - the **intent** (this toggle, i.e. the persisted flag); + /// - the **reality** (`SyncReplicationTrial.statusLine`, which reports what the store actually + /// opened with) — these differ for the whole of the launch you flip the toggle in, because + /// `wal_autocheckpoint` is applied at pool-open time and cannot be changed on a live pool; and + /// - the **result so far** (the telemetry aggregate, carried inside `statusLine`). + /// + /// Reading only the first would let a user conclude the trial is running when it is not. + @ViewBuilder private var replicationTrialCard: some View { + NoopCard { + VStack(alignment: .leading, spacing: NoopMetrics.space3) { + Text("PAGE REPLICATION TRIAL") + .font(StrandFont.overline).tracking(StrandFont.overlineTracking) + .foregroundStyle(StrandPalette.textSecondary) + Text("Measures whether shipping only the changed pages of your database could replace the whole-database Cloud Sync upload. Opt-in, off by default.") + .font(StrandFont.caption).foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + Toggle(isOn: $replicationTrialEnabled) { + Text("Hand WAL checkpointing to the replicator") + .font(StrandFont.subhead).foregroundStyle(StrandPalette.textPrimary) + } + .toggleStyle(.switch).tint(StrandPalette.accent) + .accessibilityLabel("Page replication trial") + + Text("When on, NOOP stops letting SQLite checkpoint the write-ahead log on its own and records one measurement per Cloud Sync upload. It does NOT change what gets uploaded: today's full-database upload is untouched, so the worst case is a larger write-ahead log between syncs. Takes effect the next time NOOP starts.") + .font(StrandFont.caption).foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + Divider().overlay(StrandPalette.hairline) + + // The state actually in force, which is NOT the toggle above until the next launch. + Text(SyncReplicationTrial.statusLine) + .font(StrandFont.caption).foregroundStyle(StrandPalette.textSecondary) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) + + if replicationTrialEnabled, !SyncReplicationTrial.isInForce { + Text("Restart NOOP to start measuring.") + .font(StrandFont.caption).foregroundStyle(StrandPalette.statusWarning) + } + } + } + } + #endif + // MARK: - Shared actions (same calls as the SettingsView controls these re-host) /// Re-anchor every baseline that feeds Charge from now, via the single cross-platform source of diff --git a/Strand/System/BGTaskIdentifier.swift b/Strand/System/BGTaskIdentifier.swift new file mode 100644 index 0000000000..34ca8c309d --- /dev/null +++ b/Strand/System/BGTaskIdentifier.swift @@ -0,0 +1,66 @@ +import Foundation + +/// The iOS `BGTaskScheduler` identifiers this build is actually permitted to register — and a canary +/// that reports a mismatch loudly instead of letting the whole background lane fail silently. +/// +/// WHY THIS EXISTS: a BGTask identifier has to be written twice — once in the iOS target's +/// `BGTaskSchedulerPermittedIdentifiers` (project.yml, which XcodeGen expands into the generated +/// `StrandiOS/Resources/Info.plist`) and once in Swift. iOS compares the two as EXACT strings and +/// refuses to register a handler whose identifier isn't on the permitted list. The permitted entries +/// are namespaced under the bundle id, so the documented "build under your own Apple ID" path (see +/// `APP_GROUP_ID` in project.yml) rewrites the plist side and leaves a hardcoded Swift literal stale. +/// `BGTaskScheduler.register` then returns `false` and `submit` throws `.notPermitted` — and both call +/// sites discarded those signals, so the lane went dark with no symptom at all. +/// +/// Same single-source-of-truth posture as `WidgetSnapshot.suiteName`: read what the bundle actually +/// carries, and fall back to the canonical upstream literal only if the key is missing. +/// +/// macOS compiles this file too (it lives under `Strand/`), where `BGTaskScheduler` doesn't exist and +/// the plist key is absent — `permitted` is simply empty and nothing calls into here. +enum BGTaskIdentifier { + /// The `BGTaskSchedulerPermittedIdentifiers` array from THIS process's bundle — the exact list iOS + /// checks a registration against. Empty on macOS and on any build that hasn't wired the key. + static let permitted: [String] = { + Bundle.main.object(forInfoDictionaryKey: "BGTaskSchedulerPermittedIdentifiers") as? [String] ?? [] + }() + + /// The identifier this build may actually register for a given lane, e.g. + /// `resolve(suffix: "debugexport", upstream: "com.noopapp.noop.debugexport")`. + /// + /// Resolution order, most authoritative first: + /// 1. **The permitted list itself.** This is the exact array iOS compares a registration against, + /// so an entry from it is correct by construction — however the plist got its values, and even + /// if someone hand-edits it to something that isn't `.`. + /// 2. **This build's bundle id + the suffix.** Covers a target that hasn't wired the plist key. + /// 3. **The canonical upstream literal**, so behaviour is unchanged on a stock build even if the + /// bundle id is somehow unreadable. + /// + /// The suffix match is anchored on a leading `.` so `debugexport` can never claim + /// `cloudsync.refresh`'s entry or vice versa. + static func resolve(suffix: String, upstream: String) -> String { + if let permittedEntry = permitted.first(where: { $0.hasSuffix(".\(suffix)") }) { return permittedEntry } + if let bundleID = Bundle.main.bundleIdentifier { return "\(bundleID).\(suffix)" } + return upstream + } + + /// Report the outcome of a `BGTaskScheduler.register` call. A failure is logged in EVERY + /// configuration, not just Debug: a sideloaded personal build is exactly where a bundle-id remap + /// breaks this, and a Debug-only signal would never reach that build. + /// + /// Deliberately does NOT `assertionFailure` the way `WidgetSnapshot.assertGroupProvisioned` does. + /// A missing App Group entitlement breaks user-visible core features (widget, Live Activity, watch + /// snapshot), so trapping in Debug is proportionate there. A missing BGTask permit only disables an + /// opportunistic lane this file's own doc comments call "never a dependency for correctness" — + /// turning that into a launch crash would be worse than the bug. Loud is enough. + static func report(_ identifier: String, registered: Bool) { + guard !registered else { + NSLog("BGTaskIdentifier: registered '\(identifier)'") + return + } + NSLog(""" + BGTaskIdentifier: FAILED to register '\(identifier)' — not in \ + BGTaskSchedulerPermittedIdentifiers \(permitted). This background lane will never run. \ + The identifier must match the iOS target's permitted list in project.yml exactly. + """) + } +} diff --git a/Strand/System/ScheduledDebugExport.swift b/Strand/System/ScheduledDebugExport.swift index 867d74327e..673f56bb18 100644 --- a/Strand/System/ScheduledDebugExport.swift +++ b/Strand/System/ScheduledDebugExport.swift @@ -127,11 +127,15 @@ enum ScheduledDebugExport { return removed } - /// iOS BGTask identifier. Derived from the running bundle id (rather than hardcoded) so it tracks - /// BUNDLE_ID_PREFIX (see Config/BundleId.xcconfig) automatically and always matches the iOS target's - /// `BGTaskSchedulerPermittedIdentifiers` (Info.plist), which is built from `$(PRODUCT_BUNDLE_IDENTIFIER)` - /// the same way. Must also be registered at launch for `submit` to succeed — wired in the app entry point. - static let bgTaskIdentifier = (Bundle.main.bundleIdentifier ?? "com.noopapp.noop") + ".debugexport" + /// iOS BGTask identifier. Must also appear in the iOS target's `BGTaskSchedulerPermittedIdentifiers` + /// (Info.plist) and be registered at launch for `submit` to succeed — wired in the app entry point. + /// + /// Resolved from the bundle rather than hardcoded, because the permitted entries are namespaced + /// under the bundle id: a build under a different Apple ID (the documented "change it in one place" + /// path — see `APP_GROUP_ID` in project.yml) rewrites the plist side, and a literal here would go + /// stale and silently kill this lane. See `BGTaskIdentifier` for the full contract. + static let bgTaskIdentifier = BGTaskIdentifier.resolve( + suffix: "debugexport", upstream: "com.noopapp.noop.debugexport") static var isEnabled: Bool { UserDefaults.standard.bool(forKey: K.enabled) } @@ -293,12 +297,15 @@ enum ScheduledDebugExport { /// the task. Both live in the iOS app target — call this from `StrandiOSApp.init()`. Safe to leave /// uncalled: `submitBackgroundRequest()` fails gracefully and the macOS path + "Run now" still work. static func register() { - BGTaskScheduler.shared.register(forTaskWithIdentifier: bgTaskIdentifier, using: nil) { task in + let registered = BGTaskScheduler.shared.register(forTaskWithIdentifier: bgTaskIdentifier, using: nil) { task in // Write the drop, then immediately request the next one (BGAppRefresh is single-shot). if isEnabled { catchUpIfDue() } submitBackgroundRequest() task.setTaskCompleted(success: true) } + // `register` returns false when the identifier isn't on the permitted list — the one signal that + // this lane is dead. Discarding it is what let a bundle-id remap silently disable the drop. + BGTaskIdentifier.report(bgTaskIdentifier, registered: registered) } /// Submit a background-refresh request for *no earlier than* the next chosen time. Honest: iOS decides diff --git a/StrandTests/BackfillContinuationTests.swift b/StrandTests/BackfillContinuationTests.swift index d7fb17866e..b952576511 100644 --- a/StrandTests/BackfillContinuationTests.swift +++ b/StrandTests/BackfillContinuationTests.swift @@ -232,7 +232,15 @@ final class BackfillContinuationTests: XCTestCase { XCTAssertLessThanOrEqual(passes, BackfillContinuation.defaultMaxAutoContinues + 1, "auto-continue must be bounded — it can't loop forever") } - // It stopped because it hit the cap (deep backlog), not because it silently stalled at pass 1. + // It stopped because it CAUGHT UP — guard 2 (strap no longer ahead of our frontier) — not because + // a cap cut it off mid-recovery, and not because it silently stalled at pass 1. + // + // This assertion used to read `count == defaultMaxAutoContinues`, which passed only because the cap + // was 6 and catching up needs 7 passes: the drain was ALWAYS severed one day short of done, and the + // test enshrined that as correct. The cap is a runaway backstop, not a drain schedule — a strap that + // is genuinely behind and genuinely handing over data must be allowed to finish. `testHistoryComplete + // SlicesAreCappedNotRunaway` is where the cap's bounding job is pinned, using a strap that never + // catches up; that is the case the cap exists for. XCTAssertEqual(count, BackfillContinuation.defaultMaxAutoContinues) } diff --git a/StrandTests/BackfillProgressTests.swift b/StrandTests/BackfillProgressTests.swift new file mode 100644 index 0000000000..caf1a81365 --- /dev/null +++ b/StrandTests/BackfillProgressTests.swift @@ -0,0 +1,72 @@ +import XCTest +@testable import Strand + +/// `BackfillProgress` — the honest answer to "is my data lost, or merely pending?". +/// +/// The gap this closes: the app said "Syncing strap history…" while a session ran and NOTHING between +/// sessions, so an idle app looked finished when it was 18 hours behind waiting out a 15-minute floor. Both +/// numbers were already tracked (the strap's reported newest, and our persisted frontier) — nothing ever +/// subtracted them and said so. +final class BackfillProgressTests: XCTestCase { + private let now = 1_800_000_000 + private var frontier18hBehind: Int { now - 18 * 3600 } + + /// The user's real case: 18h behind, actively draining. + func testRecovering() { + let p = BackfillProgress.resolve(connected: true, backfilling: true, + frontierUnix: frontier18hBehind, strapNewestUnix: now) + XCTAssertEqual(p, .recovering(behindSeconds: 18 * 3600, frontierUnix: frontier18hBehind)) + XCTAssertEqual(p.behindLabel, "~18h behind") + XCTAssertEqual(p.frontierUnix, frontier18hBehind) + } + + /// The state that used to be INVISIBLE: behind, but idle between sessions. Silence read as "done". + func testPending_isDistinctFromRecovering() { + let p = BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: frontier18hBehind, strapNewestUnix: now) + XCTAssertEqual(p, .pending(behindSeconds: 18 * 3600, frontierUnix: frontier18hBehind)) + XCTAssertEqual(p.behindLabel, "~18h behind") + } + + /// Caught up uses the SAME threshold the drain stops at, so the readout can never say "behind" about a + /// gap `BackfillContinuation` has already decided is closed. + func testUpToDate_matchesTheDrainsOwnThreshold() { + let gap = BackfillContinuation.defaultBehindGapSeconds + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: now - gap, strapNewestUnix: now), + .upToDate) + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: now - gap - 1, strapNewestUnix: now) + .behindSeconds, gap + 1) + XCTAssertNil(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: now, strapNewestUnix: now).behindLabel) + } + + /// Not connected, or the strap hasn't answered a range — claim nothing. + func testUnknown() { + XCTAssertEqual(BackfillProgress.resolve(connected: false, backfilling: false, + frontierUnix: frontier18hBehind, strapNewestUnix: now), + .unknown) + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: nil, strapNewestUnix: now), + .unknown) + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: frontier18hBehind, strapNewestUnix: nil), + .unknown) + } + + /// A frontier far AHEAD of the strap's reported newest is a stale/wrong-epoch range (#451) or a + /// future-dated clock (#928) — not negative backlog. Say nothing rather than a nonsense figure. + func testFrontierAheadOfStrap_claimsNothing() { + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: false, + frontierUnix: now, strapNewestUnix: now - 30 * 24 * 3600), + .unknown) + } + + /// Sub-hour gaps read in minutes, so a nearly-caught-up drain doesn't round to a bare "~0h behind". + func testMinuteLabel() { + XCTAssertEqual(BackfillProgress.resolve(connected: true, backfilling: true, + frontierUnix: now - 42 * 60, strapNewestUnix: now) + .behindLabel, "~42m behind") + } +} diff --git a/StrandTests/CloudEditApplierTests.swift b/StrandTests/CloudEditApplierTests.swift new file mode 100644 index 0000000000..15469a57dc --- /dev/null +++ b/StrandTests/CloudEditApplierTests.swift @@ -0,0 +1,415 @@ +// Tests the CLOUD_SYNC-gated journal → store applier; compiled only when the flag is set +// (StrandTests shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +import WhoopStore +import WhoopProtocol +@testable import Strand + +final class CloudEditApplierTests: XCTestCase { + + // MARK: - Fixture helpers + + private func json(_ obj: [String: Any]) -> String { + String(data: try! JSONSerialization.data(withJSONObject: obj), encoding: .utf8)! + } + + private func edit(seq: Int, kind: String, payload: [String: Any], beforeJSON: String? = nil, + undoneBySeq: Int? = nil) -> CloudEdit { + CloudEdit(seq: seq, editId: "e\(seq)", kind: kind, payloadJSON: json(payload), beforeJSON: beforeJSON, + rationale: nil, appliedAt: 1_752_300_000, undoneBySeq: undoneBySeq, ackedAt: nil) + } + + // MARK: - adjust_sleep_bounds + + func testAdjustSleepBoundsSurvivesRecompute() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 1000, endTs: 29000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + + let e = edit(seq: 1, kind: "adjust_sleep_bounds", + payload: ["deviceId": "my-whoop", "startTs": 1000, "newEndTs": 30000]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertEqual(summary.skipped, 0) + XCTAssertEqual(summary.needsAttention, 0) + XCTAssertTrue(summary.touchedSleep) + XCTAssertEqual(summary.appliedSeqs, [1]) + + let edited = try await store.sleepSessions(deviceId: "my-whoop", from: 1000, to: 1000, limit: 1).first + XCTAssertEqual(edited?.endTs, 30000) + XCTAssertTrue(edited?.userEdited ?? false) + + // Recompute simulation: a later strap/import upsert tries to overwrite with DETECTED (un-edited) + // values — the userEdited flag set by applySleepEdit must survive it. + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 1000, endTs: 29000, efficiency: 91, restingHr: 51, avgHrv: 61, stagesJSON: nil), + ], deviceId: "my-whoop") + let survived = try await store.sleepSessions(deviceId: "my-whoop", from: 1000, to: 1000, limit: 1).first + XCTAssertEqual(survived?.endTs, 30000, "userEdited protection must survive a recompute upsert") + XCTAssertTrue(survived?.userEdited ?? false) + } + + func testAdjustSleepBoundsNeedsAttentionWhenSessionAbsent() async throws { + let store = try await WhoopStore.inMemory() + let e = edit(seq: 5, kind: "adjust_sleep_bounds", + payload: ["deviceId": "my-whoop", "startTs": 9999, "newEndTs": 30000]) + let summary = await CloudEditApplier.apply([e], store: store) + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.appliedSeqs, [5]) + } + + func testAdjustSleepBoundsRejectsInvertedWindow() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 1000, endTs: 29000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + + // newEndTs (500) is BEFORE the session's own startTs (1000) — an inverted window that + // SleepEditGuard.clampedEditWindow (#940 belt-and-braces) must refuse outright, exactly as + // Repository.editSleepTimes refuses one from a misbehaving client UI. + let e = edit(seq: 17, kind: "adjust_sleep_bounds", + payload: ["deviceId": "my-whoop", "startTs": 1000, "newEndTs": 500]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.needsAttention, 1, "an inverted window must be refused, not silently clamped") + XCTAssertEqual(summary.appliedSeqs, [17]) + + let untouched = try await store.sleepSessions(deviceId: "my-whoop", from: 1000, to: 1000, limit: 1).first + XCTAssertEqual(untouched?.endTs, 29000, "a rejected edit must leave the stored bounds unchanged") + XCTAssertFalse(untouched?.userEdited ?? true, "a rejected edit must not set userEdited") + } + + // MARK: - edit_sleep_stages + + func testEditSleepStagesPromotesUnEditedRowAndMapsStageVocabulary() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 2000, endTs: 5000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + let before = try await store.sleepSessions(deviceId: "my-whoop", from: 2000, to: 2000, limit: 1).first + XCTAssertFalse(before?.userEdited ?? true, "fixture must start un-edited to exercise the promotion path") + + // Server vocabulary is "awake"/"light"/"deep"/"rem" — "awake" must map to the app's "wake" token. + let stages: [[String: Any]] = [ + ["start": 2000, "end": 3000, "stage": "awake"], + ["start": 3000, "end": 5000, "stage": "deep"], + ] + let e = edit(seq: 9, kind: "edit_sleep_stages", + payload: ["deviceId": "my-whoop", "startTs": 2000, "stages": stages]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertEqual(summary.needsAttention, 0) + XCTAssertTrue(summary.touchedSleep) + + let after = try await store.sleepSessions(deviceId: "my-whoop", from: 2000, to: 2000, limit: 1).first + XCTAssertTrue(after?.userEdited ?? false, "promotion must set userEdited") + XCTAssertEqual(after?.endTs, 5000, "promotion must keep the CURRENT bounds, not reset them") + + let stagesJSON = try XCTUnwrap(after?.stagesJSON) + let decodedAny = try JSONSerialization.jsonObject(with: Data(stagesJSON.utf8)) + let decoded = try XCTUnwrap(decodedAny as? [[String: Any]]) + XCTAssertEqual(decoded.first?["stage"] as? String, "wake", "server 'awake' must map to the app's 'wake' token") + XCTAssertEqual(decoded.last?["stage"] as? String, "deep", "non-awake stages pass through verbatim") + } + + func testEditSleepStagesNeedsAttentionWhenSessionAbsent() async throws { + let store = try await WhoopStore.inMemory() + let stages: [[String: Any]] = [["start": 2000, "end": 3000, "stage": "light"]] + let e = edit(seq: 3, kind: "edit_sleep_stages", + payload: ["deviceId": "my-whoop", "startTs": 2000, "stages": stages]) + let summary = await CloudEditApplier.apply([e], store: store) + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.applied, 0) + } + + func testEditSleepStagesPromotionRejectsInvertedCurrentBounds() async throws { + let store = try await WhoopStore.inMemory() + // Seed a corrupted (inverted) session directly via upsertSleepSessions, which does NOT route + // through SleepEditGuard (only applySleepEdit does) — simulates a bad import/server mirror + // reaching the store with bounds no on-device edit path could ever produce. + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 5000, endTs: 4000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + let before = try await store.sleepSessions(deviceId: "my-whoop", from: 5000, to: 5000, limit: 1).first + XCTAssertFalse(before?.userEdited ?? true, "fixture must start un-edited to exercise the promotion path") + + let stages: [[String: Any]] = [["start": 4000, "end": 5000, "stage": "light"]] + let e = edit(seq: 18, kind: "edit_sleep_stages", + payload: ["deviceId": "my-whoop", "startTs": 5000, "stages": stages]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.needsAttention, 1, "promoting an inverted stored window must be refused") + XCTAssertEqual(summary.applied, 0) + + let untouched = try await store.sleepSessions(deviceId: "my-whoop", from: 5000, to: 5000, limit: 1).first + XCTAssertEqual(untouched?.endTs, 4000, "rejected promotion must leave stored bounds unchanged") + XCTAssertFalse(untouched?.userEdited ?? true, "rejected promotion must not set userEdited") + } + + // MARK: - delete_workout + + func testDeleteWorkoutTombstonesAndBlocksResurrection() async throws { + let store = try await WhoopStore.inMemory() + let workout = WorkoutRow(startTs: 100, endTs: 700, sport: "running", source: "whoop", durationS: 600, + energyKcal: 400, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, + zonesJSON: nil, notes: nil) + _ = try await store.upsertWorkouts([workout], deviceId: "my-whoop") + + let e = edit(seq: 4, kind: "delete_workout", payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running"]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertTrue(summary.touchedWorkouts) + let remaining = try await store.workouts(deviceId: "my-whoop", from: 0, to: 1000, limit: 10) + XCTAssertEqual(remaining.count, 0) + + // Resurrection guard: a later re-import of the exact same workout must be dropped. + let resurrected = try await store.upsertWorkouts([workout], deviceId: "my-whoop") + XCTAssertEqual(resurrected, 0) + } + + // MARK: - fix_workout + + func testFixWorkoutTombstonesOriginalAndUpsertsCloudCopy() async throws { + let store = try await WhoopStore.inMemory() + let original = WorkoutRow(startTs: 100, endTs: 700, sport: "running", source: "whoop", durationS: 600, + energyKcal: 400, avgHr: 140, maxHr: 170, strain: 8.5, distanceM: 5000, + zonesJSON: nil, notes: nil) + _ = try await store.upsertWorkouts([original], deviceId: "my-whoop") + + let beforeJSON = json(["deviceId": "my-whoop", "startTs": 100, "endTs": 700, "sport": "running", + "source": "whoop", "durationS": 600, "energyKcal": 400, "distanceM": 5000]) + let e = edit(seq: 6, kind: "fix_workout", + payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running", + "patch": ["energyKcal": 450]], + beforeJSON: beforeJSON) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertTrue(summary.touchedWorkouts) + + let originalGone = try await store.workouts(deviceId: "my-whoop", from: 0, to: 1000, limit: 10) + XCTAssertEqual(originalGone.count, 0) + let tombstones = try await store.workoutTombstones(deviceId: "my-whoop") + XCTAssertEqual(tombstones.count, 1) + + let copies = try await store.workouts(deviceId: "noop-cloud", from: 0, to: 1000, limit: 10) + XCTAssertEqual(copies.count, 1) + XCTAssertEqual(copies.first?.energyKcal, 450, "patch overlay must win over the snapshot") + XCTAssertEqual(copies.first?.distanceM, 5000, "un-patched fields carry over from the snapshot") + XCTAssertEqual(copies.first?.source, "whoop", "original detection source carries over") + + let devices = try DeviceRegistryStore(dbQueue: store.registryWriter).all() + let cloudDevice = devices.first { $0.id == "noop-cloud" } + XCTAssertEqual(cloudDevice?.brand, "noop-cloud") + XCTAssertEqual(cloudDevice?.model, "Cloud edits") + XCTAssertEqual(cloudDevice?.sourceKind, .cloudImport) + XCTAssertEqual(cloudDevice?.status, .paired) + } + + func testFixWorkoutNeedsAttentionWhenBeforeJSONMissing() async throws { + let store = try await WhoopStore.inMemory() + let e = edit(seq: 7, kind: "fix_workout", + payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running", + "patch": ["energyKcal": 450]]) + let summary = await CloudEditApplier.apply([e], store: store) + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.applied, 0) + } + + func testFixWorkoutNeedsAttentionWhenNoUsableEndTs() async throws { + // beforeJSON IS present (unlike testFixWorkoutNeedsAttentionWhenBeforeJSONMissing above) but + // carries no endTs, and the patch doesn't supply one either — the previously-untested branch + // of the `finalEndTs` guard. + let store = try await WhoopStore.inMemory() + let beforeJSON = json(["deviceId": "my-whoop", "startTs": 100, "sport": "running", "source": "whoop"]) + let e = edit(seq: 15, kind: "fix_workout", + payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running", + "patch": ["energyKcal": 450]], + beforeJSON: beforeJSON) + let summary = await CloudEditApplier.apply([e], store: store) + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.skipped, 0) + } + + func testFixWorkoutSkippedWhenSameBatchDeleteAlreadyWon() async throws { + // delete_workout(seq1) then fix_workout(seq2) targeting the SAME (deviceId, startTs, sport) in + // one batch: the delete must win outright — the fix must not resurrect a noop-cloud copy of a + // workout the same batch just deleted. + let store = try await WhoopStore.inMemory() + let workout = WorkoutRow(startTs: 100, endTs: 700, sport: "running", source: "whoop", durationS: 600, + energyKcal: 400, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, + zonesJSON: nil, notes: nil) + _ = try await store.upsertWorkouts([workout], deviceId: "my-whoop") + + let beforeJSON = json(["deviceId": "my-whoop", "startTs": 100, "endTs": 700, "sport": "running", + "source": "whoop", "durationS": 600, "energyKcal": 400, "distanceM": 5000]) + let edits = [ + edit(seq: 1, kind: "delete_workout", payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running"]), + edit(seq: 2, kind: "fix_workout", + payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running", "patch": ["energyKcal": 450]], + beforeJSON: beforeJSON), + ] + let summary = await CloudEditApplier.apply(edits, store: store) + + XCTAssertEqual(summary.applied, 1, "only the delete applies") + XCTAssertEqual(summary.skipped, 1, "the same-batch fix must be skipped, not applied") + XCTAssertEqual(summary.needsAttention, 0) + XCTAssertEqual(summary.appliedSeqs, [1, 2]) + + let copies = try await store.workouts(deviceId: "noop-cloud", from: 0, to: 1000, limit: 10) + XCTAssertEqual(copies.count, 0, "the delete already won — fix must not resurrect a noop-cloud copy") + let originalGone = try await store.workouts(deviceId: "my-whoop", from: 0, to: 1000, limit: 10) + XCTAssertEqual(originalGone.count, 0) + } + + func testFixWorkoutNeedsAttentionWhenOriginalAbsentWithNoTombstone() async throws { + // The original is gone but there is no tombstone for it (no same-batch delete either) — a + // stale server mirror, not a resolved delete. This must surface for a human look, not silently + // fabricate a noop-cloud copy of a workout the store never actually held. + let store = try await WhoopStore.inMemory() + let beforeJSON = json(["deviceId": "my-whoop", "startTs": 100, "endTs": 700, "sport": "running", + "source": "whoop", "durationS": 600, "energyKcal": 400, "distanceM": 5000]) + let e = edit(seq: 16, kind: "fix_workout", + payload: ["deviceId": "my-whoop", "startTs": 100, "sport": "running", + "patch": ["energyKcal": 450]], + beforeJSON: beforeJSON) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.skipped, 0) + let copies = try await store.workouts(deviceId: "noop-cloud", from: 0, to: 1000, limit: 10) + XCTAssertEqual(copies.count, 0, "no noop-cloud copy must be created for a stale server mirror") + } + + // MARK: - add_workout + + func testAddWorkoutUpsertsUnderCloudDevice() async throws { + let store = try await WhoopStore.inMemory() + let e = edit(seq: 8, kind: "add_workout", + payload: ["startTs": 1000, "endTs": 2800, "sport": "cycling", "energyKcal": 300, + "notes": "morning ride"]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertTrue(summary.touchedWorkouts) + let added = try await store.workouts(deviceId: "noop-cloud", from: 0, to: 3000, limit: 10) + XCTAssertEqual(added.count, 1) + XCTAssertEqual(added.first?.durationS, 1800, "durationS must be derived as endTs - startTs") + XCTAssertEqual(added.first?.sport, "cycling") + XCTAssertEqual(added.first?.notes, "morning ride") + } + + // MARK: - delete_hr_range + + func testDeleteHrRangeTombstonesAndBlocksResurrection() async throws { + let store = try await WhoopStore.inMemory() + try await store.insert(Streams(hr: [HRSample(ts: 1500, bpm: 60), HRSample(ts: 2500, bpm: 61)]), deviceId: "my-whoop") + + let e = edit(seq: 10, kind: "delete_hr_range", payload: ["deviceId": "my-whoop", "fromTs": 1000, "toTs": 2000]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertTrue(summary.touchedHr) + let survivors = try await store.hrSamples(deviceId: "my-whoop", from: 0, to: 10_000, limit: 10) + XCTAssertEqual(survivors.map(\.ts), [2500]) + + // Resurrection guard: re-streaming into the deleted range must be filtered. + try await store.insert(Streams(hr: [HRSample(ts: 1500, bpm: 60)]), deviceId: "my-whoop") + let stillGone = try await store.hrSamples(deviceId: "my-whoop", from: 0, to: 10_000, limit: 10) + XCTAssertEqual(stillGone.map(\.ts), [2500]) + } + + // MARK: - delete_metric_point + + func testDeleteMetricPointTombstonesAndBlocksResurrection() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "vo2max", value: 44)], deviceId: "my-whoop") + + let e = edit(seq: 11, kind: "delete_metric_point", + payload: ["deviceId": "my-whoop", "day": "2026-07-01", "key": "vo2max"]) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.applied, 1) + XCTAssertFalse(summary.touchedSleep) + XCTAssertFalse(summary.touchedWorkouts) + XCTAssertFalse(summary.touchedHr) + + let remaining = try await store.metricSeries(deviceId: "my-whoop", key: "vo2max", from: "2026-01-01", to: "2026-12-31") + XCTAssertEqual(remaining.count, 0) + + let resurrected = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "vo2max", value: 45)], deviceId: "my-whoop") + XCTAssertEqual(resurrected, 0) + } + + // MARK: - Skip rules + + func testMalformedPayloadSkipsWithoutThrowing() async throws { + let store = try await WhoopStore.inMemory() + let e = CloudEdit(seq: 12, editId: "bad", kind: "adjust_sleep_bounds", payloadJSON: "not json at all", + beforeJSON: nil, rationale: nil, appliedAt: 1_752_300_000, undoneBySeq: nil, ackedAt: nil) + let summary = await CloudEditApplier.apply([e], store: store) + XCTAssertEqual(summary.skipped, 1) + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.needsAttention, 0) + XCTAssertEqual(summary.appliedSeqs, [12], "a malformed row is still acked so the cursor can advance") + } + + func testUndoneRowSkipped() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 1000, endTs: 29000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + let e = edit(seq: 13, kind: "adjust_sleep_bounds", + payload: ["deviceId": "my-whoop", "startTs": 1000, "newEndTs": 99999], undoneBySeq: 14) + let summary = await CloudEditApplier.apply([e], store: store) + + XCTAssertEqual(summary.skipped, 1) + XCTAssertEqual(summary.applied, 0) + let untouched = try await store.sleepSessions(deviceId: "my-whoop", from: 1000, to: 1000, limit: 1).first + XCTAssertEqual(untouched?.endTs, 29000, "an edit undone by a later row must never be applied") + } + + func testUndoAndSetBaselineNoteAndUnknownKindAllSkip() async throws { + let store = try await WhoopStore.inMemory() + let edits = [ + CloudEdit(seq: 20, editId: "u1", kind: "undo", payloadJSON: #"{"targetSeq":1}"#, beforeJSON: nil, + rationale: nil, appliedAt: 1, undoneBySeq: nil, ackedAt: nil), + edit(seq: 21, kind: "set_baseline_note", payload: ["note": "felt great"]), + edit(seq: 22, kind: "some_future_kind", payload: [:]), + ] + let summary = await CloudEditApplier.apply(edits, store: store) + XCTAssertEqual(summary.skipped, 3) + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.appliedSeqs, [20, 21, 22]) + } + + // MARK: - Ordering / summary bookkeeping + + func testProcessesInSeqOrderRegardlessOfArrayOrder() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertSleepSessions([ + CachedSleepSession(startTs: 1000, endTs: 29000, efficiency: 90, restingHr: 50, avgHrv: 60, stagesJSON: nil), + ], deviceId: "my-whoop") + // Fed out of array order (seq 2 before seq 1) — the HIGHER seq must win, proving the internal sort. + let edits = [ + edit(seq: 2, kind: "adjust_sleep_bounds", payload: ["deviceId": "my-whoop", "startTs": 1000, "newEndTs": 40000]), + edit(seq: 1, kind: "adjust_sleep_bounds", payload: ["deviceId": "my-whoop", "startTs": 1000, "newEndTs": 30000]), + ] + let summary = await CloudEditApplier.apply(edits, store: store) + XCTAssertEqual(summary.applied, 2) + XCTAssertEqual(summary.appliedSeqs, [1, 2], "appliedSeqs must reflect seq order, not array order") + + let finalState = try await store.sleepSessions(deviceId: "my-whoop", from: 1000, to: 1000, limit: 1).first + XCTAssertEqual(finalState?.endTs, 40000, "the higher-seq edit, applied last, must win") + } +} +#endif // CLOUD_SYNC diff --git a/StrandTests/CloudSyncClientTests.swift b/StrandTests/CloudSyncClientTests.swift new file mode 100644 index 0000000000..fc4a2aed52 --- /dev/null +++ b/StrandTests/CloudSyncClientTests.swift @@ -0,0 +1,248 @@ +// Tests the CLOUD_SYNC-gated cloud sync client + settings; compiled only when the flag is set +// (StrandTests shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +@testable import Strand + +final class CloudSyncClientTests: XCTestCase { + override func setUp() { super.setUp(); OuraURLProtocolStub.reset() } + + private let baseURL = URL(string: "https://cloud.example.com")! + + private func makeClient() -> CloudSyncClient { + CloudSyncClient(baseURL: baseURL, token: "tok-1", session: OuraURLProtocolStub.session()) + } + + /// One journal row. `payloadJSON`'s embedded quotes are hand-escaped at the JSON level (`\"`) + /// inside a raw string literal, so no Swift-level escaping is needed. + private let editFixtureJSON = + #"{"seq":3,"editId":"e-1","kind":"adjust_sleep_bounds","payloadJSON":"{\"deviceId\":\"d1\"}","beforeJSON":null,"rationale":"user requested","appliedAt":1752300000,"undoneBySeq":null,"ackedAt":null}"# + + // MARK: - fetchEdits + + func testFetchEditsDecodesEditsAndLatestSeq() async throws { + let body = (#"{"edits":["# + editFixtureJSON + #"],"latestSeq":3}"#).data(using: .utf8)! + OuraURLProtocolStub.queue = [.init(status: 200, body: body)] + let client = makeClient() + + let result = try await client.fetchEdits(since: 0) + + XCTAssertEqual(result.latestSeq, 3) + XCTAssertEqual(result.edits.count, 1) + let edit = result.edits[0] + XCTAssertEqual(edit.seq, 3) + XCTAssertEqual(edit.editId, "e-1") + XCTAssertEqual(edit.kind, "adjust_sleep_bounds") + XCTAssertEqual(edit.payloadJSON, #"{"deviceId":"d1"}"#) + XCTAssertNil(edit.beforeJSON) + XCTAssertEqual(edit.rationale, "user requested") + XCTAssertEqual(edit.appliedAt, 1_752_300_000) + XCTAssertNil(edit.undoneBySeq) + XCTAssertNil(edit.ackedAt) + } + + func testFetchEditsSendsSinceInQueryString() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, + body: #"{"edits":[],"latestSeq":0}"#.data(using: .utf8)!)] + let client = makeClient() + + _ = try await client.fetchEdits(since: 42) + + let url = OuraURLProtocolStub.requestedURLs.last?.absoluteString ?? "" + XCTAssertTrue(url.contains("/edits")) + XCTAssertTrue(url.contains("since=42")) + } + + func testFetchEdits401ThrowsBadResponse() async throws { + OuraURLProtocolStub.queue = [.init(status: 401, body: "nope".data(using: .utf8)!)] + let client = makeClient() + + do { + _ = try await client.fetchEdits(since: 0) + XCTFail("expected badResponse to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncError, CloudSyncError.badResponse(401, "nope")) + } + } + + func testFetchEditsMalformedJSONThrowsDecode() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, body: "not json".data(using: .utf8)!)] + let client = makeClient() + + do { + _ = try await client.fetchEdits(since: 0) + XCTFail("expected decode to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncError, CloudSyncError.decode) + } + } + + // MARK: - ack + + func testAckReturnsAckedCount() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, body: #"{"acked":2}"#.data(using: .utf8)!)] + let client = makeClient() + + let acked = try await client.ack(seqs: [1, 2]) + + XCTAssertEqual(acked, 2) + let url = OuraURLProtocolStub.requestedURLs.last?.absoluteString ?? "" + XCTAssertTrue(url.contains("/edits/ack")) + } + + func testAck401ThrowsBadResponse() async throws { + OuraURLProtocolStub.queue = [.init(status: 401, body: "denied".data(using: .utf8)!)] + let client = makeClient() + + do { + _ = try await client.ack(seqs: [1]) + XCTFail("expected badResponse to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncError, CloudSyncError.badResponse(401, "denied")) + } + } + + func testAckMalformedJSONThrowsDecode() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, body: "{".data(using: .utf8)!)] + let client = makeClient() + + do { + _ = try await client.ack(seqs: [1]) + XCTFail("expected decode to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncError, CloudSyncError.decode) + } + } +} + +final class CloudSyncSettingsTests: XCTestCase { + override func setUp() { super.setUp(); CloudSyncSettings.clear() } + override func tearDown() { CloudSyncSettings.clear(); super.tearDown() } + + /// Uses `isConfigured(info:)` — the pure, dict-driven seam — with an explicit EMPTY info dict + /// rather than the live, no-arg `isConfigured` (which falls back to `Bundle.main`). This machine's + /// `OuraSecrets.xcconfig` carries real CLOUDSYNC_URL/CLOUDSYNC_TOKEN values, so the live property + /// would read true even with the Keychain cleared, defeating the "Keychain-only" round trip this + /// test is actually checking. `serverURL`/`token`/`clear()` (the Keychain side) don't need a seam + /// — they're already fully controllable in a test. + func testSaveLoadClearRoundTrip() { + XCTAssertFalse(CloudSyncSettings.isConfigured(info: [:])) + XCTAssertNil(CloudSyncSettings.serverURL) + XCTAssertNil(CloudSyncSettings.token) + + CloudSyncSettings.serverURL = "https://vk-noop-cloud.fly.dev" + CloudSyncSettings.token = "rw-secret" + XCTAssertEqual(CloudSyncSettings.serverURL, "https://vk-noop-cloud.fly.dev") + XCTAssertEqual(CloudSyncSettings.token, "rw-secret") + XCTAssertTrue(CloudSyncSettings.isConfigured(info: [:])) + + CloudSyncSettings.clear() + XCTAssertNil(CloudSyncSettings.serverURL) + XCTAssertNil(CloudSyncSettings.token) + XCTAssertFalse(CloudSyncSettings.isConfigured(info: [:])) + } + + /// Same pure-seam reasoning as `testSaveLoadClearRoundTrip` above: `info: [:]` so a real bundle + /// CLOUDSYNC_TOKEN on this machine can't make the URL-only state read as already-configured. + func testIsConfiguredRequiresBothValues() { + CloudSyncSettings.serverURL = "https://vk-noop-cloud.fly.dev" + XCTAssertFalse(CloudSyncSettings.isConfigured(info: [:])) + + CloudSyncSettings.token = "rw-secret" + XCTAssertTrue(CloudSyncSettings.isConfigured(info: [:])) + } + + func testEmptyStringClearsValue() { + CloudSyncSettings.serverURL = "https://vk-noop-cloud.fly.dev" + XCTAssertNotNil(CloudSyncSettings.serverURL) + + CloudSyncSettings.serverURL = " " + XCTAssertNil(CloudSyncSettings.serverURL) + } + + // MARK: - Bundle fallback (Phase 3.5: zero-touch — bundle-injected CLOUDSYNC_URL/CLOUDSYNC_TOKEN) + + /// Pure precedence logic, driven with synthetic dicts — mirrors how `OuraCredentialsTests` tests + /// `OuraCredentials.from(_:)` directly rather than `fromBundle` (which reads the real + /// `Bundle.main.infoDictionary` and can't be redirected in a unit test). + func testEffectiveValuePrefersKeychainOverBundle() { + let v = CloudSyncSettings.effectiveValue(keychain: "from-keychain", infoKey: "CLOUDSYNC_URL", + info: ["CLOUDSYNC_URL": "from-bundle"]) + XCTAssertEqual(v, "from-keychain") + } + + func testEffectiveValueFallsBackToBundleWhenKeychainNil() { + let v = CloudSyncSettings.effectiveValue(keychain: nil, infoKey: "CLOUDSYNC_URL", + info: ["CLOUDSYNC_URL": "from-bundle"]) + XCTAssertEqual(v, "from-bundle") + } + + func testEffectiveValueTrimsBundleValueAndTreatsBlankAsAbsent() { + XCTAssertNil(CloudSyncSettings.effectiveValue(keychain: nil, infoKey: "CLOUDSYNC_URL", + info: ["CLOUDSYNC_URL": " "])) + XCTAssertEqual(CloudSyncSettings.effectiveValue(keychain: nil, infoKey: "CLOUDSYNC_URL", + info: ["CLOUDSYNC_URL": " https://x "]), "https://x") + } + + func testEffectiveValueNilWhenNeitherKeychainNorBundlePresent() { + XCTAssertNil(CloudSyncSettings.effectiveValue(keychain: nil, infoKey: "CLOUDSYNC_URL", info: [:])) + } + + func testEffectiveURLAndTokenReturnKeychainValuesWhenSet() { + CloudSyncSettings.serverURL = "https://from-keychain.example.com" + CloudSyncSettings.token = "keychain-token" + XCTAssertEqual(CloudSyncSettings.effectiveURL, "https://from-keychain.example.com") + XCTAssertEqual(CloudSyncSettings.effectiveToken, "keychain-token") + XCTAssertTrue(CloudSyncSettings.isConfigured) + } + + /// No Keychain override present: `isBundleConfigured` must be false regardless of whatever the + /// test runner's own bundle happens to carry for CLOUDSYNC_URL/CLOUDSYNC_TOKEN (never set in the + /// StrandTests bundle in practice), because a real Keychain override always wins the UI's + /// collapsed-vs-manual-fields decision. + func testIsBundleConfiguredFalseWhenKeychainOverridePresent() { + CloudSyncSettings.serverURL = "https://from-keychain.example.com" + CloudSyncSettings.token = "keychain-token" + XCTAssertFalse(CloudSyncSettings.isBundleConfigured) + } + + /// Uses `isBundleConfigured(info:)` with an explicit EMPTY info dict — the pure seam — rather than + /// the live, no-arg `isBundleConfigured`. This machine's `OuraSecrets.xcconfig` carries real + /// CLOUDSYNC_URL/CLOUDSYNC_TOKEN values, which reach `Bundle.main.infoDictionary` at test-run + /// time, so the live property now genuinely reads true here — asserting against it directly would + /// make this test fail specifically BECAUSE the zero-touch bundle credentials are configured + /// correctly. `info: [:]` tests the "no bundle creds" branch deterministically instead. + func testIsBundleConfiguredFalseWithNoKeychainAndNoBundleCreds() { + // Fresh state (clear()'d in setUp): no Keychain values, and a synthetic empty info dict, so + // this must read false, not silently true. + XCTAssertFalse(CloudSyncSettings.isBundleConfigured(info: [:])) + } + + // MARK: - Bundle URL validation (fold-in: a malformed CLOUDSYNC_URL is treated as absent) + + /// Pure, no `Bundle`/Keychain dependency — the same shape check `CloudSyncModel.saveSettings` + /// runs before a manually-entered URL is ever written to the Keychain. + func testIsValidServerURLAcceptsAbsoluteURLWithSchemeAndHost() { + XCTAssertTrue(CloudSyncSettings.isValidServerURL("https://vk-noop-cloud.fly.dev")) + XCTAssertTrue(CloudSyncSettings.isValidServerURL("http://localhost:8080")) + } + + func testIsValidServerURLRejectsMissingSchemeOrHost() { + XCTAssertFalse(CloudSyncSettings.isValidServerURL("not a url")) + XCTAssertFalse(CloudSyncSettings.isValidServerURL("")) + XCTAssertFalse(CloudSyncSettings.isValidServerURL("justapath")) + XCTAssertFalse(CloudSyncSettings.isValidServerURL("vk-noop-cloud.fly.dev")) // no scheme + XCTAssertFalse(CloudSyncSettings.isValidServerURL("https://")) // no host + } + + /// A malformed bundle URL must be treated as absent by BOTH `effectiveURL` and `isConfigured` — + /// not just the former. `isConfigured(info:)` derives its URL half from the same `effectiveValue` + /// call as `effectiveURL`, so this pins the two together: a build never ends up with the "Sync + /// now" button enabled (`isConfigured` true) while `effectiveURL` itself reads nil and the button + /// would immediately fail with "Add your noop-cloud server URL and token first." + func testIsConfiguredFalseWhenBundleURLIsMalformedEvenWithATokenPresent() { + XCTAssertFalse(CloudSyncSettings.isConfigured(info: ["CLOUDSYNC_URL": "not a url", + "CLOUDSYNC_TOKEN": "some-token"])) + } +} +#endif // CLOUD_SYNC diff --git a/StrandTests/CloudSyncCoordinatorTests.swift b/StrandTests/CloudSyncCoordinatorTests.swift new file mode 100644 index 0000000000..d5d545f78b --- /dev/null +++ b/StrandTests/CloudSyncCoordinatorTests.swift @@ -0,0 +1,121 @@ +// Tests the CLOUD_SYNC-gated cloud sync coordinator; compiled only when the flag is set +// (StrandTests shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +import WhoopStore +@testable import Strand + +/// A fetcher that serves canned edits + a latest seq, and can simulate an ack failure. Mirrors +/// OuraSyncCoordinatorTests's StubFetcher. +private final class StubCloudEditFetcher: CloudEditFetching { + var edits: [CloudEdit] = [] + var latestSeq = 0 + var failAck = false + private(set) var fetchSinceCalls: [Int] = [] + private(set) var ackedSeqs: [[Int]] = [] + + func fetchEdits(since: Int) async throws -> (edits: [CloudEdit], latestSeq: Int) { + fetchSinceCalls.append(since) + return (edits, latestSeq) + } + func ack(seqs: [Int]) async throws -> Int { + if failAck { throw StubAckError() } + ackedSeqs.append(seqs) + return seqs.count + } +} + +private struct StubAckError: Error {} + +/// Builds a minimal CloudEdit for a given kind + payload, with the rest of the envelope defaulted. +private func makeEdit(seq: Int, kind: String, payloadJSON: String, beforeJSON: String? = nil) -> CloudEdit { + CloudEdit(seq: seq, editId: "e\(seq)", kind: kind, payloadJSON: payloadJSON, beforeJSON: beforeJSON, + rationale: nil, appliedAt: 0, undoneBySeq: nil, ackedAt: nil) +} + +final class CloudSyncCoordinatorTests: XCTestCase { + func testAppliesEditsAcksAndAdvancesCursorOnlyAfterAck() async throws { + let store = try await WhoopStore.inMemory() + let fetcher = StubCloudEditFetcher() + // delete_metric_point applies cleanly even with nothing to delete (tombstone insert + a + // no-op DELETE both succeed), so this is a deterministic "applied" outcome. + fetcher.edits = [makeEdit(seq: 1, kind: "delete_metric_point", + payloadJSON: #"{"deviceId":"my-whoop","day":"2026-01-01","key":"steps"}"#)] + fetcher.latestSeq = 1 + + let before = try await store.cursor("cloud_edits") + XCTAssertNil(before) + + let summary = try await CloudSyncCoordinator.pull(store: store, client: fetcher) + + XCTAssertEqual(summary.applied, 1) + XCTAssertEqual(summary.appliedSeqs, [1]) + XCTAssertEqual(fetcher.ackedSeqs, [[1]]) + let after = try await store.cursor("cloud_edits") + XCTAssertEqual(after, 1) + + // A second pull starts from the advanced cursor, not from 0 again. + fetcher.edits = [] + fetcher.latestSeq = 1 + _ = try await CloudSyncCoordinator.pull(store: store, client: fetcher) + XCTAssertEqual(fetcher.fetchSinceCalls, [0, 1]) + } + + func testAckFailureKeepsCursorUnchanged() async throws { + let store = try await WhoopStore.inMemory() + let fetcher = StubCloudEditFetcher() + fetcher.edits = [makeEdit(seq: 5, kind: "delete_metric_point", + payloadJSON: #"{"deviceId":"my-whoop","day":"2026-01-01","key":"steps"}"#)] + fetcher.latestSeq = 5 + fetcher.failAck = true + + do { + _ = try await CloudSyncCoordinator.pull(store: store, client: fetcher) + XCTFail("expected the ack failure to propagate") + } catch { + // expected — ack threw, so pull() must throw rather than silently swallow it. + } + + // Cursor must stay unset: re-applying the same batch on retry is idempotent, but only if the + // cursor never advanced past an edit that was never actually acked. + let cursor = try await store.cursor("cloud_edits") + XCTAssertNil(cursor) + } + + func testNeedsAttentionSurfacesInSummaryAndStillAdvancesCursor() async throws { + let store = try await WhoopStore.inMemory() + let fetcher = StubCloudEditFetcher() + // adjust_sleep_bounds against a session that doesn't exist in the (empty) store — + // CloudEditApplier can't find it, so this lands in needsAttention, not applied. + fetcher.edits = [makeEdit(seq: 9, kind: "adjust_sleep_bounds", + payloadJSON: #"{"deviceId":"my-whoop","startTs":1700000000,"newStartTs":null,"newEndTs":null}"#)] + fetcher.latestSeq = 9 + + let summary = try await CloudSyncCoordinator.pull(store: store, client: fetcher) + + XCTAssertEqual(summary.needsAttention, 1) + XCTAssertEqual(summary.applied, 0) + // needsAttention rows are still "processed" — acked and cursor-advanced like any other seq. + XCTAssertEqual(fetcher.ackedSeqs, [[9]]) + let cursor = try await store.cursor("cloud_edits") + XCTAssertEqual(cursor, 9) + } + + func testEmptyFetchNoOps() async throws { + let store = try await WhoopStore.inMemory() + let fetcher = StubCloudEditFetcher() + fetcher.edits = [] + fetcher.latestSeq = 42 // server is ahead, but there's nothing THIS device hasn't seen + + let summary = try await CloudSyncCoordinator.pull(store: store, client: fetcher) + + XCTAssertEqual(summary.applied, 0) + XCTAssertEqual(summary.skipped, 0) + XCTAssertEqual(summary.needsAttention, 0) + XCTAssertTrue(summary.appliedSeqs.isEmpty) + XCTAssertTrue(fetcher.ackedSeqs.isEmpty) // no ack call at all + let cursor = try await store.cursor("cloud_edits") + XCTAssertNil(cursor) // nothing processed, so the cursor is left alone + } +} +#endif // CLOUD_SYNC diff --git a/StrandTests/CloudSyncUploaderTests.swift b/StrandTests/CloudSyncUploaderTests.swift new file mode 100644 index 0000000000..7b49bbb40b --- /dev/null +++ b/StrandTests/CloudSyncUploaderTests.swift @@ -0,0 +1,415 @@ +// Tests the CLOUD_SYNC-gated cloud sync uploader (Phase 3.5: zero-touch); compiled only when the flag +// is set (StrandTests shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +import WhoopStore +@testable import Strand + +/// A spy conforming to `CloudIngesting` that records how many times `ingest` was invoked — used by +/// the empty-store guard test to assert the network is never reached, without needing a real +/// `CloudSyncClient`/`OuraURLProtocolStub` at all (mirrors `StubCloudEditFetcher` in +/// `CloudSyncCoordinatorTests.swift`). +private final class SpyCloudIngester: CloudIngesting { + private(set) var ingestCallCount = 0 + func ingest(fileURL: URL) async throws -> (bytes: Int, latestDay: String?) { + ingestCallCount += 1 + return (0, nil) + } +} + +final class CloudSyncUploaderTests: XCTestCase { + override func setUp() { super.setUp(); OuraURLProtocolStub.reset() } + + private let baseURL = URL(string: "https://cloud.example.com")! + + private func makeClient() -> CloudSyncClient { + CloudSyncClient(baseURL: baseURL, token: "tok-upload", session: OuraURLProtocolStub.session()) + } + + /// A fake exporter that writes canned bytes to `dest` without touching the real on-disk database or + /// `StorePaths.defaultDatabasePath()` — see `CloudSyncUploader.Exporter`'s doc comment for why the + /// real `DataBackup.writeBackup` can't run against a `WhoopStore.inMemory()` test store. `capture` + /// lets a test observe exactly which temp URL the uploader chose (to assert on its location and + /// that it's gone afterward). + private func fakeExporter(bytes: Data = Data("fake-noopbak-bytes".utf8), + capture: @escaping (URL) -> Void = { _ in }) -> CloudSyncUploader.Exporter { + { _, dest in + capture(dest) + try? bytes.write(to: dest) + return .exported(dest) + } + } + + /// An in-memory store seeded with one `dailyMetric` row, so it clears the empty-store guard added + /// to `CloudSyncUploader.upload` (see the "Empty store guard" section below, which tests that guard + /// directly against an UNSEEDED store) and these pre-existing tests keep exercising what they + /// actually test — network/export behavior beyond that guard — instead of being short-circuited by it. + private func nonEmptyStore() async throws -> WhoopStore { + let store = try await WhoopStore.inMemory() + try await store.upsertDailyMetrics([ + DailyMetric(day: "2026-07-11", totalSleepMin: nil, efficiency: nil, deepMin: nil, remMin: nil, + lightMin: nil, disturbances: nil, restingHr: nil, avgHrv: nil, recovery: nil, + strain: nil, exerciseCount: nil), + ], deviceId: "devA") + return store + } + + // MARK: - Happy path + + func testUploadHappyPathSendsBearerAndOctetStreamAndParsesResponse() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, + body: #"{"ok":true,"bytes":12345,"latestDay":"2026-07-11"}"#.data(using: .utf8)!)] + let store = try await nonEmptyStore() + let client = makeClient() + var capturedDest: URL? + + let result = try await CloudSyncUploader.upload(store: store, client: client, + exporter: fakeExporter(capture: { capturedDest = $0 })) + + XCTAssertEqual(result.bytes, 12345) + XCTAssertEqual(result.latestDay, "2026-07-11") + + let req = try XCTUnwrap(OuraURLProtocolStub.requestedRequests.last) + XCTAssertEqual(req.httpMethod, "POST") + XCTAssertEqual(req.value(forHTTPHeaderField: "Authorization"), "Bearer tok-upload") + XCTAssertEqual(req.value(forHTTPHeaderField: "Content-Type"), "application/octet-stream") + XCTAssertTrue(req.url?.absoluteString.contains("/ingest") ?? false) + + // The temp file lived in Caches (never Documents) and was cleaned up afterward. + let dest = try XCTUnwrap(capturedDest) + let cachesDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + XCTAssertTrue(dest.path.hasPrefix(cachesDir.path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: dest.path)) + } + + // MARK: - Network failure + + func testUploadNon2xxThrowsBadResponse() async throws { + OuraURLProtocolStub.queue = [.init(status: 401, body: "nope".data(using: .utf8)!)] + let store = try await nonEmptyStore() + let client = makeClient() + + do { + _ = try await CloudSyncUploader.upload(store: store, client: client, exporter: fakeExporter()) + XCTFail("expected badResponse to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncError, CloudSyncError.badResponse(401, "nope")) + } + } + + func testUploadCleansUpTempFileEvenOnNetworkFailure() async throws { + OuraURLProtocolStub.queue = [.init(status: 500, body: Data())] + let store = try await nonEmptyStore() + let client = makeClient() + var capturedDest: URL? + + do { + _ = try await CloudSyncUploader.upload(store: store, client: client, + exporter: fakeExporter(capture: { capturedDest = $0 })) + XCTFail("expected badResponse to be thrown") + } catch { + // expected — asserted below via the cleaned-up temp file + } + let dest = try XCTUnwrap(capturedDest) + XCTAssertFalse(FileManager.default.fileExists(atPath: dest.path)) + } + + // MARK: - Export failure (never reaches the network) + + func testUploadExportFailurePropagatesAsTypedErrorAndNeverPosts() async throws { + let store = try await nonEmptyStore() + let client = makeClient() + let failingExporter: CloudSyncUploader.Exporter = { _, _ in .failure("disk full") } + + do { + _ = try await CloudSyncUploader.upload(store: store, client: client, exporter: failingExporter) + XCTFail("expected exportFailed to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncUploadError, CloudSyncUploadError.exportFailed("disk full")) + } + XCTAssertTrue(OuraURLProtocolStub.requestedRequests.isEmpty) // never reached the network + } + + // MARK: - Empty store guard (incident: a test-host's empty DB must never upload) + + /// Reproduces the production incident: an empty store (a fresh `WhoopStore.inMemory()` has zero + /// `dailyMetric` rows, same shape as the macOS TEST HOST's never-populated database) must be + /// refused BEFORE export, BEFORE the network — proven here with no real `CloudSyncClient` at all, + /// just a `SpyCloudIngester` that would record a call if `upload` ever reached it. + func testUploadRefusesEmptyStoreAndNeverExportsOrIngests() async throws { + let store = try await WhoopStore.inMemory() + let ingester = SpyCloudIngester() + var exporterCallCount = 0 + let countingExporter: CloudSyncUploader.Exporter = { _, dest in + exporterCallCount += 1 + return .exported(dest) + } + + do { + _ = try await CloudSyncUploader.upload(store: store, client: ingester, exporter: countingExporter) + XCTFail("expected emptyStore to be thrown") + } catch { + XCTAssertEqual(error as? CloudSyncUploadError, CloudSyncUploadError.emptyStore) + } + XCTAssertEqual(exporterCallCount, 0, "the export step must never run for an empty store") + XCTAssertEqual(ingester.ingestCallCount, 0, "ingest must never be called for an empty store") + } + + /// The mirror-image case: once the store has at least one `dailyMetric` row, upload proceeds + /// exactly as before (happy-path regression guard for the new empty-store check). + func testUploadProceedsWhenStoreHasData() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, + body: #"{"ok":true,"bytes":99,"latestDay":"2026-07-11"}"#.data(using: .utf8)!)] + let store = try await nonEmptyStore() + let client = makeClient() + + let result = try await CloudSyncUploader.upload(store: store, client: client, exporter: fakeExporter()) + XCTAssertEqual(result.bytes, 99) + } +} + +/// The 20h auto-sync gate, extracted as a pure `(lastRun, now) -> Bool` function so the threshold is +/// unit-testable without UserDefaults, a store, or a network. Mirrors `BackupSyncTests`' coverage of +/// `FolderBackup`'s pure day-gate helpers. +final class CloudSyncModelAutoSyncGateTests: XCTestCase { + func testIsAutoSyncDueNeverRunBefore() { + XCTAssertTrue(CloudSyncModel.isAutoSyncDue(lastRun: 0, now: 1_752_300_000)) + } + + func testIsAutoSyncDueRespectsTwentyHourThreshold() { + let now: TimeInterval = 1_752_300_000 + XCTAssertFalse(CloudSyncModel.isAutoSyncDue(lastRun: now - 19 * 3600, now: now)) // too soon + XCTAssertTrue(CloudSyncModel.isAutoSyncDue(lastRun: now - 20 * 3600, now: now)) // exactly due (>=) + XCTAssertTrue(CloudSyncModel.isAutoSyncDue(lastRun: now - 21 * 3600, now: now)) // overdue + } + + /// Incident guard: `autoSyncIfDue` must never fire inside an XCTest runner (the macOS TEST HOST + /// runs `StrandTests` inside the full `Staging.app` via `TEST_HOST`, so `RootView`'s launch `.task` + /// executes for real — see `CloudSyncModel.isRunningUnderXCTest`'s doc comment). This is + /// inherently env-dependent and can only be pinned from the TRUE side: every `StrandTests` run, + /// including this one, IS an XCTest run, so the helper reading true here is exactly what "detects + /// the test runner" means. There is no way to assert the false branch from inside XCTest itself. + func testIsRunningUnderXCTestIsTrueInThisTestRunner() { + XCTAssertTrue(CloudSyncModel.isRunningUnderXCTest) + } +} + +/// The push-telemetry hook on the upload path. This is the instrument for the page-replication +/// trial, so the properties that matter are the ones that keep the *measurement* honest: a record +/// per delivered sync, none for a sync that delivered nothing, and a WAL size that was sampled at a +/// moment when it means something. +final class CloudSyncUploaderTelemetryTests: XCTestCase { + override func setUp() { super.setUp(); OuraURLProtocolStub.reset() } + + private func makeClient() -> CloudSyncClient { + CloudSyncClient(baseURL: URL(string: "https://cloud.example.com")!, token: "tok", + session: OuraURLProtocolStub.session()) + } + + private func makeTelemetry() -> SyncPushTelemetry { + let url = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("push-telemetry-\(UUID().uuidString).json") + addTeardownBlock { try? FileManager.default.removeItem(at: url) } + return SyncPushTelemetry(url: url) + } + + private func nonEmptyStore() async throws -> WhoopStore { + let store = try await WhoopStore.inMemory() + try await store.upsertDailyMetrics([ + DailyMetric(day: "2026-07-11", totalSleepMin: nil, efficiency: nil, deepMin: nil, remMin: nil, + lightMin: nil, disturbances: nil, restingHr: nil, avgHrv: nil, recovery: nil, + strain: nil, exerciseCount: nil), + ], deviceId: "devA") + return store + } + + private func fakeExporter() -> CloudSyncUploader.Exporter { + { _, dest in try? Data("bytes".utf8).write(to: dest); return .exported(dest) } + } + + /// One record per successful upload, carrying the server's own byte count. `snapshotted` is true + /// with reason `full-ingest` because `/ingest` genuinely ships the whole database — this is the + /// baseline the replicator is measured against, and labelling it as anything else would make the + /// eventual comparison meaningless. + func testRecordsOneSnapshotObservationPerSuccessfulUpload() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, + body: #"{"ok":true,"bytes":153400000,"latestDay":"2026-07-11"}"#.data(using: .utf8)!)] + let telemetry = makeTelemetry() + + _ = try await CloudSyncUploader.upload(store: try await nonEmptyStore(), client: makeClient(), + exporter: fakeExporter(), telemetry: telemetry) + + let stats = telemetry.stats + XCTAssertEqual(stats.pushes, 1) + XCTAssertEqual(stats.snapshots, 1) + XCTAssertEqual(stats.bytesUploaded, 153_400_000) + XCTAssertEqual(stats.snapshotRate, 1.0, accuracy: 0.0001, + "today every /ingest IS a full snapshot; the baseline must say so") + XCTAssertEqual(stats.reasonCounts["full-ingest"], 1) + } + + /// A failed POST delivered no bytes. Counting it would understate cost-per-delivered-sync and + /// would make the trial's byte comparison against the replicator wrong in the replicator's favour. + func testDoesNotRecordWhenTheUploadFails() async throws { + OuraURLProtocolStub.queue = [.init(status: 507, body: Data())] + let telemetry = makeTelemetry() + + do { + _ = try await CloudSyncUploader.upload(store: try await nonEmptyStore(), client: makeClient(), + exporter: fakeExporter(), telemetry: telemetry) + XCTFail("a 507 must propagate") + } catch {} + + XCTAssertEqual(telemetry.stats.pushes, 0) + } + + /// The empty-store guard refuses before export or network. Nothing was pushed, so nothing is + /// recorded — otherwise a device that never syncs would still accumulate a push history. + func testDoesNotRecordWhenTheEmptyStoreGuardRefuses() async throws { + let telemetry = makeTelemetry() + + do { + _ = try await CloudSyncUploader.upload(store: try await WhoopStore.inMemory(), + client: makeClient(), + exporter: fakeExporter(), telemetry: telemetry) + XCTFail("an empty store must be refused") + } catch {} + + XCTAssertEqual(telemetry.stats.pushes, 0) + } + + /// Repeated uploads accumulate, and the gap between them is derived rather than passed in — the + /// push cadence is half the trial's answer (a snapshot every launch is fine at 1/day and fatal at + /// 8/day), and a caller that had to remember to supply it would eventually not. + func testAccumulatesAcrossPushesAndDerivesTheInterval() async throws { + OuraURLProtocolStub.queue = [ + .init(status: 200, body: #"{"ok":true,"bytes":10,"latestDay":"2026-07-11"}"#.data(using: .utf8)!), + .init(status: 200, body: #"{"ok":true,"bytes":20,"latestDay":"2026-07-12"}"#.data(using: .utf8)!), + ] + let telemetry = makeTelemetry() + let store = try await nonEmptyStore() + + _ = try await CloudSyncUploader.upload(store: store, client: makeClient(), + exporter: fakeExporter(), telemetry: telemetry) + _ = try await CloudSyncUploader.upload(store: store, client: makeClient(), + exporter: fakeExporter(), telemetry: telemetry) + + XCTAssertEqual(telemetry.stats.pushes, 2) + XCTAssertEqual(telemetry.stats.bytesUploaded, 30) + XCTAssertNil(telemetry.recentRecords.first?.secondsSinceLastPush) + XCTAssertNotNil(telemetry.recentRecords.last?.secondsSinceLastPush, + "the second push must carry a measured interval") + } + + /// Telemetry is an instrument, not a dependency. A caller that passes none still uploads. + func testUploadSucceedsWithNoTelemetry() async throws { + OuraURLProtocolStub.queue = [.init(status: 200, + body: #"{"ok":true,"bytes":7,"latestDay":"2026-07-11"}"#.data(using: .utf8)!)] + + let result = try await CloudSyncUploader.upload(store: try await nonEmptyStore(), + client: makeClient(), + exporter: fakeExporter(), telemetry: nil) + + XCTAssertEqual(result.bytes, 7) + } +} + +// MARK: - Liters fallback policy (pure) + +/// The 2026-08-03 fix: `uploaded == 0` is four states with opposite correct responses, and the +/// escalation to `/ingest` is a counted decision, not a reflex. These tests pin the classifier and +/// the escalation policy — pure functions over scalars, no replicator, no network, no LITERS +/// xcframework required. +final class LitersFallbackPolicyTests: XCTestCase { + override func setUp() { + super.setUp() + CloudSyncUploader.resetLitersRetryStreak() + } + + override func tearDown() { + CloudSyncUploader.resetLitersRetryStreak() + super.tearDown() + } + + // MARK: classification + + func testShippedBytesClassifyAsPushed() { + XCTAssertEqual( + CloudSyncUploader.classifyLitersPush(uploaded: 1, synced: true, + txid: 5, remoteTxid: 5, bytesUploaded: 65_789), + .pushed(bytes: 65_789)) + } + + /// The healthy no-op: lineage has content and the remote matches it exactly. The old code + /// answered this with a ~208 MB `/ingest` that then reset the lineage (observed 2026-08-03). + func testVerifiedInSyncIsSuccessNotFallback() { + XCTAssertEqual( + CloudSyncUploader.classifyLitersPush(uploaded: 0, synced: true, + txid: 27, remoteTxid: 27, bytesUploaded: 0), + .inSync) + } + + /// The vacuous case observed 2026-08-03 17:16, minutes after a server-side lineage reset: + /// `synced=true` at `txid=0` means both sides agree on NOTHING — no content is verifiably on + /// the server, so this must NOT be blessed as success (that would stamp `lastUploadToken` and + /// make the next sync skip the upload entirely). It must not trigger `/ingest` either — the + /// next liters push re-baselines. + func testVacuousSyncAtTxidZeroIsRetryable() { + let outcome = CloudSyncUploader.classifyLitersPush(uploaded: 0, synced: true, + txid: 0, remoteTxid: 0, bytesUploaded: 0) + guard case .retryable = outcome else { + return XCTFail("txid=0 vacuous sync must be retryable, got \(outcome)") + } + } + + func testCapturedButUnshippedIsRetryable() { + let outcome = CloudSyncUploader.classifyLitersPush(uploaded: 0, synced: false, + txid: 9, remoteTxid: 8, bytesUploaded: 0) + guard case .retryable = outcome else { + return XCTFail("captured-but-unshipped must be retryable, got \(outcome)") + } + } + + func testSyncedButRemoteBehindIsRetryable() { + // synced flag claiming true while remoteTxid lags txid is contradictory — never trust it + // as delivered. + let outcome = CloudSyncUploader.classifyLitersPush(uploaded: 0, synced: true, + txid: 9, remoteTxid: 7, bytesUploaded: 0) + guard case .retryable = outcome else { + return XCTFail("remote behind local must be retryable, got \(outcome)") + } + } + + // MARK: escalation + + /// Two retryable syncs throw-and-retry; the third runs `/ingest` anyway. Below-threshold + /// withholding protects the lineage from transient failures; the threshold itself keeps a + /// genuinely dead liters path flowing through the proven upload. + func testEscalationOpensAtThreshold() { + XCTAssertFalse(CloudSyncUploader.ingestEscalationAllowed(retryStreak: 1)) + XCTAssertFalse(CloudSyncUploader.ingestEscalationAllowed(retryStreak: 2)) + XCTAssertTrue(CloudSyncUploader.ingestEscalationAllowed(retryStreak: 3)) + XCTAssertTrue(CloudSyncUploader.ingestEscalationAllowed(retryStreak: 4), + "past-threshold streaks keep /ingest open — the streak only resets on a " + + "liters success, so a dead liters path ships every sync, not every third") + } + + func testStreakPersistsAndResets() { + XCTAssertEqual(CloudSyncUploader.litersRetryStreak(), 0) + XCTAssertEqual(CloudSyncUploader.bumpLitersRetryStreak(), 1) + XCTAssertEqual(CloudSyncUploader.bumpLitersRetryStreak(), 2) + XCTAssertEqual(CloudSyncUploader.litersRetryStreak(), 2, + "streak reads back from UserDefaults — it must survive a process relaunch, " + + "because background syncs are usually fresh processes") + CloudSyncUploader.resetLitersRetryStreak() + XCTAssertEqual(CloudSyncUploader.litersRetryStreak(), 0) + } + + func testRetryPendingErrorNamesTheAttemptAndTheReason() { + let error = CloudSyncUploadError.litersRetryPending(streak: 2, reason: "push threw: 503") + let text = error.errorDescription ?? "" + XCTAssertTrue(text.contains("attempt 2"), "user-facing text must carry the attempt count") + XCTAssertTrue(text.contains("retry"), "must say it will retry, not read as a dead end") + } +} +#endif // CLOUD_SYNC diff --git a/StrandTests/DeepBufferUploadPlanTests.swift b/StrandTests/DeepBufferUploadPlanTests.swift new file mode 100644 index 0000000000..e2d6901c9b --- /dev/null +++ b/StrandTests/DeepBufferUploadPlanTests.swift @@ -0,0 +1,133 @@ +// Tests the CLOUD_SYNC-gated deep-buffer upload planning logic; compiled only when the flag is set +// (StrandTests shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +@testable import Strand + +/// Pins `DeepBufferUploadPlan` — the pure rules that decide which bytes of the append-only +/// deep-buffer log have already been shipped. Every function under test is value-in/value-out, so +/// this whole file runs with no strap, no server, no filesystem and no `UserDefaults`. That matters +/// more here than usual: the log holds the ONLY copy of high-rate buffers the strap's ack-trim has +/// already freed, so a re-upload wastes VK's bandwidth but a SKIP silently destroys research data, +/// and neither shows up as an error anywhere. +final class DeepBufferUploadPlanTests: XCTestCase { + + // MARK: - nextRange + + func testNextRangeStartsAtZeroForAnUnseenGeneration() { + XCTAssertEqual(DeepBufferUploadPlan.nextRange(uploaded: 0, fileSize: 100, maxChunk: 1000), 0..<100) + } + + func testNextRangeResumesFromTheWatermark() { + // The whole point of the lane: bytes below the watermark are never read again. + XCTAssertEqual(DeepBufferUploadPlan.nextRange(uploaded: 60, fileSize: 100, maxChunk: 1000), 60..<100) + } + + func testNextRangeIsNilWhenCaughtUp() { + XCTAssertNil(DeepBufferUploadPlan.nextRange(uploaded: 100, fileSize: 100, maxChunk: 1000)) + XCTAssertNil(DeepBufferUploadPlan.nextRange(uploaded: 0, fileSize: 0, maxChunk: 1000)) + } + + func testNextRangeCapsAtMaxChunk() { + XCTAssertEqual(DeepBufferUploadPlan.nextRange(uploaded: 0, fileSize: 100, maxChunk: 40), 0..<40) + XCTAssertEqual(DeepBufferUploadPlan.nextRange(uploaded: 80, fileSize: 100, maxChunk: 40), 80..<100) + } + + // MARK: - sanitizedWatermark (the inode-reuse backstop) + + func testWatermarkAboveFileSizeResetsToZero() { + // Impossible for an append-only file, so the identity is lying: a recycled inode inside the + // same millisecond, or UserDefaults restored onto a different device. Re-uploading from 0 is + // idempotent server-side; resuming from a bogus offset would ship meaningless bytes. + XCTAssertEqual(DeepBufferUploadPlan.sanitizedWatermark(uploaded: 500, fileSize: 100), 0) + XCTAssertEqual(DeepBufferUploadPlan.nextRange(uploaded: 500, fileSize: 100, maxChunk: 1000), 0..<100) + } + + func testNegativeWatermarkResetsToZero() { + XCTAssertEqual(DeepBufferUploadPlan.sanitizedWatermark(uploaded: -1, fileSize: 100), 0) + } + + func testWatermarkExactlyAtFileSizeIsKept() { + // The caught-up case is legitimate, not corruption — it must NOT reset and re-ship the file. + XCTAssertEqual(DeepBufferUploadPlan.sanitizedWatermark(uploaded: 100, fileSize: 100), 100) + } + + // MARK: - lastCompleteLineEnd + + func testLastCompleteLineEndCutsATrailingPartialLine() { + // The writer appends concurrently, so a read can land mid-line. The partial tail is left for + // the next drain (by which time the line is finished), never shipped as broken JSONL. + let data = Data("{\"a\":1}\n{\"b\":2}\n{\"c\":".utf8) + XCTAssertEqual(DeepBufferUploadPlan.lastCompleteLineEnd(data), 16) + } + + func testLastCompleteLineEndOnAnExactBoundaryKeepsEverything() { + let data = Data("{\"a\":1}\n{\"b\":2}\n".utf8) + XCTAssertEqual(DeepBufferUploadPlan.lastCompleteLineEnd(data), data.count) + } + + func testLastCompleteLineEndIsZeroWithNoNewlineAtAll() { + // One in-progress line longer than the whole read — nothing complete to ship yet. + XCTAssertEqual(DeepBufferUploadPlan.lastCompleteLineEnd(Data("{\"a\":1".utf8)), 0) + XCTAssertEqual(DeepBufferUploadPlan.lastCompleteLineEnd(Data()), 0) + } + + func testLastCompleteLineEndWorksOnASlice() { + // `readLineAlignedChunk` hands this whatever `FileHandle.read` returned, and Data slices do not + // start at index 0 — an implementation using a raw Int index instead of `distance(from:to:)` + // passes every test above and corrupts exactly this case. + let full = Data("XXXX{\"a\":1}\n{\"b\":".utf8) + let slice = full.dropFirst(4) + XCTAssertEqual(DeepBufferUploadPlan.lastCompleteLineEnd(slice), 8) + } + + // MARK: - generationId + + func testGenerationIdDistinguishesInodes() { + let d = Date(timeIntervalSince1970: 1_752_600_000) + XCTAssertNotEqual(DeepBufferUploadPlan.generationId(creationDate: d, inode: 1), + DeepBufferUploadPlan.generationId(creationDate: d, inode: 2)) + } + + func testGenerationIdDistinguishesRecycledInodes() { + // The reason the creation date is in the id at all: rotation deletes `.jsonl.1`, freeing its + // inode for the filesystem to hand straight back to the next `.jsonl`. Same inode, different + // file — if the ids collided, the new generation would inherit the old one's watermark and + // skip real data. + XCTAssertNotEqual( + DeepBufferUploadPlan.generationId(creationDate: Date(timeIntervalSince1970: 1_752_600_000), inode: 7), + DeepBufferUploadPlan.generationId(creationDate: Date(timeIntervalSince1970: 1_752_600_060), inode: 7)) + } + + func testGenerationIdIsStableForTheSameFile() { + let d = Date(timeIntervalSince1970: 1_752_600_000.123) + XCTAssertEqual(DeepBufferUploadPlan.generationId(creationDate: d, inode: 42), + DeepBufferUploadPlan.generationId(creationDate: d, inode: 42)) + } + + func testGenerationIdSeparatesSubSecondCreations() { + // Millisecond precision, not seconds: a delete/create pair inside one second is exactly the + // case a seconds-resolution id would collapse. + XCTAssertNotEqual( + DeepBufferUploadPlan.generationId(creationDate: Date(timeIntervalSince1970: 1_752_600_000.100), inode: 7), + DeepBufferUploadPlan.generationId(creationDate: Date(timeIntervalSince1970: 1_752_600_000.900), inode: 7)) + } + + func testGenerationIdToleratesAMissingCreationDate() { + XCTAssertFalse(DeepBufferUploadPlan.generationId(creationDate: nil, inode: 5).isEmpty) + } + + // MARK: - prunedWatermarks + + func testPruneDropsGenerationsThatNoLongerExist() { + // Rotation destroys `.jsonl.1` permanently; without this the map grows one dead entry per + // rotation forever inside a UserDefaults value rewritten on every drain. + let pruned = DeepBufferUploadPlan.prunedWatermarks(["a": 10, "b": 20, "c": 30], live: ["b", "c"]) + XCTAssertEqual(pruned, ["b": 20, "c": 30]) + } + + func testPruneKeepsEverythingWhenAllGenerationsAreLive() { + XCTAssertEqual(DeepBufferUploadPlan.prunedWatermarks(["a": 1], live: ["a"]), ["a": 1]) + } +} +#endif // CLOUD_SYNC diff --git a/StrandTests/DeepBufferUploaderTests.swift b/StrandTests/DeepBufferUploaderTests.swift new file mode 100644 index 0000000000..9f3a6ba8c3 --- /dev/null +++ b/StrandTests/DeepBufferUploaderTests.swift @@ -0,0 +1,431 @@ +// Tests the CLOUD_SYNC-gated deep-buffer drain; compiled only when the flag is set (StrandTests +// shares the app's OuraConfig.xcconfig, so flag + creds arrive together). +#if CLOUD_SYNC +import XCTest +@testable import Strand + +/// A spy conforming to `DeepBufferIngesting` that records every chunk instead of reaching a network — +/// mirrors `SpyCloudIngester` in `CloudSyncUploaderTests.swift`. `failFrom` makes the Nth call onward +/// throw, so the watermark's failure semantics can be tested without a server. +private final class SpyDeepBufferClient: DeepBufferIngesting { + struct Call: Equatable { + let generation: String + let byteStart: Int + let byteEnd: Int + let compressed: Data + } + private(set) var calls: [Call] = [] + var failFrom: Int? + var reportDuplicate = false + /// Runs INSIDE an upload, standing in for the seconds a real network call takes. The only way to + /// exercise something rotating the log mid-drain — the window the identity re-check exists for. + var onUpload: (() -> Void)? + + func uploadDeepBuffer(generation: String, byteStart: Int, byteEnd: Int, + compressed: Data) async throws -> DeepBufferUploadReceipt { + onUpload?() + if let failFrom, calls.count >= failFrom { throw CloudSyncError.network("simulated failure") } + calls.append(Call(generation: generation, byteStart: byteStart, byteEnd: byteEnd, compressed: compressed)) + return DeepBufferUploadReceipt(storedBytes: compressed.count, lines: 1, duplicate: reportDuplicate) + } + + /// Every chunk's payload inflated and concatenated, in the order it was sent. + func reassembled() -> Data { + calls.reduce(into: Data()) { acc, c in + acc.append((try? (c.compressed as NSData).decompressed(using: .zlib) as Data) ?? Data()) + } + } +} + +/// In-memory watermarks, so a test never races the test host's real `UserDefaults` (`StrandTests` runs +/// inside the full app via TEST_HOST — see `CloudSyncModel.isRunningUnderXCTest`). +private final class MemoryWatermarks: DeepBufferWatermarkStoring { + var marks: [String: Int] + init(_ marks: [String: Int] = [:]) { self.marks = marks } + func watermarks() -> [String: Int] { marks } + func setWatermarks(_ m: [String: Int]) { marks = m } +} + +/// Behavioural tests for `DeepBufferUploader.drain` against REAL files in a temp directory — the +/// append-only + rotation semantics it exists to get right are filesystem semantics (`rename(2)` +/// preserving an inode is the load-bearing fact), so faking the filesystem would fake away the thing +/// under test. Only the network is stubbed. +final class DeepBufferUploaderTests: XCTestCase { + + private var dir: URL! + + override func setUpWithError() throws { + try super.setUpWithError() + dir = FileManager.default.temporaryDirectory + .appendingPathComponent("deepbuf-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: dir) + try super.tearDownWithError() + } + + // MARK: - Helpers + + private var liveURL: URL { dir.appendingPathComponent(DeepBufferUploadPlan.liveFilename) } + private var rotatedURL: URL { dir.appendingPathComponent(DeepBufferUploadPlan.rotatedFilename) } + + /// `n` JSONL lines shaped like the real log's, from `PuffinDeepBufferLog.appendIfDeepBuffer`. + private func lines(_ n: Int, startTs: Int = 1_752_600_000) -> Data { + var out = Data() + for i in 0.. .1 (rename preserves the inode), then a fresh live. + try FileManager.default.moveItem(at: liveURL, to: rotatedURL) + let genB = lines(4, startTs: 1_752_800_000) + try append(genB, to: liveURL) + + _ = await DeepBufferUploader.drain(client: client, watermarks: marks, directory: dir, + maxChunks: 100, maxChunkBytes: 10_000) + + let generations = Set(client.calls.map(\.generation)) + XCTAssertEqual(generations.count, 2, "the rotated file and the fresh file must be distinct generations") + + // A's identity survived the rename, so its tail resumed at the watermark rather than restarting. + let aCalls = client.calls.filter { $0.generation == client.calls[0].generation } + XCTAssertEqual(aCalls.count, 2) + XCTAssertEqual(aCalls[1].byteStart, drainedFromA, "the rotated generation must resume, not restart") + XCTAssertEqual(aCalls[1].byteEnd, genA.count) + + // B is a different file and must start from 0 — not inherit A's offset. + let bCalls = client.calls.filter { $0.generation != client.calls[0].generation } + XCTAssertEqual(bCalls.first?.byteStart, 0, "a fresh generation must start at 0") + + let inflatedA = aCalls.reduce(into: Data()) { $0.append((try! ($1.compressed as NSData).decompressed(using: .zlib)) as Data) } + XCTAssertEqual(inflatedA, genA, "generation A must arrive whole across the rotation") + } + + func testRotationDuringADrainCannotSpliceTheWrongGenerationsBytes() async throws { + // `testRotationResumes…` rotates BETWEEN drains, which is the easy case and cannot catch this. + // The real gap is INSIDE one drain: `scanGenerations` binds identity to (creationDate, inode), + // but the read re-opens by PATH, and between them sits a network upload — seconds. A rotation + // there leaves the path valid and pointing at a DIFFERENT file, so the read silently returns + // generation B's bytes, which get stapled into A's archive under A's id AND acked — advancing + // A's watermark past a tail that was never uploaded. Two silent corruptions, no error raised, + // of the only copy of buffers the strap's ack-trim has already freed. + let genA = lines(8) + try append(genA, to: liveURL) + let client = SpyDeepBufferClient(), marks = MemoryWatermarks() + + // Rotate at the moment the first chunk is in flight — exactly when a 60 MB cap would trip. + client.onUpload = { [self] in + client.onUpload = nil // once + try? FileManager.default.moveItem(at: liveURL, to: rotatedURL) + try? append(lines(8, startTs: 1_752_900_000), to: liveURL) // a fresh, DIFFERENT live file + } + + _ = await DeepBufferUploader.drain(client: client, watermarks: marks, directory: dir, + maxChunks: 4, maxChunkBytes: 400) + + // The invariant that was broken: every chunk stored under a generation must BE that + // generation's bytes at the offsets it claims. genA is the only generation with a known + // ground truth here (the post-rotation live file is a different, freshly-written one). + XCTAssertFalse(client.calls.isEmpty, "precondition: the drain must actually ship something") + let aGen = client.calls[0].generation + let aCalls = client.calls.filter { $0.generation == aGen }.sorted { $0.byteStart < $1.byteStart } + for c in aCalls { + let inflated = (try! (c.compressed as NSData).decompressed(using: .zlib)) as Data + XCTAssertEqual(inflated, genA.subdata(in: c.byteStart.. DailyMetric { + DailyMetric(day: key, totalSleepMin: nil, efficiency: nil, deepMin: nil, remMin: nil, + lightMin: nil, disturbances: nil, restingHr: nil, avgHrv: hrv, recovery: nil, + strain: nil, exerciseCount: nil, spo2Pct: spo2, skinTempDevC: skinTemp, respRateBpm: nil) + } + + // MARK: - The coalesce order: today wins, then the whole-row carry, then the per-field carry + + func testTodaysOwnValueWinsOverEitherCarry() { + XCTAssertEqual(LiquidTodayView.perFieldVital(today: 97, vitalsCarry: 95, fieldCarry: 90), 97, + "today's own reading must never be displaced by a carry") + } + + func testWholeRowVitalsCarryWinsOverThePerFieldCarry() { + // When last night's vitals row ALSO carries the field, that fresher night beats the older per-field + // row — today-first, then freshest-row-first, never the reverse. + XCTAssertEqual(LiquidTodayView.perFieldVital(today: nil, vitalsCarry: 95, fieldCarry: 90), 95) + } + + // MARK: - THE regression: the per-field carry is the only thing keeping the tile off "–" + + func testPerFieldCarrySurfaces_whenTodayAndTheWholeRowCarryAreBothNil() { + // The exact rollover shape: today's row is empty and the whole-row vitals carry landed on a computed + // "-noop" row whose spo2Pct/skinTempDevC is engine-written nil. Before the fix Liquid stopped at + // `nil ?? nil` and blanked; the per-field carry supplies the real reading. + XCTAssertEqual(LiquidTodayView.perFieldVital(today: nil, vitalsCarry: nil, fieldCarry: 96), 96, + "a null today AND null whole-row field must fall through to the per-field carry") + } + + func testNothingAnywhereStaysNil_soTheTileHonestlyShowsNoData() { + XCTAssertNil(LiquidTodayView.perFieldVital(today: nil, vitalsCarry: nil, fieldCarry: nil), + "no reading anywhere is honest No-Data — never a fabricated 0") + } + + // MARK: - End-to-end rollover, composed with the real selectors (mirrors the classic behavior) + + func testSpo2Rollover_fallsThroughTheNullVitalsRowToTheImportedReading() { + // Post-rollover: today empty; last night's computed row has HRV but spo2Pct = nil (so the WHOLE-ROW + // lastVitalsDay lands on it with a null SpO₂); an older imported row holds a real 96%. + let days = [day("2026-07-13", spo2: 96), // imported, real reading + day("2026-07-14", hrv: 42, spo2: nil), // last night: vitals yes, SpO₂ nil + day("2026-07-15")] // today, empty + let tkey = "2026-07-15" + let vitals = Repository.lastVitalsDay(days: days, todayKey: tkey) + XCTAssertEqual(vitals?.day, "2026-07-14", "the whole-row carry lands on last night (it has HRV)…") + XCTAssertNil(vitals?.spo2Pct, "…whose SpO₂ is nil — this is exactly what blanked the tile") + let spo2 = LiquidTodayView.perFieldVital( + today: days.last?.spo2Pct, + vitalsCarry: vitals?.spo2Pct, + fieldCarry: Repository.lastSpo2Day(days: days, todayKey: tkey)?.spo2Pct) + XCTAssertEqual(spo2, 96, "SpO₂ must resolve to the imported reading, not blank on the vitals row's nil") + } + + func testSkinTempRollover_fallsThroughToThePriorDeviation() { + let days = [day("2026-07-13", skinTemp: -0.3), // prior reading (signed deviation from baseline) + day("2026-07-14", hrv: 42), // last night: vitals, no skin temp + day("2026-07-15")] // today, empty + let tkey = "2026-07-15" + let vitals = Repository.lastVitalsDay(days: days, todayKey: tkey) + XCTAssertNil(vitals?.skinTempDevC, "the whole-row carry row has no skin temp") + let skin = LiquidTodayView.perFieldVital( + today: days.last?.skinTempDevC, + vitalsCarry: vitals?.skinTempDevC, + fieldCarry: Repository.lastSkinTempDay(days: days, todayKey: tkey)?.skinTempDevC) + XCTAssertEqual(skin, -0.3, "skin temp must carry the prior day's deviation, not blank") + } +} diff --git a/StrandTests/LitersRoundTripTests.swift b/StrandTests/LitersRoundTripTests.swift new file mode 100644 index 0000000000..b4a57e0b5b --- /dev/null +++ b/StrandTests/LitersRoundTripTests.swift @@ -0,0 +1,414 @@ +// The liters round trip, run against real SQLite files on disk. +// +// Gated on BOTH flags. CLOUD_SYNC because this is fork-local sync work; LITERS because the symbols +// come from Liters.xcframework, which `Rust/build-ios.sh` produces and which is deliberately not +// tracked (see Config/Liters.xcconfig). A checkout that has never built the Rust side compiles none +// of this rather than failing to link. +#if CLOUD_SYNC && LITERS +import XCTest +import SQLite3 +import WhoopProtocol +import WhoopStore +@testable import Strand + +/// End-to-end proof that the Rust archive in `Liters.xcframework` actually replicates a database, +/// rather than merely linking. +/// +/// ## Why these tests open SQLite through `import SQLite3` and not GRDB +/// +/// The single load-bearing decision in `Rust/Cargo.toml` is `default-features = false`, which makes +/// liters link the *platform* `libsqlite3` instead of compiling its own copy of the amalgamation. +/// The reason is that two SQLite libraries in one process do not share the process-global +/// `unixInodeInfo` table SQLite uses to work around POSIX's "close any descriptor, lose all locks" +/// rule, so either can silently drop the other's advisory locks — and liters' correctness rests on a +/// long-running read lock. +/// +/// `import SQLite3` resolves to exactly the `libsqlite3.dylib` GRDB links through +/// `.systemLibrary(name: "CSQLite")`. So these tests are not a stand-in for the app's arrangement; +/// they *are* it: one process, one SQLite, opened from Swift on one side and from Rust on the other, +/// against the same file. A test written through GRDB would prove the same thing less directly and +/// drag a package dependency into a target that does not otherwise need one. +/// +/// ## Why a real file and not an in-memory database +/// +/// liters replicates WAL frames. An in-memory database has no WAL, no `-wal` sidecar, and no file +/// locks, so it would exercise none of the machinery under test and would pass whether or not the +/// linkage were correct. +final class LitersRoundTripTests: XCTestCase { + + private var dir: URL! + + override func setUpWithError() throws { + try super.setUpWithError() + dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("liters-rt-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + if let dir { try? FileManager.default.removeItem(at: dir) } + try super.tearDownWithError() + } + + private var originPath: String { dir.appendingPathComponent("origin.sqlite").path } + private var bucketPath: String { dir.appendingPathComponent("bucket", isDirectory: true).path } + private var replicaPath: String { dir.appendingPathComponent("replica.sqlite").path } + + // MARK: - The writer, which is the half the phone is + + /// A push captures the database's committed content and uploads it. + /// + /// This is the whole premise in one test: if the Swift bindings, the UniFFI scaffolding, the Rust + /// archive, and the platform `libsqlite3` are all wired correctly, a database written through + /// Swift's SQLite becomes LTX files in a bucket that only Rust ever wrote. Any break in that + /// chain fails here rather than three integration layers later. + func testWriterPushesCommittedContent() throws { + try makeOriginDatabase(rows: 1...50) + // Establishes the baseline the replica half will be compared against once liters' restore + // works under system SQLite (see testReplicaRestoreIsBrokenUnderSystemSQLite), and proves the + // fixture really is 50 committed rows rather than an empty file liters would happily push. + XCTAssertEqual(try readIds(at: originPath), Array(1...50)) + + let writer = try LitersWriter(dbPath: originPath, storage: .dir(path: bucketPath)) + defer { writer.close() } + let push = try writer.push() + + XCTAssertTrue(push.synced, "the first push must capture the database's committed content") + XCTAssertGreaterThan(push.uploaded, 0, "the first push must upload at least one LTX file") + XCTAssertGreaterThan(push.bytesUploaded, 0) + XCTAssertGreaterThan(push.txid, 0) + XCTAssertGreaterThan(push.remoteTxid, 0, "the bucket must know about the push that just happened") + } + + /// The second push must ship a WAL delta, not another copy of the database. + /// + /// `snapshotted` is the number the whole trial exists to measure. A replicator that snapshots on + /// every push is a full upload with extra steps, which is precisely what the current `/ingest` + /// path already does. Asserting it is `false` for a plain "write more rows, push again" is the + /// minimum bar; anything less and there is no reason to adopt this at all. + func testASecondPushIsIncrementalAndNotASnapshot() throws { + try makeOriginDatabase(rows: 1...10) + + let writer = try LitersWriter(dbPath: originPath, storage: .dir(path: bucketPath)) + defer { writer.close() } + let first = try writer.push() + XCTAssertTrue(first.synced) + + try appendRows(11...20, to: originPath) + let second = try writer.push() + + XCTAssertFalse(second.snapshotted, + "an ordinary follow-up push must ship a WAL delta. reason: \(second.snapshotReason ?? "nil")") + XCTAssertGreaterThan(second.txid, first.txid) + } + + // MARK: - The `/ingest` fallback, which shares the file with the replicator + + /// **The regression this group exists for**, and the measurement that decided its shape. + /// + /// `CloudSyncUploader`'s `/ingest` fallback used to checkpoint the live store before archiving + /// it. That checkpoint runs on exactly the syncs where the page-replication push did not — a push + /// that failed to open a writer at all, or a launch where the app foregrounds and backs up before + /// it syncs — so it lands in the one window where no replicator read lock is holding the WAL. + /// + /// Both modes are asserted because the gentler one is the trap. `FULL` leaves the `-wal` file at + /// its full size, which looks harmless and is not: once the WAL is fully backfilled and no reader + /// needs those frames, SQLite restarts it on the **next write transaction**, new salt and all. + /// The replicator's resume offset is just as gone as after a `TRUNCATE`. + /// + /// This is VK's device failure reproduced: 2026-07-28, `/ingest` at 06:03, then a push at 14:23 + /// reporting `wal truncated by another process` and uploading 640 MB. + func testAnyForeignCheckpointCostsTheNextPushASnapshot() throws { + for mode in ["TRUNCATE", "FULL"] { + let scratch = dir.appendingPathComponent("ckpt-\(mode)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + let db = scratch.appendingPathComponent("origin.sqlite").path + let bucket = scratch.appendingPathComponent("bucket", isDirectory: true).path + + try makeDatabase(at: db, rows: 1...200) + let first = try LitersWriter(dbPath: db, storage: .dir(path: bucket)) + _ = try first.push() + try appendRows(201...210, to: db) + // Closed, because that is the window the fallback's checkpoint actually runs in. With the + // writer open its read lock blocks a foreign checkpoint outright — which protects the + // replicator but leaves the archive missing rows, so it is not a fix either. + first.close() + + try foreignCheckpoint(mode, at: db) + // The next write is what converts a backfilled WAL into a restarted one. + try appendRows(211...220, to: db) + + let second = try LitersWriter(dbPath: db, storage: .dir(path: bucket)) + defer { second.close() } + let push = try second.push() + + XCTAssertTrue(push.snapshotted, + "\(mode): a completed foreign checkpoint must cost the replicator its resume " + + "point — if this stops being true, liters changed and the staged-copy export " + + "may no longer be needed") + print("[liters] foreign \(mode): snapshotted=\(push.snapshotted) " + + "reason=\(push.snapshotReason ?? "nil") bytes=\(push.bytesUploaded)") + } + } + + /// **The fix, end to end.** The identical sequence, with the export staged through + /// `WhoopStore.writeConsistentCopy(to:)` — SQLite's Online Backup API, which reads through the + /// WAL and never checkpoints — instead of a checkpoint. The next push ships a delta. + /// + /// Through a real `WhoopStore` rather than raw `SQLite3` on purpose: GRDB's `DatabasePool` is + /// what actually holds this file in the app, it links the same platform `libsqlite3` liters does + /// (see the type doc), and the copy has to survive a pool's connections, not a single handle's. + func testAStagedConsistentCopyDoesNotCostTheNextPushASnapshot() async throws { + let db = dir.appendingPathComponent("store.sqlite").path + let store = try await WhoopStore(path: db, walCheckpointing: .external, walBackstop: .disabled) + try await insertHR(store, count: 20) + + let first = try LitersWriter(dbPath: db, storage: .dir(path: bucketPath)) + let firstPush = try first.push() + XCTAssertTrue(firstPush.synced) + try await insertHR(store, count: 2, startingAt: 20 * 400) + first.close() + + // The export step, in the same window the checkpoint above ran in. + let staged = dir.appendingPathComponent("staged.sqlite").path + try await store.writeConsistentCopy(to: staged) + try await insertHR(store, count: 2, startingAt: 22 * 400) + + let second = try LitersWriter(dbPath: db, storage: .dir(path: bucketPath)) + defer { second.close() } + let push = try second.push() + + XCTAssertFalse(push.snapshotted, + "staging a copy must leave the replicator's resume point intact. reason: " + + "\(push.snapshotReason ?? "nil")") + XCTAssertGreaterThan(push.txid, firstPush.txid) + // The size claim, against the only honest yardstick available: the FIRST push, which was a + // snapshot of this same database. Comparing against the `.sqlite` file's size would be + // meaningless here — on a replicated store nothing checkpoints, so almost the whole database + // lives in the `-wal` and the main file stays near empty. + XCTAssertLessThan(push.bytesUploaded, firstPush.bytesUploaded / 2, + "a delta must be a fraction of the snapshot it follows") + print("[liters] staged copy: snapshotted=\(push.snapshotted) bytes=\(push.bytesUploaded) " + + "vs first-push snapshot \(firstPush.bytesUploaded)B") + } + + // MARK: - The iOS-shaped failure mode + + /// Close the writer, reopen it, push. This is what iOS does to the app on every relaunch, and the + /// design document names it as the one thing that must be measured before committing: + /// + /// > the issue-#927 "expected truncation" shortcut is deliberately not trusted across an app + /// > restart — which on iOS is every single time. + /// + /// The test does not assert `snapshotted == false`, because a snapshot here would be *correct* + /// behaviour, not a bug — liters is refusing to trust a stale shortcut. It asserts the weaker, + /// genuinely required property: a reopened writer resumes against the same bucket and the replica + /// still converges. Whether the reopen costs a snapshot in practice is what + /// `SyncPushTelemetry.snapshotRate` measures on VK's device; recording the observed value here + /// documents which way it fell on this platform. + func testWriterReopenResumesAgainstTheSameBucket() throws { + try makeOriginDatabase(rows: 1...10) + + let first = try LitersWriter(dbPath: originPath, storage: .dir(path: bucketPath)) + let firstPush = try first.push() + first.close() + + try appendRows(11...25, to: originPath) + + let second = try LitersWriter(dbPath: originPath, storage: .dir(path: bucketPath)) + defer { second.close() } + let secondPush = try second.push() + + XCTAssertGreaterThan(secondPush.txid, firstPush.txid, + "a reopened writer must resume from the bucket, not start over at txid 0") + XCTAssertTrue(secondPush.synced, "the commits made while closed must be captured on reopen") + + // Not an assertion: the measurement. Printed so a CI log records which way the reopen fell. + print("[liters] reopen push snapshotted=\(secondPush.snapshotted) reason=\(secondPush.snapshotReason ?? "nil")") + } + + // MARK: - The replica, which is the half the SERVER is, and which system SQLite breaks + + /// **This test asserts a bug, on purpose.** When it starts failing, liters has been fixed and the + /// assertion below should be inverted into the real round trip (restore, then compare + /// `readIds(at: replicaPath)` against the origin's rows). + /// + /// `Replica.sync()` cannot restore when liters is linked against the platform `libsqlite3` — + /// which is the configuration NOOP ships, and the whole point of `default-features = false`. + /// The cause is `crates/liters/src/replica.rs:648`, the only read-only open in the crate, reached + /// only from `check_integrity` after a restore: + /// + /// ```rust + /// rusqlite::Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + /// ``` + /// + /// Apple's libsqlite3 (3.51.0) cannot run a statement on a WAL-mode database opened + /// `SQLITE_OPEN_READONLY` when the `-shm` sidecar is absent — and the restore deletes `-wal`/ + /// `-shm` immediately before this call. Reduced to a four-case C program against + /// `/usr/lib/libsqlite3.dylib`; opening the same file READWRITE, or leaving the sidecars in + /// place, succeeds. liters' own suite passes 80/80 with its bundled SQLite and 31/80 without, + /// and every one of those 49 failures is on this path. + /// + /// **It does not block the phone.** NOOP is a Writer; `writer.rs` contains no read-only open at + /// all, and the tests above prove the writer half works in exactly this linkage. The Replica runs + /// on the server, in a Rust process with no GRDB, which therefore builds liters with bundled + /// SQLite and never reaches this. The test exists so that "we knew, and here is the boundary" + /// is a property of the codebase rather than of someone's memory. + func testReplicaRestoreIsBrokenUnderSystemSQLite() throws { + try makeOriginDatabase(rows: 1...10) + let writer = try LitersWriter(dbPath: originPath, storage: .dir(path: bucketPath)) + defer { writer.close() } + _ = try writer.push() + + let replica = try LitersReplica(dbPath: replicaPath, storage: .dir(path: bucketPath), autoReset: true) + defer { replica.close() } + + XCTAssertThrowsError(try replica.sync(), "if this no longer throws, liters is fixed — invert this test") { error in + XCTAssertTrue("\(error)".contains("unable to open database file"), + "expected the known SQLITE_CANTOPEN from check_integrity, got: \(error)") + } + } + + // MARK: - Errors cross the FFI as errors + + /// A failure has to arrive in Swift as a thrown `LitersError`, not a crash and not a silent + /// success. UniFFI's error mapping is generated code, so this checks the generated code works for + /// the one path the app will actually hit at 3am. + func testOpeningAMissingDatabaseThrows() { + let missing = dir.appendingPathComponent("nope/absent.sqlite").path + XCTAssertThrowsError(try LitersWriter(dbPath: missing, storage: .dir(path: bucketPath))) + } + + // MARK: - SQLite helpers (the platform libsqlite3, the same one GRDB links) + + /// Creates a WAL-mode database and inserts `rows`, through Apple's system SQLite. + private func makeOriginDatabase(rows: ClosedRange) throws { + try makeDatabase(at: originPath, rows: rows) + } + + private func makeDatabase(at path: String, rows: ClosedRange) throws { + var db: OpaquePointer? + let rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nil) + defer { sqlite3_close(db) } + guard rc == SQLITE_OK, db != nil else { + throw SQLiteFailure(code: rc, message: "open \(path)") + } + + // WAL is not incidental. liters replicates WAL frames, so a rollback-journal database would + // have nothing for it to ship and the round trip would prove nothing. + let mode = try queryString(db, "PRAGMA journal_mode = WAL;") + guard mode == "wal" else { throw SQLiteFailure(code: -1, message: "journal_mode is \(mode)") } + try exec(db, "CREATE TABLE sample (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);") + try insert(db, rows) + } + + /// `PRAGMA wal_checkpoint()` from a SECOND connection, which is what + /// `WhoopStore.checkpointWALForBackup()` is from liters' point of view: a foreign checkpointer on + /// the same file, while the writer holds its read lock. Returns SQLite's own three-integer answer + /// — `busy` is 1 when a reader stopped the checkpoint before it finished, and `log`/`checkpointed` + /// are frames present versus frames transferred into the database file. + @discardableResult + private func foreignCheckpoint(_ mode: String, at path: String) throws + -> (busy: Int, log: Int, checkpointed: Int) { + var db: OpaquePointer? + let rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, nil) + defer { sqlite3_close(db) } + guard rc == SQLITE_OK, db != nil else { throw SQLiteFailure(code: rc, message: "reopen \(path)") } + + var stmt: OpaquePointer? + let sql = "PRAGMA wal_checkpoint(\(mode));" + let prepared = sqlite3_prepare_v2(db, sql, -1, &stmt, nil) + guard prepared == SQLITE_OK else { throw SQLiteFailure(code: prepared, message: sql) } + defer { sqlite3_finalize(stmt) } + guard sqlite3_step(stmt) == SQLITE_ROW else { throw SQLiteFailure(code: -1, message: "no row from \(sql)") } + return (Int(sqlite3_column_int(stmt, 0)), + Int(sqlite3_column_int(stmt, 1)), + Int(sqlite3_column_int(stmt, 2))) + } + + /// Runs a statement that returns one text column. `PRAGMA journal_mode = WAL` is such a + /// statement — it reports the mode actually in force, which is the only way to know the switch + /// took (it silently stays in rollback mode inside a transaction, for instance). + private func queryString(_ db: OpaquePointer?, _ sql: String) throws -> String { + var stmt: OpaquePointer? + let rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nil) + guard rc == SQLITE_OK else { throw SQLiteFailure(code: rc, message: sql) } + defer { sqlite3_finalize(stmt) } + guard sqlite3_step(stmt) == SQLITE_ROW, let text = sqlite3_column_text(stmt, 0) else { + throw SQLiteFailure(code: -1, message: "no row from \(sql)") + } + return String(cString: text) + } + + /// Writes through a SECOND connection while the liters writer holds its read lock open. That is + /// the app's real arrangement — GRDB writing while the replicator watches — and it is the case + /// the read lock is designed to allow: it blocks foreign *checkpoints*, not foreign writes. + private func appendRows(_ rows: ClosedRange, to path: String) throws { + var db: OpaquePointer? + let rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, nil) + defer { sqlite3_close(db) } + guard rc == SQLITE_OK, db != nil else { throw SQLiteFailure(code: rc, message: "reopen \(path)") } + try insert(db, rows) + } + + private func insert(_ db: OpaquePointer?, _ rows: ClosedRange) throws { + try exec(db, "BEGIN;") + for i in rows { + // Padded so each row is big enough that a batch spans more than one 4 KB page, which is + // what makes "the second push shipped a delta" a statement about pages and not about an + // empty WAL. + try exec(db, "INSERT INTO sample (id, payload) VALUES (\(i), '\(String(repeating: "x", count: 512))');") + } + try exec(db, "COMMIT;") + } + + /// Reads back every id, ascending. The assertion target for "did the bytes arrive". + /// One `WhoopStore` transaction per call, mirroring a `Collector` flush — the commit shape whose + /// *count*, not payload, is what fills the WAL. + private func insertHR(_ store: WhoopStore, count: Int, rowsEach: Int = 400, + startingAt ts0: Int = 0) async throws { + var ts = ts0 + for _ in 0.. [Int] { + var db: OpaquePointer? + // READWRITE, not READONLY, and that is not arbitrary: Apple's libsqlite3 cannot run a + // statement on a WAL-mode database opened SQLITE_OPEN_READONLY when the -shm sidecar is + // absent, which it is right after a liters restore. Opening read-only here would fail with + // SQLITE_CANTOPEN for reasons that have nothing to do with whether the bytes arrived. + let rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, nil) + defer { sqlite3_close(db) } + guard rc == SQLITE_OK, db != nil else { throw SQLiteFailure(code: rc, message: "open replica \(path)") } + + var stmt: OpaquePointer? + let prepared = sqlite3_prepare_v2(db, "SELECT id FROM sample ORDER BY id;", -1, &stmt, nil) + guard prepared == SQLITE_OK else { throw SQLiteFailure(code: prepared, message: "prepare select") } + defer { sqlite3_finalize(stmt) } + + var ids: [Int] = [] + while sqlite3_step(stmt) == SQLITE_ROW { ids.append(Int(sqlite3_column_int64(stmt, 0))) } + return ids + } + + private func exec(_ db: OpaquePointer?, _ sql: String) throws { + var err: UnsafeMutablePointer? + let rc = sqlite3_exec(db, sql, nil, nil, &err) + guard rc == SQLITE_OK else { + let message = err.map { String(cString: $0) } ?? sql + sqlite3_free(err) + throw SQLiteFailure(code: rc, message: message) + } + } + + private struct SQLiteFailure: Error, CustomStringConvertible { + let code: Int32 + let message: String + var description: String { "sqlite rc \(code): \(message)" } + } +} +#endif diff --git a/StrandTests/OuraURLProtocolStub.swift b/StrandTests/OuraURLProtocolStub.swift index 8bd46945d1..0db122b5c7 100644 --- a/StrandTests/OuraURLProtocolStub.swift +++ b/StrandTests/OuraURLProtocolStub.swift @@ -8,13 +8,19 @@ final class OuraURLProtocolStub: URLProtocol { nonisolated(unsafe) static var queue: [Stubbed] = [] /// Captured request URLs, in order, for assertions. nonisolated(unsafe) static var requestedURLs: [URL] = [] + /// Captured full requests (headers included), in order — for assertions that need more than the + /// URL, e.g. a bearer `Authorization` header or a `Content-Type` (CloudSyncUploaderTests). Upload + /// tasks built via `URLSession.upload(for:fromFile:)` still carry their headers on `request` even + /// though the body itself is streamed from disk rather than exposed here. + nonisolated(unsafe) static var requestedRequests: [URLRequest] = [] - static func reset() { queue = []; requestedURLs = [] } + static func reset() { queue = []; requestedURLs = []; requestedRequests = [] } override class func canInit(with request: URLRequest) -> Bool { true } override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } override func startLoading() { if let u = request.url { Self.requestedURLs.append(u) } + Self.requestedRequests.append(request) let s = Self.queue.isEmpty ? Stubbed(status: 500, body: Data()) : Self.queue.removeFirst() let resp = HTTPURLResponse(url: request.url!, statusCode: s.status, httpVersion: nil, headerFields: nil)! client?.urlProtocol(self, didReceive: resp, cacheStoragePolicy: .notAllowed) diff --git a/StrandTests/PuffinDeepBufferLogTests.swift b/StrandTests/PuffinDeepBufferLogTests.swift index 72855f072d..20697c4d5f 100644 --- a/StrandTests/PuffinDeepBufferLogTests.swift +++ b/StrandTests/PuffinDeepBufferLogTests.swift @@ -18,6 +18,29 @@ final class PuffinDeepBufferLogTests: XCTestCase { XCTAssertTrue(PuffinDeepBufferLog.isDeepBuffer(g)) } + func testHexStringMatchesFormatterForEveryByteValue() { + // Byte-for-byte parity with the `String(format: "%02x")` encoding it replaces, across the whole + // domain — the JSONL is a research archive, so a wrong nibble is silent corruption of the only + // copy of buffers the strap has already freed. + let all = (0...255).map { UInt8($0) } + XCTAssertEqual(PuffinDeepBufferLog.hexString(all), + all.map { String(format: "%02x", $0) }.joined()) + XCTAssertEqual(PuffinDeepBufferLog.hexString([]), "") + XCTAssertEqual(PuffinDeepBufferLog.hexString([0x00, 0x0F, 0xF0, 0xFF]), "000ff0ff") + } + + func testHexStringIsFastEnoughForTheAckPath() { + // Guards the regression that matters: this runs on the MainActor inside the offload ack path, so + // a per-byte Foundation formatter (~3 ms per 2140-B buffer) throttles the whole drain. Encoding + // one buffer must stay far under a millisecond. Threshold is deliberately loose — it's catching + // an 80x cliff, not micro-benchmarking. + let frame = (0..<2140).map { UInt8($0 & 0xFF) } + let start = Date() + for _ in 0..<100 { _ = PuffinDeepBufferLog.hexString(frame) } + let perBuffer = Date().timeIntervalSince(start) / 100 + XCTAssertLessThan(perBuffer, 0.001, "hex encode regressed to \(perBuffer * 1000) ms/buffer") + } + func testRejectsSmallType2FRecord() { // The ~124-B type-0x2F frame is the 1 Hz rollup NOOP already decodes — below the size gate, // it must NOT be logged as a high-rate buffer. diff --git a/StrandTests/RawHistoryArchiveEvictionTests.swift b/StrandTests/RawHistoryArchiveEvictionTests.swift index 80267d8405..e922b81b57 100644 --- a/StrandTests/RawHistoryArchiveEvictionTests.swift +++ b/StrandTests/RawHistoryArchiveEvictionTests.swift @@ -113,6 +113,123 @@ final class RawHistoryArchiveEvictionTests: XCTestCase { return "{\"capturedAtMs\":1,\"trim\":1,\"family\":\"\(family)\",\"frameHex\":\"\(hex)\"}\n" } + // MARK: - zero-payload frames lose retention priority + + /// A synthetic WHOOP 5/MG type-47 record: a non-zero 21-byte header (type @8 = 47, hist_version @9), + /// then `payloadBytes` of payload, then a 4-byte CRC trailer. + /// + /// `payloadByte: 0` reproduces the measured `enable_raw_data_w_ecg` placeholder — the strap banks one + /// such record per second and every byte from offset 21 to the CRC is zero. `marker` writes a single + /// distinguishing byte into the payload, which is also what makes a frame informative. + private func whoop5Frame(version: UInt8, payloadByte: UInt8, + payloadBytes: Int = 64, marker: UInt8? = nil) -> [UInt8] { + var f = [UInt8](repeating: 0x11, count: 21) // non-zero header (seq/ts/const bytes) + f[0] = 0xAA; f[8] = 47; f[9] = version + var payload = [UInt8](repeating: payloadByte, count: payloadBytes) + if let marker { payload[0] = marker } + f.append(contentsOf: payload) + f.append(contentsOf: [0xDE, 0xAD, 0xBE, 0xEF]) // CRC32 trailer (never part of the payload) + return f + } + + /// The strict test: entirely-zero payload region → yes; ONE non-zero byte anywhere → no. + func testZeroPayloadDetectionIsStrictAndHeaderAware() { + let empty = whoop5Frame(version: 22, payloadByte: 0) + XCTAssertTrue(RawHistoryArchive.hasZeroPayload(empty, family: .whoop5), + "the header and CRC bytes are non-zero but the PAYLOAD region is all zero") + // A single populated byte in an otherwise-empty buffer is exactly what this archive is for. + var oneByte = empty + oneByte[21 + 40] = 0x01 + XCTAssertFalse(RawHistoryArchive.hasZeroPayload(oneByte, family: .whoop5), + "'mostly zero' must NOT count as empty — one non-zero byte makes it informative") + // The last payload byte, immediately before the CRC trailer, is inside the region. + var lastByte = empty + lastByte[lastByte.count - 5] = 0x01 + XCTAssertFalse(RawHistoryArchive.hasZeroPayload(lastByte, family: .whoop5)) + // A payload region too short to judge is never called empty. + XCTAssertFalse(RawHistoryArchive.hasZeroPayload(whoop5Frame(version: 22, payloadByte: 0, + payloadBytes: 8), family: .whoop5)) + XCTAssertFalse(RawHistoryArchive.hasZeroPayload([0xAA, 0x01], family: .whoop5)) + } + + /// The DEFECT: with `enable_raw_data_w_ecg` armed the strap banks one all-zero record per second, so + /// a purely age-ordered archive evicts a real contact-window record within the half hour. An + /// informative frame must outrank an all-zero one of the SAME layout version, even when it is the + /// oldest line in the file and the empties are the newest. + func testZeroPayloadFramesAreEvictedBeforeInformativeOnes() { + let dir = tmpDir("zero"); defer { try? FileManager.default.removeItem(at: dir) } + let archive = RawHistoryArchive(directory: dir, maxBytes: 4_096, + perVersionFloor: 2, zeroPayloadFloor: 2) + + // Three informative records land FIRST — the oldest lines in the archive. Two would survive on + // the per-version floor alone; the third can only survive if the empties are evicted before it. + let informative = (0..<3).map { whoop5Frame(version: 22, payloadByte: 0x00, marker: UInt8(0xE0 + $0)) } + for f in informative { _ = archive.archive([f], trim: 1, family: .whoop5) } + XCTAssertFalse(RawHistoryArchive.hasZeroPayload(informative[0], family: .whoop5), + "precondition: a marker byte makes these frames informative") + + // Then the placeholder flood: same layout version, all-zero payload, newest in the file. + for _ in 0..<200 { _ = archive.archive([whoop5Frame(version: 22, payloadByte: 0)], + trim: 2, family: .whoop5) } + + let back = archive.readAll().map(\.frame) + let size = (try? FileManager.default.attributesOfItem(atPath: archive.fileURL.path)) + .flatMap { $0[.size] as? Int } ?? 0 + XCTAssertLessThanOrEqual(size, 4_096, "the archive must still respect its cap") + for (i, f) in informative.enumerated() { + XCTAssertTrue(back.contains(f), "informative frame #\(i) must outrank 200 empty records") + } + // …and the empties really were the thing evicted (a handful kept to date the artefact). + let zeros = back.filter { RawHistoryArchive.hasZeroPayload($0, family: .whoop5) } + XCTAssertLessThan(zeros.count, 200, "the placeholder flood must have been evicted, not the data") + } + + /// Same ordering in the pure core, with the informative line the OLDEST of its version so no floor + /// can be what saves it. + func testEvictLinesEvictsZeroPayloadBeforeInformative() { + let informative = archiveLine(whoop5Frame(version: 22, payloadByte: 0, marker: 0xE7)) + var lines = [informative] + for _ in 0..<200 { lines.append(archiveLine(whoop5Frame(version: 22, payloadByte: 0))) } + let kept = RawHistoryArchive.evictLines(lines, maxBytes: 4_096, floor: 2, zeroFloor: 2) + XCTAssertLessThanOrEqual(kept.reduce(0) { $0 + $1.utf8.count }, 4_096) + XCTAssertTrue(kept.contains(informative), + "the one informative line must survive a flood of all-zero-payload lines") + XCTAssertLessThan(kept.count, 200, "the zero-payload surplus must actually have been evicted") + } + + /// A version whose records are ALL empty still keeps `zeroFloor` dated samples — enough to prove the + /// artefact exists without letting it own the archive. + func testAllZeroVersionKeepsItsSmallFloor() { + var lines: [String] = [] + for _ in 0..<200 { lines.append(archiveLine(whoop5Frame(version: 22, payloadByte: 0))) } + for i in 0..<200 { lines.append(archiveLine(whoop5Frame(version: 18, payloadByte: 0, + marker: UInt8(i & 0xFF)))) } + let kept = RawHistoryArchive.evictLines(lines, maxBytes: 4_096, floor: 2, zeroFloor: 2) + let keptZeroVersion = kept.filter { $0.contains("2f16") } // 0x2f = type 47, 0x16 = v22 + XCTAssertGreaterThanOrEqual(keptZeroVersion.count, 2, + "an all-empty version must keep its small floor, not vanish") + } + + /// One archived JSONL line for `frame`, in the exact shape `archive` writes. + private func archiveLine(_ frame: [UInt8], family: DeviceFamily = .whoop5) -> String { + let hex = frame.map { String(format: "%02x", $0) }.joined() + return "{\"capturedAtMs\":1,\"trim\":1,\"family\":\"\(family.rawValue)\",\"frameHex\":\"\(hex)\"}\n" + } + + // MARK: - operator summary + + /// The summary is the after-an-experiment check: it must separate informative frames from + /// placeholders rather than reporting one healthy-looking line count. + func testSummarySplitsInformativeFromZeroPayload() { + var lines = [archiveLine(whoop5Frame(version: 22, payloadByte: 0, marker: 0xE7))] + for _ in 0..<5 { lines.append(archiveLine(whoop5Frame(version: 22, payloadByte: 0))) } + let text = RawHistoryArchive.summaryText(lines: lines, sizeBytes: 4_096, newest: 3) + XCTAssertTrue(text.contains("whoop5 v22"), text) + XCTAssertTrue(text.contains("1 informative, 5 all-zero payload"), text) + XCTAssertTrue(text.contains("Newest 3 capture(s)"), text) + XCTAssertTrue(RawHistoryArchive.summaryText(lines: [], sizeBytes: 0).contains("EMPTY")) + } + /// `evictLines` is the pure core: a flood of common-version lines plus two rare-version lines (the /// oldest), capped tight → the rare lines must survive and the result must fit the cap. func testEvictLinesKeepsRareVersionUnderCap() { diff --git a/StrandTests/StalledHandshakePolicyTests.swift b/StrandTests/StalledHandshakePolicyTests.swift new file mode 100644 index 0000000000..fc5b05375d --- /dev/null +++ b/StrandTests/StalledHandshakePolicyTests.swift @@ -0,0 +1,77 @@ +import XCTest +@testable import Strand + +/// `StalledHandshakePolicy` — the cancel-and-rescan decision for a link CoreBluetooth calls connected but +/// that never became usable. Pure value logic, no CoreBluetooth seam. +/// +/// The case that matters is `testRestoredZombie_recovers`: it encodes the real on-device failure (iOS state +/// restoration returns a `.connected` peripheral whose strap is dead, the CLIENT_HELLO is never acked, so +/// `didWriteValueFor` never fires — and that callback is the ONLY caller of `startKeepAlive()`, so the 120 s +/// liveness watchdog that would have bounced the link never even arms). Every other test here is a guard +/// AGAINST firing, because a watchdog that bounces healthy links is worse than the bug it fixes. +final class StalledHandshakePolicyTests: XCTestCase { + + /// Defaults describe a link in the exact stalled state; each test flips ONE input. + private func recover(connected: Bool = true, + handshakeDone: Bool = false, + sawData: Bool = false, + intentionalDisconnect: Bool = false, + autoReconnectPaused: Bool = false, + bondRefused: Bool = false) -> Bool { + StalledHandshakePolicy.shouldRecover(connected: connected, + handshakeDone: handshakeDone, + sawData: sawData, + intentionalDisconnect: intentionalDisconnect, + autoReconnectPaused: autoReconnectPaused, + bondRefused: bondRefused) + } + + /// THE bug: connected, no handshake, not one byte ever received, and nothing else owns the link. + /// Left alone this spun for 40+ minutes ("Backfill: deferred — connect handshake not done yet") and + /// cost a full night of biometrics. + func testRestoredZombie_recovers() { + XCTAssertTrue(recover()) + } + + /// A completed handshake means the strap acked us — this is a healthy link, hands off. + func testCompletedHandshake_neverRecovers() { + XCTAssertFalse(recover(handshakeDone: true)) + XCTAssertFalse(recover(handshakeDone: true, sawData: true)) + } + + /// The strap is talking (e.g. a 5/MG streaming HR over the standard 0x2A37 profile while its puffin + /// handshake lags). Bouncing would throw away a working HR stream; the 120 s liveness fuse already owns + /// a link that goes quiet later. + func testDataFlowing_neverRecovers() { + XCTAssertFalse(recover(sawData: true)) + } + + /// The link already dropped — didDisconnectPeripheral's rescan owns it. + func testNotConnected_neverRecovers() { + XCTAssertFalse(recover(connected: false)) + } + + /// A user teardown is in flight; reconnecting would fight it. + func testIntentionalDisconnect_neverRecovers() { + XCTAssertFalse(recover(intentionalDisconnect: true)) + } + + /// #617/#747: the bond-loop give-up DELIBERATELY stopped hammering this strap and told the user to free + /// it. A bounce here restarts precisely the loop that machinery just paused. + func testAutoReconnectPaused_neverRecovers() { + XCTAssertFalse(recover(autoReconnectPaused: true)) + } + + /// #78/#221: a refused bond ("Connected · not paired") is diagnosed, surfaced and user-actionable — not + /// a stall. Churning the link would bury the guidance under reconnect noise. + func testBondRefused_neverRecovers() { + XCTAssertFalse(recover(bondRefused: true)) + } + + /// The fuse must clear the 60 s reboot settle backstop (so a reboot reconnect resolves itself first) and + /// stay under the 120 s liveness fuse (so the data-driven watchdog owns any link that has data). + func testFuseSitsBetweenTheRebootSettleAndTheLivenessFuse() { + XCTAssertGreaterThan(StalledHandshakePolicy.fuseSeconds, 60) + XCTAssertLessThan(StalledHandshakePolicy.fuseSeconds, 120) + } +} diff --git a/StrandTests/StrapBatteryReadPolicyTests.swift b/StrandTests/StrapBatteryReadPolicyTests.swift new file mode 100644 index 0000000000..057c4ba871 --- /dev/null +++ b/StrandTests/StrapBatteryReadPolicyTests.swift @@ -0,0 +1,52 @@ +import XCTest +import WhoopProtocol +@testable import Strand + +/// `StrapBatteryReadPolicy` — whether to (re)issue a standard Battery Level (0x2A19) read. +/// +/// `testWhoop4_neverReads` is the load-bearing one: it pins #77. A WHOOP 4.0's 0x2A19 is a firmware stub +/// that always reports 100, so reading it flashes a fake 100% over the strap's real charge (the 4.0's true +/// value comes only from the proprietary GET_BATTERY_LEVEL command). The retry that fixes the 5/MG must +/// never reach the 4.0, and that is a family gate a future refactor could silently drop. +final class StrapBatteryReadPolicyTests: XCTestCase { + private let floor = StrapBatteryReadPolicy.minIntervalSeconds // 30 + + /// #77: the 4.0's 0x2A19 is a stub-100. No read, ever — not even the first one. + func testWhoop4_neverReads() { + XCTAssertFalse(StrapBatteryReadPolicy.shouldRead(family: .whoop4, canRead: true, + lastReadAt: nil, now: 1000)) + XCTAssertFalse(StrapBatteryReadPolicy.shouldRead(family: .whoop4, canRead: true, + lastReadAt: 0, now: 1_000_000)) + } + + /// The self-heal: a 5/MG that has never had a successful read reads immediately. This is the retry the + /// pre-bond rejection ("Authentication is insufficient") needed and never got. + func testWhoop5_firstReadAlwaysRuns() { + XCTAssertTrue(StrapBatteryReadPolicy.shouldRead(family: .whoop5, canRead: true, + lastReadAt: nil, now: 1000)) + } + + /// The throttle: `enableLiveNotifications` re-runs on EVERY .withResponse ack, including every + /// HISTORY_END during an offload. Without the floor the retry would storm the strap with ATT reads. + func testWhoop5_throttledWithinTheFloor() { + XCTAssertFalse(StrapBatteryReadPolicy.shouldRead(family: .whoop5, canRead: true, + lastReadAt: 1000, now: 1000 + floor - 1)) + XCTAssertTrue(StrapBatteryReadPolicy.shouldRead(family: .whoop5, canRead: true, + lastReadAt: 1000, now: 1000 + floor)) + XCTAssertTrue(StrapBatteryReadPolicy.shouldRead(family: .whoop5, canRead: true, + lastReadAt: 1000, now: 1000 + floor * 10)) + } + + /// A characteristic without the .read property can't be read regardless of family. + func testNoReadProperty_neverReads() { + XCTAssertFalse(StrapBatteryReadPolicy.shouldRead(family: .whoop5, canRead: false, + lastReadAt: nil, now: 1000)) + } + + /// The floor matches the keep-alive tick, so the steady-state 5/MG cadence is one read per tick — + /// strictly lighter than the 4.0's existing every-60s GET_BATTERY_LEVEL poll. + func testFloorMatchesTheKeepAliveTick() { + XCTAssertEqual(StrapBatteryReadPolicy.minIntervalSeconds, + TimeInterval(BLEManager.keepAliveIntervalSeconds)) + } +} diff --git a/StrandTests/StrapChargeInferenceTests.swift b/StrandTests/StrapChargeInferenceTests.swift new file mode 100644 index 0000000000..9c40eb96d5 --- /dev/null +++ b/StrandTests/StrapChargeInferenceTests.swift @@ -0,0 +1,94 @@ +import XCTest +import StrandAnalytics +@testable import Strand + +/// `StrapChargeInference` — the charging state the app is willing to stand behind. +/// +/// The bug: a 5/MG on its charging puck reported `battery_charging = 0` while its SoC climbed, so the app +/// stated "not charging" and showed a "~2 days left" DISCHARGE estimate to a user watching it charge. The +/// 5.0 charge bit's offset (@30) is unverified — the decoder's own docs confirm only the SoC and mV fields +/// against a real capture — so the bit alone is not a fact worth reporting. +/// +/// ⚠️ THESE TESTS PASSING DOES NOT MEAN THE INFERENCE WORKS. Every fixture below steps the SoC by far +/// more than the 1.0 pp `isRising` demands, so they exercise the decision but NOT the magnitude a real +/// charge actually produces. Mirror data (2026-07-26) shows a confirmed charge stepping +0.3…+0.7 pp per +/// reading at a ~30 s cadence — under the threshold throughout, i.e. the inference would never fire on +/// it. See the KNOWN ISSUE block on `StrapChargeInference` and §A6 of +/// docs/bugs/2026-07-15-strap-battery-backfill-observability.md. A fix wants a real-cadence regression +/// case here, not just a smaller number in these fixtures. +final class StrapChargeInferenceTests: XCTestCase { + private let now = 1_800_000_000 + /// Two readings ~8 min apart. NOTE: this was written as "the strap's real BATTERY_LEVEL cadence", and + /// that assumption is the bug — observed charging readings arrive ~30 s apart (see the note above). + private func series(_ a: Double, _ b: Double, gap: Int = 8 * 60, age: Int = 60) -> [(ts: Int, soc: Double)] { + [(ts: now - age - gap, soc: a), (ts: now - age, soc: b)] + } + + /// THE bug: the flag says "not charging" and the SoC is visibly climbing. Trust the charge, not the bit. + func testRisingSoc_overridesAFalseFlag() { + XCTAssertEqual(StrapChargeInference.resolve(flag: false, samples: series(31.6, 38.2), nowUnix: now), + true) + } + + /// Same rescue when the flag never arrives at all. + func testRisingSoc_rescuesASilentFlag() { + XCTAssertEqual(StrapChargeInference.resolve(flag: nil, samples: series(31.6, 38.2), nowUnix: now), + true) + } + + /// A confirmed `true` is always honoured, no corroboration needed. + func testConfirmedFlagWins() { + XCTAssertEqual(StrapChargeInference.resolve(flag: true, samples: series(50, 49), nowUnix: now), true) + XCTAssertEqual(StrapChargeInference.resolve(flag: true, samples: [], nowUnix: now), true) + } + + /// A normal discharge is not a charge — the estimate must keep working. + func testDischarging_isNotCharging() { + XCTAssertEqual(StrapChargeInference.resolve(flag: false, samples: series(50, 49.7), nowUnix: now), + false) + } + + /// Nothing to go on ⇒ nil (unknown), NOT false. The UI must be able to say nothing rather than assert + /// "not charging" — the false statement this whole type exists to stop. + func testNoEvidence_isUnknownNotFalse() { + XCTAssertNil(StrapChargeInference.resolve(flag: nil, samples: [], nowUnix: now)) + XCTAssertNil(StrapChargeInference.resolve(flag: nil, samples: series(50, 50), nowUnix: now)) + } + + /// The quantisation seam: a 5/MG's live SoC arrives from 0x2A19 as a whole-percent u8, so consecutive + /// readings can legitimately step by exactly 1.0 pp with no charge. The threshold is strictly + /// greater-than `chargeStepPct`, so a 1.0 step must NOT read as charging. + func testWholePercentStep_isNotMistakenForACharge() { + XCTAssertEqual(StrapChargeInference.resolve(flag: false, samples: series(31, 32), nowUnix: now), + false) + XCTAssertFalse(StrapChargeInference.isRising(samples: series(31, 32), nowUnix: now)) + // A real charge is nowhere near that subtle (~50 pp/h vs ~1.65 pp/h discharge). + XCTAssertTrue(StrapChargeInference.isRising(samples: series(31, 38), nowUnix: now)) + } + + /// The threshold is BatteryEstimator's existing charge step, not a second invented notion of "charging". + func testUsesTheEstimatorsOwnChargeThreshold() { + let justUnder = BatteryEstimator.chargeStepPct + let justOver = BatteryEstimator.chargeStepPct + 0.2 + XCTAssertFalse(StrapChargeInference.isRising(samples: series(50, 50 + justUnder), nowUnix: now)) + XCTAssertTrue(StrapChargeInference.isRising(samples: series(50, 50 + justOver), nowUnix: now)) + } + + /// A rise from an hour ago says nothing about NOW — the strap may already be off the puck. + func testStaleRise_doesNotClaimChargingNow() { + let old = [(ts: now - 7200, soc: 31.0), (ts: now - 6800, soc: 40.0)] + XCTAssertFalse(StrapChargeInference.isRising(samples: old, nowUnix: now)) + XCTAssertEqual(StrapChargeInference.resolve(flag: false, samples: old, nowUnix: now), false) + } + + /// Two readings separated by a huge gap aren't a slope — a strap that reconnects hours later at a higher + /// SoC was charged in the meantime, but is not necessarily charging now. + func testWideGapBetweenReadings_isNotASlope() { + XCTAssertFalse(StrapChargeInference.isRising(samples: series(31, 90, gap: 4 * 3600), nowUnix: now)) + } + + /// A single reading can't show a trend. + func testSingleReading_isNotARise() { + XCTAssertFalse(StrapChargeInference.isRising(samples: [(ts: now, soc: 50)], nowUnix: now)) + } +} diff --git a/StrandTests/SyncPushTelemetryTests.swift b/StrandTests/SyncPushTelemetryTests.swift new file mode 100644 index 0000000000..4cc6fbc3a0 --- /dev/null +++ b/StrandTests/SyncPushTelemetryTests.swift @@ -0,0 +1,178 @@ +// Compiled ONLY when the CLOUD_SYNC compilation condition is set (StrandTests shares the app's +// OuraConfig.xcconfig, so the flag arrives with the app's). +#if CLOUD_SYNC +import XCTest +@testable import Strand + +/// `SyncPushTelemetry` exists to answer one question about a real device: how often does a +/// page-replication push degrade to a full snapshot? Everything asserted here is a property that +/// number depends on — if trimming lost the counters, or a corrupt file threw, or the gap between +/// pushes were computed from the wrong record, the trial would produce a plausible wrong answer +/// rather than an obvious failure. That is the risk being tested. +final class SyncPushTelemetryTests: XCTestCase { + + private var dir: URL! + + override func setUpWithError() throws { + dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("push-telemetry-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: dir) + } + + private func makeURL() -> URL { dir.appendingPathComponent("push-telemetry.json") } + + private func record(_ t: SyncPushTelemetry, + at: Date, + snapshotted: Bool = false, + reason: String? = nil, + bytes: Int64 = 1_000, + wal: Int64 = 4_096, + txid: UInt64 = 1) { + t.record(at: at, snapshotted: snapshotted, snapshotReason: reason, + bytesUploaded: bytes, walBytes: wal, txid: txid) + } + + // MARK: - The number the trial turns on + + func testSnapshotRateAndByteShare() { + let t = SyncPushTelemetry(url: makeURL()) + let t0 = Date(timeIntervalSince1970: 1_000) + record(t, at: t0, snapshotted: true, reason: "first sync, no local position", bytes: 800_000_000) + for i in 1...9 { record(t, at: t0.addingTimeInterval(Double(i) * 600), bytes: 400_000) } + + let s = t.stats + XCTAssertEqual(s.pushes, 10) + XCTAssertEqual(s.snapshots, 1) + XCTAssertEqual(s.snapshotRate, 0.1, accuracy: 1e-9) + // Frequency and cost are different questions: 10% of pushes, >99% of the bytes. + XCTAssertGreaterThan(s.snapshotByteShare, 0.99) + } + + /// The reason histogram is what turns "the rate is bad" into "and here is which branch of + /// verify() caused it" — the difference between a decision and a shrug. + func testReasonsAreCountedPerBranch() { + let t = SyncPushTelemetry(url: makeURL()) + let t0 = Date(timeIntervalSince1970: 0) + record(t, at: t0, snapshotted: true, reason: "wal truncated by another process") + record(t, at: t0.addingTimeInterval(10), snapshotted: true, reason: "wal truncated by another process") + record(t, at: t0.addingTimeInterval(20), snapshotted: true, reason: "full or restart checkpoint detected, snapshotting") + record(t, at: t0.addingTimeInterval(30)) + + XCTAssertEqual(t.stats.reasonCounts, [ + "wal truncated by another process": 2, + "full or restart checkpoint detected, snapshotting": 1, + ]) + } + + /// A non-snapshotting push must not carry a reason even if one is passed — otherwise the + /// histogram would count branches that did not fire. + func testNonSnapshotPushDropsAnyReason() { + let t = SyncPushTelemetry(url: makeURL()) + let o = t.record(at: Date(), snapshotted: false, snapshotReason: "stale", + bytesUploaded: 10, walBytes: 20, txid: 3) + XCTAssertNil(o.snapshotReason) + XCTAssertTrue(t.stats.reasonCounts.isEmpty) + } + + // MARK: - Bounded, but not lossy where it counts + + /// The record window is capped so the file cannot grow without bound over a long trial — but + /// the aggregate counters must survive trimming, or the rate would silently become "the rate + /// over the last 200 pushes" while still being read as "the rate". + func testTrimmingDropsRecordsButNeverTheCounters() { + let t = SyncPushTelemetry(url: makeURL()) + let t0 = Date(timeIntervalSince1970: 0) + let n = SyncPushTelemetry.maxRecords + 50 + for i in 0.. [(ts: Int, steps: Int)] { + guard let type = HKQuantityType.quantityType(forIdentifier: .stepCount) else { return [] } + let cal = Calendar.current + let anchor = cal.startOfDay(for: start) + let predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [ + HKQuery.predicateForSamples(withStart: start, end: end, options: .strictStartDate), + Self.notNoopAuthored, + ]) + return try await withCheckedThrowingContinuation { (cont: CheckedContinuation<[(ts: Int, steps: Int)], Error>) in + let q = HKStatisticsCollectionQuery(quantityType: type, quantitySamplePredicate: predicate, + options: .cumulativeSum, anchorDate: anchor, + intervalComponents: DateComponents(hour: 1)) + q.initialResultsHandler = { _, results, error in + if let error { + cont.resume(throwing: error) + return + } + var rows: [(ts: Int, steps: Int)] = [] + results?.enumerateStatistics(from: start, to: end) { stats, _ in + if let sum = stats.sumQuantity() { + let steps = Int(sum.doubleValue(for: .count()).rounded()) + rows.append((ts: Int(stats.startDate.timeIntervalSince1970), steps: steps)) + } + } + cont.resume(returning: rows) + } + store.execute(q) + } + } + // MARK: - Workouts (#835) /// Read the workouts the user logged in Apple Health over `[start, end)` and map each to a diff --git a/StrandiOS/Resources/Info.plist b/StrandiOS/Resources/Info.plist index d6243035b3..200a9e57aa 100644 --- a/StrandiOS/Resources/Info.plist +++ b/StrandiOS/Resources/Info.plist @@ -7,6 +7,7 @@ BGTaskSchedulerPermittedIdentifiers $(PRODUCT_BUNDLE_IDENTIFIER).debugexport + $(PRODUCT_BUNDLE_IDENTIFIER).cloudsync.refresh CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) @@ -48,6 +49,10 @@ CFBundleVersion $(CURRENT_PROJECT_VERSION) + CLOUDSYNC_TOKEN + $(CLOUDSYNC_TOKEN) + CLOUDSYNC_URL + $(CLOUDSYNC_URL) LSApplicationCategoryType public.app-category.healthcare-fitness LSSupportsOpeningDocumentsInPlace @@ -80,6 +85,7 @@ bluetooth-central location fetch + remote-notification UIFileSharingEnabled diff --git a/Tools/doc_comment_lint.py b/Tools/doc_comment_lint.py index a82755c2a6..e147202668 100644 --- a/Tools/doc_comment_lint.py +++ b/Tools/doc_comment_lint.py @@ -47,7 +47,17 @@ # after it is conventional and correct, so it must not be reported. FILE_LEVEL_PREFIXES = ("package ", "import ", "@file:") -SKIP_PARTS = ("/build/", "/.build/", "/DerivedData/") +# Build output, plus CODE GENERATOR output. `/Generated/` is the same kind of exclusion as the three +# build directories, not a baseline raise: this lint exists to catch a HAND-EDITING failure mode — +# "editing by INSERTION rather than in place" — and a generator never edits, it re-emits. The one +# directory this currently matches, Strand/CloudSync/Generated/, is `rm -rf`'d and rewritten by +# `Rust/build-ios.sh` on every build; its header reads "Do not edit by hand" and it already carries +# `// swiftlint:disable all`, so a fix applied there is deleted by the next regeneration and the +# finding returns. Two of uniffi's emissions defeat a fix outright: it puts a blank line between a +# `/** */` block and the `public enum` it documents (4 sites), and it inlines a parameter's doc INSIDE +# the argument list (`public init(/** … */restored: Bool, …)`), which this file's scanner mis-walks — +# that block is correctly bound to its parameter, so there is nothing there to repair. +SKIP_PARTS = ("/build/", "/.build/", "/DerivedData/", "/Generated/") def _is_doc_open(line: str) -> bool: diff --git a/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt b/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt index 480f70b859..ac48302a6e 100644 --- a/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt @@ -9,6 +9,8 @@ import com.noop.data.SkinTempSample import com.noop.data.Spo2Sample import com.noop.data.RespSample import com.noop.data.RrInterval +import com.noop.data.SPO2_CANDIDATE_IN_BAND +import com.noop.data.Spo2PctRow import com.noop.data.StepSample import com.noop.protocol.DeviceFamily import com.noop.protocol.Whoop4SkinTemp @@ -183,6 +185,13 @@ object AnalyticsEngine { // decoded" data, NOT a calibrated blood-oxygen % (that needs WHOOP's proprietary curve). Default // empty keeps pure-function callers/tests + non-4.0 nights null. spo2: List = emptyList(), + // Durable 5/MG `@82` SpO2 percentages for the night window (v34 / MIGRATION_25_26). The + // ramp-trimmed nightly MEDIAN is banked on `DailyMetric.spo2Pct` — the strap's OWN computed + // percentage, not a curve NOOP invented. Distinct from [spo2] above, which is the WHOOP 4.0 v24 + // raw red/IR ADC pair and feeds `spo2Red`/`spo2Ir`; the two device generations report different + // physical quantities and neither substitutes for the other. Default empty keeps pure-function + // callers/tests + every non-5/MG night null. + spo2PctSamples: List = emptyList(), profile: UserProfile, baselines: ProfileBaselines = ProfileBaselines(), maxHROverride: Double? = null, @@ -232,6 +241,12 @@ object AnalyticsEngine { // flag. Default false keeps every pure-function caller/test byte-identical; IntelligenceEngine // threads PuffinExperiment.from(context).motionAwareWake. Mirrors Swift. useMotionAwareWake: Boolean = false, + // Wake-gap bridging (experimental, default OFF — see the section comment above + // [SleepStager.wakeBridgeMaxMin]). When true, detection assembles one session per sleep + // period across ≤ 45-min wake gaps, so a brief get-up stays inside one session as staged + // wake. Default false keeps every pure-function caller/test byte-identical; + // IntelligenceEngine threads PuffinExperiment.from(context).sleepWakeBridge. Mirrors Swift. + useWakeBridge: Boolean = false, // Sleep & Rest test-mode trace sink (E11). null = byte-identical default. When non-null the gate // trace from detectSleep and the Rest sub-score line are forwarded line-by-line. Mirrors Swift. traceSink: ((String) -> Unit)? = null, @@ -255,6 +270,7 @@ object AnalyticsEngine { hr = hr, rr = rr, resp = resp, gravity = gravity, tzOffsetSeconds = tzOffsetSeconds, wristOff = wristOff, bandSleepState = bandSleepState, useSleepStagerV2 = useSleepStagerV2, + wakeBridge = useWakeBridge, traceSink = traceSink, ) // Motion-aware wake refinement (#364 follow-up) runs AFTER V1/V2 staging, over every detected @@ -441,6 +457,14 @@ object AnalyticsEngine { // as-is for the Health "Raw SpO2" tile — NOT a calibrated blood-oxygen %. (#93) val nightlySpo2Raw = nightlySpo2RawMeans(matched, spo2) + // ── SpO2 percentage (5/MG @82, durable) ─────────────────────────────── + // Ramp-trimmed nightly median of the strap's OWN computed SpO2 %, over the detected in-bed spans. + // Null on a WHOOP 4.0, on any night with no in-band reading, and on every window offloaded before + // MIGRATION_25_26. Unlike the raw red/IR means above this IS a percentage — it is the number the + // strap computed, banked verbatim per-second and aggregated here, not a calibration NOOP applied + // to an ADC. See [nightlySpo2Pct] for the run/ramp policy and why the statistic is a median. + val nightlySpo2PctVal = nightlySpo2Pct(matched, spo2PctSamples) + // ── Rest (sleep_performance composite, 0–100) ───────────────────────── // Replaces the bare efficiency proxy: duration-vs-personal-need 0.50 + efficiency 0.20 + // restorative (deep+REM)/asleep 0.20 + consistency 0.10. Stored under the sleep_performance @@ -606,7 +630,20 @@ object AnalyticsEngine { recovery = recovery, strain = strain, exerciseCount = workouts.size, - spo2Pct = null, + // v34: was unconditionally null here — the on-device engine had no percentage to bank, so the + // Blood Oxygen card read "No Data" on every WHOOP night while the same field rendered fine + // for imported rows. It now carries the 5/MG's own `@82` median when the night has one, and + // stays null otherwise (WHOOP 4.0, pre-MIGRATION_25_26 windows, no in-band samples) — an + // absent reading must stay absent rather than become a fabricated number. + // + // NOTE FOR ANYONE CHANGING THIS: `dailyMetric.spo2Pct` is not display-only. + // [com.noop.ingest.HealthConnectWriter] writes it back as an `OxygenSaturationRecord`, so a + // value banked here can leave the app and land in the user's Health Connect blood-oxygen + // history alongside clinical readings. That export is gated behind + // `HealthConnectWriter.EXPORTS_SPO2_TO_HEALTH_CONNECT` for exactly that reason — read the + // constant before touching either. This field's bar is "the device measured this", never + // "this is our best guess". + spo2Pct = nightlySpo2PctVal?.first, skinTempDevC = skinTempDevC, respRateBpm = respRateDaily, steps = stepsTotal, @@ -765,7 +802,9 @@ object AnalyticsEngine { var kept = 0 for (a in aux) { val v = a.auxByte82 ?: continue - if (v < 70L || v > 100L) continue + // The same named window the store's fork and [nightlySpo2Pct] apply, so the three agree + // structurally rather than by three matching literals. + if (v.toInt() !in SPO2_CANDIDATE_IN_BAND) continue if (sessions.none { a.ts in it.start..it.end }) continue sum += v kept += 1 @@ -774,6 +813,101 @@ object AnalyticsEngine { return Pair((sum / kept).toInt(), kept) } + /** + * Nightly SpO2 percentage from the DURABLE `@82` stream (`spo2PctSample`, v34 / MIGRATION_25_26) over + * the detected in-bed [sessions] — the value banked on `DailyMetric.spo2Pct` for a 5/MG night — with + * the sample count that survived the ramp trim. Null when no reading fell inside any span. + * + * HOW THIS DIFFERS FROM [nightlySpo2CandidateMean]. That one is a naive mean over the raw candidate + * byte read out of the CAPPED aux table, built as a hand-checkable diagnostic. This is the scored + * statistic, over the durable stream, and it corrects the two things that make the naive mean wrong. + * + * THE ACQUISITION RAMP. The strap does not sample SpO2 continuously: it takes a RUN of ~30 one-second + * samples roughly once per ~1,200 s while asleep. The first samples of each run are the optical front + * end settling, and they read LOW — so a mean over every in-band sample is biased downward by a + * contamination that is entirely one-sided, and the bias grows with how much of the night is ramp + * (i.e. it is worse on nights with more, shorter runs — precisely when you would least suspect it). + * Runs are separated here by a gap of more than [SPO2_RUN_GAP_SECONDS]; at 1 Hz within a run and + * ~1,200 s between runs, that boundary is not a close call. The leading [SPO2_RAMP_SAMPLES] of each + * run are dropped — but never more than HALF a run, so a short run contributes its settled tail + * instead of vanishing and silently reweighting the night toward whichever runs happened to be long. + * + * WHY THE MEDIAN AND NOT A TRIMMED MEAN. Three reasons, in order of weight: + * 1. It is the statistic the cloud reader already reports (the per-night 94-97 medians every + * cross-check against this data has been quoted against). A phone that shipped a trimmed mean + * would disagree with the server on the same rows for no reason anyone could see. + * 2. Ramp removal is a heuristic, not a proof. A partially-settled sample just past the cut, or a + * motion artifact mid-run, is residual one-sided contamination that a mean absorbs in full and a + * median very largely ignores. + * 3. The in-band values live in a 31-wide integer band with a hard mode at 97, so a median is + * stable and interpretable here in a way it would not be on a sparse continuous quantity. + * The trim and the median are complementary, not redundant: the trim removes a KNOWN, structured, + * large block of low readings that would move a median too, and the median absorbs what is left. + * + * READ-TIME POLICY, DELIBERATELY. None of this is applied on the way into the store — + * `spo2PctSample` holds raw in-band bytes. These rules have already changed once on the cloud side, + * and they will change again if a run's shape turns out to differ by firmware; keeping them here + * means that costs a re-score, not a re-capture of data the strap has long since trimmed. + * + * Pure + deterministic. Byte-parity twin of the Swift `AnalyticsEngine.nightlySpo2Pct`; the returned + * pair is (median percentage, surviving sample count), matching Swift's `(pct:samples:)`. + */ + fun nightlySpo2Pct( + sessions: List, + samples: List, + ): Pair? { + if (sessions.isEmpty() || samples.isEmpty()) return null + // In-bed first, so run boundaries are computed over the samples that will actually be used. Note + // what that does and does not mean: a session edge splits a run only when the stretch it excludes + // is itself longer than [SPO2_RUN_GAP_SECONDS] — a brief out-of-bed moment mid-run leaves the run + // intact and singly-trimmed. Gap size is the ONLY run signal; session boundaries are not a second + // one. That is deliberate, because the ramp is a property of the sensor restarting, and the + // sensor does not restart because the sleep detector drew a line. + val inBed = samples + .filter { s -> sessions.any { s.ts >= it.start && s.ts <= it.end } } + .sortedBy { it.ts } + if (inBed.isEmpty()) return null + + val kept = ArrayList(inBed.size) + fun flushRun(fromIdx: Int, toIdx: Int) { + val run = inBed.subList(fromIdx, toIdx) + // Never trim more than half a run: a 4-sample run keeps its last 2 rather than disappearing. + val drop = minOf(SPO2_RAMP_SAMPLES, run.size / 2) + for (i in drop until run.size) kept.add(run[i].pct) + } + var runStart = 0 + for (i in 1 until inBed.size) { + if (inBed[i].ts - inBed[i - 1].ts > SPO2_RUN_GAP_SECONDS) { + flushRun(runStart, i) + runStart = i + } + } + flushRun(runStart, inBed.size) + if (kept.isEmpty()) return null + + kept.sort() + val mid = kept.size / 2 + val median = + if (kept.size % 2 == 0) (kept[mid - 1] + kept[mid]) / 2.0 + else kept[mid].toDouble() + return Pair(median, kept.size) + } + + /** + * Gap (seconds) above which two consecutive in-band `@82` samples belong to different measurement + * runs. Samples inside a run are 1 s apart and runs are ~1,200 s apart, so anything in between + * separates them; 60 s sits in the middle of that gulf and tolerates a dropped second or two inside a + * run without splitting it. Swift twin: `AnalyticsEngine.spo2RunGapSeconds`. + */ + const val SPO2_RUN_GAP_SECONDS: Long = 60L + + /** + * Leading samples dropped from each measurement run as the optical front end's acquisition ramp. + * ~5 of a ~30-sample run. Capped at half a run by the caller so short runs still contribute. + * Swift twin: `AnalyticsEngine.spo2RampSamples`. + */ + const val SPO2_RAMP_SAMPLES: Int = 5 + /** Plausible worn skin-temperature range (°C). Off-wrist/charging samples drift to ambient and are * excluded; the strap's own decode gate is the looser 20–45. (PR #85) */ private const val SKIN_TEMP_MIN_C: Double = 28.0 diff --git a/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt b/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt index 2e950ec9cb..d2d81aa02f 100644 --- a/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt +++ b/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt @@ -176,7 +176,14 @@ enum class BaselineStatus(val raw: String) { PROVISIONAL("provisional"), /** At least MIN_NIGHTS_TRUST valid nights. */ TRUSTED("trusted"), - /** Usable but no update for > STALE_DAYS nights. */ + /** + * Seeded (nValid ≥ MIN_NIGHTS_SEED) but not updated for > STALE_DAYS nights — so NOT [BaselineState.usable], + * and consumers fall back to population norms. This doc used to read "Usable but no update for > + * STALE_DAYS nights", which collides with the `usable` property (it excludes STALE) and reads as a + * code/comment contradiction. The CODE is the correct half: a personal baseline nobody has confirmed in + * over two weeks is exactly one not to trust. Pinned by `VitalBandsTest.staleBaseline_fallsBackToPopulation` + * on both platforms. (F1, docs/bugs/2026-07-15-strap-battery-backfill-observability.md) + */ STALE("stale"), } @@ -202,7 +209,12 @@ data class BaselineState( /** True iff fully trusted (not calibrating or stale). */ val trusted: Boolean get() = status == BaselineStatus.TRUSTED - /** True iff at least provisionally usable (nValid ≥ MIN_NIGHTS_SEED). */ + /** + * True iff this personal baseline should be trusted at all. NOT simply "nValid ≥ MIN_NIGHTS_SEED": + * [BaselineStatus.STALE] also clears that bar and is still excluded, because it has gone > STALE_DAYS + * nights with no confirming value. Callers that get `false` here fall back to population norms rather + * than score off a baseline that may no longer describe this person. + */ val usable: Boolean get() = status == BaselineStatus.PROVISIONAL || status == BaselineStatus.TRUSTED } diff --git a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt index d40b742514..0e166976fe 100644 --- a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt @@ -184,6 +184,10 @@ object IntelligenceEngine { // OWN observed gravity + step density (see [WakeMotionRefinement]), so existing callers / tests // and any night too sparse to trust (e.g. a WHOOP 4.0) are unaffected either way. useMotionAwareWake: Boolean = false, + // Wake-gap bridging (experimental, default OFF), threaded the same Context-free way: the + // Context-aware caller reads PuffinExperiment.from(context).sleepWakeBridge and passes it down + // to AnalyticsEngine.analyzeDay. See [SleepStager.wakeBridgeMaxMin]'s section comment. Mirrors Swift. + useWakeBridge: Boolean = false, // Sleep & Rest test-mode trace sink (Test Centre E5). The analytics layer is Context-free, so the // Context-aware caller (AppViewModel / WhoopBleClient) reads TestCentre.active(SLEEP) and passes a // non-null sink ONLY when the mode is on, routing each line to the .sleep-tagged strap log. null (the @@ -231,7 +235,8 @@ object IntelligenceEngine { analyzeGate.withLock { val (out, healed) = analyzeRecentOnCpu(repo, profile, maxDays, importedDeviceId, maxHROverride, nowSeconds, ownerSource, manualStepCoefficient, persistStepsCalibration, baselineEpoch, - recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, sleepTraceSink, recoveryTraceSink, + recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, useWakeBridge, + sleepTraceSink, recoveryTraceSink, stepsTraceSink, universalSink, workoutsTraceSink, hrvTraceSink, deepHrvWindow) if (healed == 0) out // #899 heal re-pass: the pass above deleted overlapping duplicate sleep sessions AFTER its days @@ -241,7 +246,8 @@ object IntelligenceEngine { // are gone), so this can never loop. Mirrors the Swift pendingForcedRescore re-arm. else analyzeRecentOnCpu(repo, profile, maxDays, importedDeviceId, maxHROverride, nowSeconds, ownerSource, manualStepCoefficient, persistStepsCalibration, baselineEpoch, - recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, sleepTraceSink, recoveryTraceSink, + recoveryEpoch, diag, useExperimentalSleepV2, useMotionAwareWake, useWakeBridge, + sleepTraceSink, recoveryTraceSink, stepsTraceSink, universalSink, workoutsTraceSink, hrvTraceSink, deepHrvWindow).first } } @@ -307,6 +313,8 @@ object IntelligenceEngine { useExperimentalSleepV2: Boolean = false, // Opt-in motion-aware wake refinement (#364 follow-up), threaded the same way. Default false. useMotionAwareWake: Boolean = false, + // Wake-gap bridging (experimental, default OFF), threaded the same way. Default false. + useWakeBridge: Boolean = false, // Sleep & Rest test-mode trace sink (Test Centre E5). null = byte-identical default; when non-null // each scored day threads it into AnalyticsEngine.analyzeDay so detectSleep's gate trace + the Rest // sub-score line forward line-by-line to the .sleep-tagged strap log. Mirrors Swift. @@ -482,6 +490,12 @@ object IntelligenceEngine { // #93: WHOOP 4.0 raw SpO2 PPG samples for the night; analyzeDay banks the nightly red/IR ADC // means on the DailyMetric. Empty on a 5/MG (no v24 spo2 channels) → the raw means stay null. val spo2 = repo.spo2Samples(owner, from, to, STREAM_LIMIT) + // v34: the 5/MG's own per-second `@82` SpO2 percentages for the night; analyzeDay banks the + // ramp-trimmed nightly MEDIAN on `DailyMetric.spo2Pct`. Read from the durable table, which is + // never pruned — so re-scoring an old night still finds its readings, where the capped aux + // table they were forked from has held only ~7 days. Empty on a WHOOP 4.0 and on windows + // offloaded before MIGRATION_25_26 → the percentage stays null rather than fabricated. + val spo2Pct = repo.spo2PctSamples(owner, from, to, STREAM_LIMIT) // #938: the strap family that WROTE this owner's skin-temp rows, so analyzeDay converts the raw // register on the right scale (5/MG banks centidegrees, a WHOOP 4.0 v24 banks a raw ADC). The // owner source resolves it from the registry; unknown/non-WHOOP owners fall back to WHOOP5 (the @@ -568,6 +582,7 @@ object IntelligenceEngine { skinTempFamily = skinFamily, // #938 skinTempAnchorRaw = skinAnchorRaw, // #938 second capture: per-device worn anchor spo2 = spo2, // #93 + spo2PctSamples = spo2Pct, // v34 profile = profile, baselines = baselines1, maxHROverride = maxHROverride, @@ -585,6 +600,8 @@ object IntelligenceEngine { useSleepStagerV2 = useExperimentalSleepV2, // #364 follow-up: same threading for the motion-aware wake refinement post-pass. useMotionAwareWake = useMotionAwareWake, + // Wake-gap bridging (experimental, default OFF). + useWakeBridge = useWakeBridge, // Sleep & Rest test mode (Test Centre E5): thread the trace sink straight through. null (the // default) keeps analyzeDay's byte-identical untraced path; when the caller passed a non-null // sink (mode on), detectSleep's gate trace + the Rest sub-score line route to the .sleep-tagged diff --git a/android/app/src/main/java/com/noop/analytics/SleepStager.kt b/android/app/src/main/java/com/noop/analytics/SleepStager.kt index 177f2a1aa4..84a48116fd 100644 --- a/android/app/src/main/java/com/noop/analytics/SleepStager.kt +++ b/android/app/src/main/java/com/noop/analytics/SleepStager.kt @@ -587,6 +587,73 @@ object SleepStager { return out } + // ── Wake-gap bridging (experimental, default OFF) ──────────────────────── + // + // A brief mid-night get-up splits the night into separate stored sessions: the CENTERED 15-min + // rolling stillness window smears an ~8-min bathroom trip past mergeMin into an "active" run (a + // real MG night split at a 16-min seam), detection emits one stored session per still-run, and + // nothing at the detection layer re-merges them — the #561/#861 bridges only group sessions + // later, at selection/aggregation, so sub-minSleepMin sleep fragments between arousals are + // dropped by the ladder before those bridges ever see them. When enabled (the live call site + // threads PuffinExperiment.sleepWakeBridge, default OFF), [bridgeWakeGaps] assembles adjacent + // sleep runs separated by ≤ [wakeBridgeMaxMin] into ONE candidate run between the run spine and + // the gate ladder; the stager then scores the wake INSIDE the single session, and every + // downstream gate still applies to the ASSEMBLED run. A merge that would assemble a span over + // [maxMainSleepSpanS] is REFUSED (without that guard a pathological chain would assemble, then + // be dropped WHOLE by the H4 cap — converting a partially-scored night into a zero-score night). + // + // HISTORY — an HR-led "rescue" of motion-vetoed runs was built beside this bridge and WITHDRAWN + // after a 5-night real-data replay against the reporting wearer's server mirror (2026-08-24): + // on a low-HR wearer the quiet awake HR (median ~58 bpm, awake-resting low-50s, RHR ~44) also + // sits inside the ×1.05 sleep band off the ~64 window baseline, so whole awake evenings were + // rescued and chained — the assembled run then either exceeded the H4 cap (night dropped to + // ZERO) or squeaked under it (a 15 h "night" starting at 18:00). A follow-up decode + // investigation then verified the MG gravity decode CLEAN and showed the premise was partly + // wrong: on the flagship "dropped" night the strap's band state read wake across the disputed + // hours with corroborating 70–75 bpm stretches — likely genuine in-bed wakefulness the + // stillness spine was RIGHT to reject. See the Swift history note above `wakeBridgeMaxMin` + // before re-attempting an HR-band rescue. Mirrors Swift. + + /** Adjacent sleep runs separated by at most this many minutes are assembled into ONE candidate + * run (wake-bridge pass only). Below the selector-side `SleepStageTotals.gapBridgeMaxMin` (60) + * so selection/aggregation behaviour is unchanged for anything this bridge leaves split; well + * above the ~16-min smeared seam a brief real get-up produces. Mirrors Swift `wakeBridgeMaxMin`. */ + const val wakeBridgeMaxMin: Int = 45 + + /** + * Assemble adjacent sleep runs separated by at most [wakeBridgeMaxMin] minutes (whatever lies + * between — an active run, or a plain data hole) into one candidate run; the intervening runs + * are absorbed and the stager later scores the wake INSIDE the single session. A gap over the + * threshold, a negative/overlapping gap (pinned `gap >= 0` semantics, matching + * [bridgeSparseSleep]), or a merge whose assembled span would exceed [maxMainSleepSpanS] (the + * H4 guard) never bridges. Pure + deterministic; mirrors Swift `bridgeWakeGaps`. + */ + internal fun bridgeWakeGaps( + periods: List, traceSink: ((String) -> Unit)? = null, + ): List { + val bridgeS = (wakeBridgeMaxMin * 60).toLong() + val out = ArrayList() + for (p in periods) { + val j = out.indexOfLast { it.stage == "sleep" } + if (p.stage == "sleep" && j >= 0 && + p.start - out[j].end >= 0 && p.start - out[j].end <= bridgeS && + maxOf(out[j].end, p.end) - out[j].start <= maxMainSleepSpanS + ) { + val gapMin = (p.start - out[j].end) / 60 + if (gapMin > 0) { + traceSink?.invoke(SleepStagerTrace.runLine(-1, out[j].end, p.start, + SleepStagerTrace.Verdict.KEPT, "wakeBridge", + "gapMin=$gapMin max=$wakeBridgeMaxMin")) + } + out[j] = Period(stage = "sleep", start = out[j].start, end = maxOf(out[j].end, p.end)) + while (out.size > j + 1) out.removeAt(out.size - 1) + } else { + out.add(p) + } + } + return out + } + // ── HR refinement ──────────────────────────────────────────────────────── private inline fun rowsBetween(rows: List, start: Long, end: Long, ts: (T) -> Long): List = @@ -876,6 +943,7 @@ object SleepStager { private data class DetectKey( val grav: Long, val hr: Long, val rr: Long, val resp: Long, val tz: Long, val wristOff: Long, val band: Long, val v2: Boolean, + val bridge: Boolean, val sleepHRBaseline: Double?, ) @@ -947,6 +1015,12 @@ object SleepStager { // #277, extended to every strap family in #351), so a normal user's nights are staged by V2. // Mirrors Swift. useSleepStagerV2: Boolean = false, + // Wake-gap bridging (experimental, default OFF — see the section comment above + // [wakeBridgeMaxMin]). When true, adjacent sleep runs across ≤ wakeBridgeMaxMin wake gaps + // are assembled into one candidate run before the gate ladder, so a brief get-up stays + // INSIDE one session as staged wake. The default keeps every existing caller byte-identical; + // the live call site threads PuffinExperiment.sleepWakeBridge. Mirrors Swift. + wakeBridge: Boolean = false, // Motion-corroborated wake (#462, directive b): the wearer's PERSONALISED overnight HR band // ([adaptiveOvernightHRBaseline]), used by [confirmSleepWithHR] in place of the day-median so a // supplement / fitness era self-calibrates the sleep band. Default null keeps the day-median @@ -964,7 +1038,7 @@ object SleepStager { // Swift detectSleep traceSink bypass (#707). if (traceSink != null) { return detectSleepUncached(hr, rr, resp, gravity, tzOffsetSeconds, wristOff, - bandSleepState, useSleepStagerV2, sleepHRBaseline, traceSink) + bandSleepState, useSleepStagerV2, wakeBridge, sleepHRBaseline, traceSink) } val key = DetectKey( // Fold the three gravity axes SEPARATELY (raw IEEE-754 bits, like StagerCache.fingerprint) @@ -985,6 +1059,7 @@ object SleepStager { wristOff = streamFingerprint(wristOff, { it.first }) { it.second }, band = streamFingerprint(bandSleepState, { it.first }) { it.second.toLong() }, v2 = useSleepStagerV2, + bridge = wakeBridge, sleepHRBaseline = sleepHRBaseline, ) synchronized(detectCacheLock) { detectCache[key] }?.let { return copyDetected(it) } @@ -992,7 +1067,7 @@ object SleepStager { // serialises another thread's lookup, and a racing duplicate compute just overwrites the entry // with an identical value — benign, the function is deterministic. val sessions = detectSleepUncached(hr, rr, resp, gravity, tzOffsetSeconds, wristOff, - bandSleepState, useSleepStagerV2, sleepHRBaseline, null) + bandSleepState, useSleepStagerV2, wakeBridge, sleepHRBaseline, null) synchronized(detectCacheLock) { detectCache[key] = copyDetected(sessions) } return sessions } @@ -1008,6 +1083,7 @@ object SleepStager { wristOff: List>, bandSleepState: List>, useSleepStagerV2: Boolean, + wakeBridge: Boolean, sleepHRBaseline: Double?, traceSink: ((String) -> Unit)?, ): List { @@ -1041,6 +1117,16 @@ object SleepStager { "sparseBridge", "sparse=true gapMin=$sparseBridgeGapMin runsBefore=$runsBeforeBridge runsAfter=$runsAfterBridge")) } + // Wake-gap bridge (experimental, default OFF — see the section comment above + // [wakeBridgeMaxMin]): assemble one candidate run per sleep period across short + // (≤ wakeBridgeMaxMin) wake gaps, BEFORE the ladder — so sub-minSleepMin fragments between + // arousals survive as part of the night instead of being dropped, and a brief get-up stays + // INSIDE one session as staged wake. Every gate below still applies to the assembled runs. + // Mirrors Swift. + if (wakeBridge) { + runs = bridgeWakeGaps(runs, traceSink = traceSink) + } + val minSleepS = (minSleepMin * 60).toLong() val sessions = ArrayList() diff --git a/android/app/src/main/java/com/noop/analytics/SleepStagerV2.kt b/android/app/src/main/java/com/noop/analytics/SleepStagerV2.kt index e4f63d909c..4ced7dfd53 100644 --- a/android/app/src/main/java/com/noop/analytics/SleepStagerV2.kt +++ b/android/app/src/main/java/com/noop/analytics/SleepStagerV2.kt @@ -214,11 +214,11 @@ object SleepStagerV2 { * Measured on one wearer's 36 recorded nights, against the strap's own band `sleep_state` (an * independent reference the recipe cannot contaminate — 21 nights, 15 554 epochs): sleep/wake kappa * 0.105 → 0.118 and wake sensitivity 16.0 % → 17.6 %, with the healthy-stratum wake fraction essentially - * unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out). Confirmed afterwards against - * human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / 26 773 epochs, #991): 4-class kappa - * 0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → 30.84 %, and the #437 stage-fraction - * guard holds against truth as well. n = 1 wearer for the band figures; see `Tools/SleepBench` and the - * PR for the full ablation and its limits. */ + * unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out) and first-REM latency MAE 53.9 → 41.6 + * min. Confirmed afterwards against human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / + * 26 773 epochs, #991): 4-class kappa 0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → + * 30.84 %, and the #437 stage-fraction guard holds against truth as well. n = 1 wearer for the band + * figures; see `Tools/SleepBench` and the PR for the full ablation and its limits. */ internal val transition: Map> = mapOf( "deep" to mapOf("deep" to 0.86, "rem" to 0.007, "light" to 0.126, "awake" to 0.007), "rem" to mapOf("deep" to 0.005, "rem" to 0.88, "light" to 0.10, "awake" to 0.015), diff --git a/android/app/src/main/java/com/noop/ble/PuffinExperiment.kt b/android/app/src/main/java/com/noop/ble/PuffinExperiment.kt index 8f070188c0..6e0c1f2d14 100644 --- a/android/app/src/main/java/com/noop/ble/PuffinExperiment.kt +++ b/android/app/src/main/java/com/noop/ble/PuffinExperiment.kt @@ -96,6 +96,21 @@ class PuffinExperiment(private val prefs: SharedPreferences) { get() = prefs.getBoolean(KEY_MOTION_AWARE_WAKE, false) set(v) = prefs.edit().putBoolean(KEY_MOTION_AWARE_WAKE, v).apply() + /** Opt-in "Bridge brief wakes" (default OFF): inside [com.noop.analytics.SleepStager.detectSleep], + * adjacent sleep runs across a ≤ 45-min wake gap are assembled into ONE session with the wake + * staged inside it, so a brief get-up no longer splits the night into separate stored sessions, + * and sub-1h sleep fragments between arousals are no longer dropped outright. Every existing + * guard (min duration, the 16-h span cap — which the bridge also refuses to assemble past — HR + * confirmation, off-wrist, daytime nap bars) still applies to the assembled run. Default OFF per + * the varying-input validation discipline (#194/#345). NOTE: an HR-led rescue of + * restless-but-asleep nights was prototyped behind this flag and WITHDRAWN after a real-data + * replay — see the Swift history note above `SleepStager.wakeBridgeMaxMin` before re-attempting + * it. Pure analysis flag (sends nothing to the strap), model-agnostic. Mirrors the macOS + * `PuffinExperiment.wakeBridgeKey`. */ + var sleepWakeBridge: Boolean + get() = prefs.getBoolean(KEY_SLEEP_WAKE_BRIDGE, false) + set(v) = prefs.edit().putBoolean(KEY_SLEEP_WAKE_BRIDGE, v).apply() + /** * Turn OFF every 5/MG-only experimental probe: protocol probes ([isEnabled]), raw capture * ([isCaptureEnabled]), the R22 deep-data strap write ([isDeepDataEnabled]) and broadcast-HR @@ -146,6 +161,7 @@ class PuffinExperiment(private val prefs: SharedPreferences) { /** "Motion-aware wake refinement" opt-in (mirrors macOS `PuffinExperiment.motionAwareWakeKey`). */ const val KEY_MOTION_AWARE_WAKE = "noopMotionAwareWake" + const val KEY_SLEEP_WAKE_BRIDGE = "noopSleepWakeBridge" fun from(context: Context): PuffinExperiment = PuffinExperiment(context.getSharedPreferences(PREFS, Context.MODE_PRIVATE)) diff --git a/android/app/src/main/java/com/noop/ble/RawHistoryArchive.kt b/android/app/src/main/java/com/noop/ble/RawHistoryArchive.kt index 51a8e786bd..cca3688130 100644 --- a/android/app/src/main/java/com/noop/ble/RawHistoryArchive.kt +++ b/android/app/src/main/java/com/noop/ble/RawHistoryArchive.kt @@ -21,12 +21,14 @@ import java.io.FileOutputStream * {"capturedAtMs":,"trim":,"family":"whoop4"|"whoop5","frameHex":""} * Each [append] flushes + fsyncs before returning, so a row is durable before the caller acks. * - * Size cap ([maxBytes], ~5 MB): if appending would push the file past the cap, [append] EVICTS oldest - * surplus lines to make room rather than refusing the write — but only down to a per-version retention - * floor ([PER_VERSION_FLOOR]), so a brand-new layout version is never binned merely because common - * versions filled the file. Only when the incoming frames alone can't fit even an empty archive does - * [append] skip them ([AppendResult.written] = false); the caller records those as unarchived so the - * sync status never falsely claims they were preserved. (#344) + * Size cap ([maxBytes], ~5 MB): if appending would push the file past the cap, [append] EVICTS surplus + * lines to make room rather than refusing the write — weakest-value first, and never below a per-version + * retention floor ([PER_VERSION_FLOOR] for informative lines, [ZERO_PAYLOAD_FLOOR] for lines whose + * payload region is entirely zero), so a brand-new layout version is never binned merely because common + * versions — or a 1 Hz flood of empty placeholder records — filled the file. See [evictLines] for the + * full priority order. Only when the incoming frames alone can't fit even an empty archive does [append] + * skip them ([AppendResult.written] = false); the caller records those as unarchived so the sync status + * never falsely claims they were preserved. (#344) * * A genuine WRITE FAILURE (I/O error) instead throws — the caller treats that as "do NOT ack", so the * strap keeps the records and re-sends them on the next offload. No data is lost either way. @@ -36,6 +38,7 @@ class RawHistoryArchive( private val maxBytes: Long = REJECTED_ARCHIVE_MAX_BYTES, // Overridable so tests can drive eviction with a small archive instead of 5 MB of frames. (#344) private val perVersionFloor: Int = PER_VERSION_FLOOR, + private val zeroPayloadFloor: Int = ZERO_PAYLOAD_FLOOR, ) { /** * Outcome of an [append]. [ok] is true whenever the offload may proceed to ack; [written] is true @@ -87,7 +90,7 @@ class RawHistoryArchive( // the newest [PER_VERSION_FLOOR] of its version. Atomic rewrite (temp file + rename) so the // archive is durable BEFORE the ack. [evictLines] is the pure, unit-tested core. val existing = if (f.exists()) f.readLines().filter { it.isNotEmpty() }.map { "$it\n" } else emptyList() - val kept = evictLines(existing + newLines, maxBytes, perVersionFloor) + val kept = evictLines(existing + newLines, maxBytes, perVersionFloor, zeroPayloadFloor) val tmp = File(context.filesDir, "$REJECTED_ARCHIVE_FILE.tmp") FileOutputStream(tmp).use { out -> out.write(kept.joinToString("").toByteArray(Charsets.UTF_8)) @@ -168,7 +171,15 @@ class RawHistoryArchive( /** Archive filename in the app-private filesDir. */ const val REJECTED_ARCHIVE_FILE = "rejected_history.jsonl" - /** ~5 MB cap; above this [append] evicts oldest surplus (floor-aware) to make room (#344). */ + /** + * ~5 MB cap; above this [append] evicts surplus (weakest-value first, floor-aware) to make room + * (#344). See [evictLines] for the full priority order. + * + * Deliberately NOT raised to survive a placeholder flood. At ~3.2 KB per hex line a 1 Hz stream of + * 1584-byte records fills any cap in hours, so a bigger number buys a linear factor of time and + * costs every user disk — while [ZERO_PAYLOAD_FLOOR] removes the pressure outright by evicting the + * informationless records first. Value, not volume, is what was missing. + */ const val REJECTED_ARCHIVE_MAX_BYTES = 5L * 1024 * 1024 /** @@ -181,6 +192,69 @@ class RawHistoryArchive( */ const val PER_VERSION_FLOOR = 64 + /** + * Per-version retention floor for lines whose PAYLOAD REGION IS ENTIRELY ZERO — far smaller than + * [PER_VERSION_FLOOR], because such a record carries no information beyond "the strap banked a + * record at time T". + * + * Measured motivation: with `enable_raw_data_w_ecg` armed, a WHOOP MG banks one 1584-byte record + * PER SECOND whose every byte from offset 21 to the CRC32 trailer is zero (invariant across 1,307 + * captured frames). At ~3.2 KB per hex JSONL line that fills the whole 5 MB archive in ~25 + * minutes, so under a purely age-ordered policy an actual ECG record captured during the + * electrode-contact window is evicted by placeholders within the half hour. Keeping a handful + * still proves the artefact exists (and dates it); keeping thousands buys nothing and costs the + * evidence. Twin of the Swift `RawHistoryArchive.zeroPayloadFloor`. + */ + const val ZERO_PAYLOAD_FLOOR = 8 + + /** Bytes of the CRC32 trailer excluded from the payload region. */ + const val CRC_TRAILER_BYTES = 4 + + /** + * A payload region shorter than this is not judged at all ([hasZeroPayload] returns false): a + * truncated or tiny frame is not evidence of an empty record, and must not lose retention + * priority on the strength of a few zero bytes. + */ + const val MIN_PAYLOAD_BYTES_TO_JUDGE = 32 + + /** + * First byte of a record's PAYLOAD region — everything past the record header, ending at the + * 4-byte CRC32 trailer. 21 on WHOOP 5/MG (magic @0, seq @11, unix ts @15-18, a constant @19-20) + * and 17 on WHOOP 4: the same +4 that shifts the type byte 4→8 and the version byte 5→9, because + * the puffin envelope is 4 bytes longer. Starting AFTER the header is load-bearing — a region + * that included the non-zero seq/timestamp bytes would never test as all-zero and + * [hasZeroPayload] would never fire. Twin of the Swift `RawHistoryArchive.payloadStart`. + */ + fun payloadStart(family: DeviceFamily): Int = if (family == DeviceFamily.WHOOP5) 21 else 17 + + /** + * True when EVERY byte of [frame]'s payload region is zero — an informationless record. + * + * Deliberately the STRICT test: a single non-zero byte anywhere in the region makes the frame + * informative and gives it full retention priority. "Mostly zero" is NOT the test — a novel + * record that carries one small populated block inside an otherwise-empty buffer is precisely the + * thing this archive exists to catch. Twin of the Swift `RawHistoryArchive.hasZeroPayload`. + */ + fun hasZeroPayload(frame: ByteArray, family: DeviceFamily): Boolean { + val start = payloadStart(family) + val end = frame.size - CRC_TRAILER_BYTES + if (end - start < MIN_PAYLOAD_BYTES_TO_JUDGE) return false + for (i in start until end) if (frame[i].toInt() != 0) return false + return true + } + + /** + * Fraction of the cap freed once eviction has to run at all: the rewrite evicts down to + * `maxBytes - maxBytes/LOW_WATER_DIVISOR` rather than to exactly the cap. + * + * Without this, an archive sitting AT the cap does a full read-parse-rewrite of the whole file on + * EVERY subsequent batch — and the batches that matter here arrive during an offload at up to one + * record per strap-second. Leaving an eighth of the cap free amortises that to one rewrite per + * ~600 KB of new frames. The bytes given up are the ones eviction would have taken next anyway, + * in the same priority order. Twin of the Swift `RawHistoryArchive.lowWaterDivisor`. + */ + const val LOW_WATER_DIVISOR = 8 + /** * The hist_version byte distinguishing one historical layout from another: `frame[5]` on WHOOP * 4, `frame[9]` on WHOOP 5/MG (the puffin envelope is 4 bytes longer). Same indices the reject @@ -198,46 +272,91 @@ class RawHistoryArchive( private data class VersionKey(val family: String, val version: Int) /** - * The [VersionKey] of one archived JSONL line, re-derived from its stored frame + family; null - * for a malformed line (which then gets NO retention floor — it is evicted before any real, - * floor-protected line, so garbage can never occupy a slot a real rare version could use). + * How one archived line is classified for retention: which version bucket it belongs to, and + * whether its payload region is entirely zero (informationless — see [hasZeroPayload]). + */ + private data class LineClass(val key: VersionKey, val zeroPayload: Boolean) + + /** + * Classify one archived JSONL line from its stored frame + family; null for a + * malformed/unparseable line — those get NO retention floor (evict them first, never let garbage + * occupy a floor slot a real rare version could use), and are always evictable before any real, + * floor-protected line. */ - private fun lineVersionKey(line: String): VersionKey? { - val parsed = parseArchiveLine(line) ?: return null - val tag = if (parsed.second == DeviceFamily.WHOOP5) "whoop5" else "whoop4" - return VersionKey(tag, versionByte(parsed.first, parsed.second)) + private fun lineClass(line: String): LineClass? { + val (frame, family) = parseArchiveLine(line) ?: return null + val tag = if (family == DeviceFamily.WHOOP5) "whoop5" else "whoop4" + return LineClass(VersionKey(tag, versionByte(frame, family)), hasZeroPayload(frame, family)) } /** * Floor-aware cap eviction (#344), PURE so it is JVM-unit-testable without a Context. [lines] is - * the full newline-terminated JSONL set, oldest-first (existing ++ incoming). Drops oldest - * SURPLUS lines until the UTF-8 byte total fits [maxBytes], but NEVER a line within the newest - * [floor] lines of its version — so a rare never-seen layout (WHOOP 4 v19, WHOOP 5 v20/v21) - * always survives even when common versions fill the archive. Bounded: if everything left is - * floor-protected it stops even if still over cap (the floor wins — floor x distinctVersions is - * tiny). Mirrors the macOS RawHistoryArchive.evictLines. + * the full newline-terminated JSONL set, oldest-first (existing ++ incoming). + * + * Retention priority, weakest first — everything in a band is evicted before anything in the next: + * 1. malformed lines (unparseable — no version, no floor); + * 2. surplus lines whose payload region is ENTIRELY ZERO ([hasZeroPayload]), oldest first; + * 3. surplus informative lines, oldest first; + * 4. floor-protected lines, never evicted: the newest [zeroFloor] zero-payload lines of each + * version and the newest [floor] informative lines of each version — so a rare never-seen + * layout (WHOOP 4 v19, WHOOP 5 v20/v21) always survives even when common versions fill the + * archive. + * + * Band 2 is what keeps an ECG capture recoverable. A 5/MG with `enable_raw_data_w_ecg` armed banks + * one all-zero 1584-byte record per second; age-ordered eviction alone would flush a real + * contact-window record out of a 5 MB archive within ~25 minutes of placeholders, and an all-zero + * payload demonstrably carries nothing that would be worth that trade. Band 4's split floor means + * a version whose records are ALL empty still keeps a few dated samples of the artefact, while the + * full [floor] stays reserved for records that actually contain something. + * + * Once eviction has to run at all it evicts to [LOW_WATER_DIVISOR] below the cap rather than to + * exactly the cap, so a 1 Hz stream no longer rewrites the whole file once per record. + * + * Bounded: if everything left is floor-protected it stops even if still over cap (the floor wins — + * `(floor + zeroFloor) x distinctVersions` is tiny). Mirrors the macOS + * `RawHistoryArchive.evictLines`; [zeroFloor] is defaulted for exactly the reason it is defaulted + * on the Swift side — so existing call sites do not have to churn. */ - fun evictLines(lines: List, maxBytes: Long, floor: Int): List { - // Mark the newest [floor] indices of each version as protected (scan newest -> oldest). - // Malformed lines (null key) are never protected — evicted before any real, floor-kept line. - val keys = lines.map { lineVersionKey(it) } + fun evictLines( + lines: List, + maxBytes: Long, + floor: Int, + zeroFloor: Int = ZERO_PAYLOAD_FLOOR, + ): List { + // Mark the newest [floor] informative and newest [zeroFloor] zero-payload indices of each + // version as protected (scan newest -> oldest). Malformed lines (null class) are never + // protected — evicted before any real, floor-kept line. + val classes = lines.map { lineClass(it) } val protectedIdx = HashSet() - val seen = HashMap() + val seenInformative = HashMap() + val seenZero = HashMap() for (idx in lines.indices.reversed()) { - val k = keys[idx] ?: continue - val c = seen[k] ?: 0 - if (c < floor) { protectedIdx.add(idx); seen[k] = c + 1 } + val c = classes[idx] ?: continue + if (c.zeroPayload) { + val n = seenZero[c.key] ?: 0 + if (n < zeroFloor) { protectedIdx.add(idx); seenZero[c.key] = n + 1 } + } else { + val n = seenInformative[c.key] ?: 0 + if (n < floor) { protectedIdx.add(idx); seenInformative[c.key] = n + 1 } + } } var total = lines.sumOf { it.toByteArray(Charsets.UTF_8).size.toLong() } if (total <= maxBytes) return lines - // Evict oldest unprotected lines first. + val target = maxBytes - maxBytes / LOW_WATER_DIVISOR + // Evict in weakest-first bands; within a band, oldest first (ascending index). val dropped = HashSet() - for (idx in lines.indices) { // ascending = oldest first - if (total <= maxBytes) break - if (idx in protectedIdx) continue - dropped.add(idx) - total -= lines[idx].toByteArray(Charsets.UTF_8).size.toLong() + fun evict(admits: (Int) -> Boolean) { + for (idx in lines.indices) { // ascending = oldest first + if (total <= target) break + if (idx in protectedIdx || idx in dropped) continue + if (!admits(idx)) continue + dropped.add(idx) + total -= lines[idx].toByteArray(Charsets.UTF_8).size.toLong() + } } + evict { classes[it] == null } // 1. malformed + evict { classes[it]?.zeroPayload == true } // 2. zero-payload surplus + evict { true } // 3. informative surplus return lines.filterIndexed { idx, _ -> idx !in dropped } } diff --git a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt index cd1bf21b8e..bbc1ff2b68 100644 --- a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt +++ b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt @@ -691,9 +691,21 @@ class WhoopBleClient( /** * Idle watchdog: if no genuine offload frame arrives for this long mid-session, end the * session (the durable strap_trim cursor means the next session resumes where we left off). - * Generous (60s, not 20s) because the type-43 raw flood eats BLE airtime between chunks. + * Must clear the real inter-frame tail: the type-43 raw flood eats BLE airtime between chunks, so + * a watchdog shorter than the tail cuts sessions short mid-drain. + * + * 60s was set without measuring that tail; it is ~24x the real worst case, and every session pays + * it in full as dead air on exit. A 3,018-buffer arrival trace from a real 19 h drain puts + * inter-frame gaps at p50 64 ms / p90 106 ms / p99 273 ms / max 2,520 ms, with the tail at chunk + * boundaries (~51 records, where decode + insert + cursor + ack happen), not mid-chunk. 15s is ~6x + * that measured max and still clears the flood. + * + * Deliberately NOT the ~5s the tail alone would justify: that tail is one drain on one firmware, + * and the risk is asymmetric — firing early is safe but wasteful (the in-memory chunk is dropped + * un-acked, so the strap keeps it and the next session refetches), while firing late only costs + * idle seconds. Mirrors Swift BLEManager.backfillIdleTimeoutSeconds. */ - private const val BACKFILL_IDLE_TIMEOUT_MS = 60_000L + private const val BACKFILL_IDLE_TIMEOUT_MS = 15_000L /** Deferral before the first connect-time offload, so SET_CLOCK/GET_DATA_RANGE round-trip first. */ private const val INITIAL_BACKFILL_DELAY_MS = 1_500L /** 5/MG fail-open gate: how long to wait for a GET_DATA_RANGE SUCCESS before requesting @@ -2114,6 +2126,8 @@ class WhoopBleClient( useExperimentalSleepV2 = PuffinExperiment.from(context).experimentalSleepV2, // Opt-in motion-aware wake refinement (#364 follow-up) — same Context-free threading. useMotionAwareWake = PuffinExperiment.from(context).motionAwareWake, + // Wake-gap bridging (experimental, default OFF) — same Context-free threading. + useWakeBridge = PuffinExperiment.from(context).sleepWakeBridge, // Sleep & Rest test mode (Test Centre E5): when the SLEEP domain is on, route this // post-backfill pass's per-day sleep gate trace into the .sleep-tagged strap log, so a // shared report carries the staging proof from THIS scoring pass too, not only the UI diff --git a/android/app/src/main/java/com/noop/data/DeviceRegistry.kt b/android/app/src/main/java/com/noop/data/DeviceRegistry.kt index 435635ef12..30b625944e 100644 --- a/android/app/src/main/java/com/noop/data/DeviceRegistry.kt +++ b/android/app/src/main/java/com/noop/data/DeviceRegistry.kt @@ -85,6 +85,7 @@ class DeviceRegistry( dao.reKeyPpgWaveform(activeId, serialId); dao.deletePpgWaveformFor(activeId) dao.reKeyRawImu(activeId, serialId); dao.deleteRawImuFor(activeId) dao.reKeyV18Aux(activeId, serialId); dao.deleteV18AuxFor(activeId) + dao.reKeySpo2Pct(activeId, serialId); dao.deleteSpo2PctFor(activeId) dao.reKeyEvents(activeId, serialId); dao.deleteEventsFor(activeId) dao.reKeyBattery(activeId, serialId); dao.deleteBatteryFor(activeId) dao.reKeyDailyMetrics(activeId, serialId); dao.deleteDailyMetricsFor(activeId) @@ -153,6 +154,7 @@ class DeviceRegistry( dao.deletePpgWaveformFor(id) dao.deleteRawImuFor(id) // #423 dao.deleteV18AuxFor(id) + dao.deleteSpo2PctFor(id) dao.deleteEventsFor(id) dao.deleteBatteryFor(id) dao.deleteDailyMetricsFor(id) diff --git a/android/app/src/main/java/com/noop/data/DeviceRegistryDao.kt b/android/app/src/main/java/com/noop/data/DeviceRegistryDao.kt index 4f3d86666d..76d594146a 100644 --- a/android/app/src/main/java/com/noop/data/DeviceRegistryDao.kt +++ b/android/app/src/main/java/com/noop/data/DeviceRegistryDao.kt @@ -78,6 +78,11 @@ interface DeviceRegistryDao { @Query("DELETE FROM ppgWaveformSample WHERE deviceId = :deviceId") suspend fun deletePpgWaveformFor(deviceId: String) @Query("DELETE FROM rawImuSample WHERE deviceId = :deviceId") suspend fun deleteRawImuFor(deviceId: String) // #423 @Query("DELETE FROM v18AuxSample WHERE deviceId = :deviceId") suspend fun deleteV18AuxFor(deviceId: String) + // v34/MIGRATION_25_26: the durable `@82` SpO2 percentages OUTLIVE the aux table they are forked from + // (never pruned), which makes covering them here MORE important, not less — a wipe that cleared + // `v18AuxSample` but left this behind would leave years of blood-oxygen readings on the device after + // the user asked for them to be gone. Twin of the Swift `DeviceRegistryStore.deviceScopedTables` entry. + @Query("DELETE FROM spo2PctSample WHERE deviceId = :deviceId") suspend fun deleteSpo2PctFor(deviceId: String) @Query("DELETE FROM event WHERE deviceId = :deviceId") suspend fun deleteEventsFor(deviceId: String) @Query("DELETE FROM battery WHERE deviceId = :deviceId") suspend fun deleteBatteryFor(deviceId: String) @Query("DELETE FROM dailyMetric WHERE deviceId = :deviceId") suspend fun deleteDailyMetricsFor(deviceId: String) @@ -115,6 +120,7 @@ interface DeviceRegistryDao { @Query("UPDATE OR IGNORE ppgWaveformSample SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyPpgWaveform(from: String, to: String) @Query("UPDATE OR IGNORE rawImuSample SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyRawImu(from: String, to: String) @Query("UPDATE OR IGNORE v18AuxSample SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyV18Aux(from: String, to: String) + @Query("UPDATE OR IGNORE spo2PctSample SET deviceId = :to WHERE deviceId = :from") suspend fun reKeySpo2Pct(from: String, to: String) @Query("UPDATE OR IGNORE event SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyEvents(from: String, to: String) @Query("UPDATE OR IGNORE battery SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyBattery(from: String, to: String) @Query("UPDATE OR IGNORE dailyMetric SET deviceId = :to WHERE deviceId = :from") suspend fun reKeyDailyMetrics(from: String, to: String) diff --git a/android/app/src/main/java/com/noop/data/Entities.kt b/android/app/src/main/java/com/noop/data/Entities.kt index 698031132c..08ce1de07a 100644 --- a/android/app/src/main/java/com/noop/data/Entities.kt +++ b/android/app/src/main/java/com/noop/data/Entities.kt @@ -687,3 +687,42 @@ data class V18AuxSampleEntity( return result } } + +/** + * One in-band `@82` SpO2 percentage from a WHOOP 5/MG v18 record, banked DURABLY + * (MIGRATION_25_26 / Swift GRDB `v34-spo2-pct-durable`). + * + * WHY THIS EXISTS AS ITS OWN TABLE. The byte is already carried raw inside [V18AuxSampleEntity.fields] + * (slot [V18AuxSlot.AUX_BYTE_82]), but `v18AuxSample` is CAPPED at + * [WhoopRepository.V18_AUX_RETENTION_ROWS] (604,800 rows/device, ~7 days at 1 Hz) because the aux blob + * costs ~30 MB/day. So every SpO2 reading rolled off after a week and was gone permanently: the strap + * trims its own history the moment an offload is acked, so no second copy exists anywhere. Raising the + * aux cap is the wrong fix — it buys a year of SpO2 at the price of ~11 GB of unrelated aux bytes. The + * in-band signal is only ~2,000 samples/week (~100k rows/year), so this narrow sibling retains it forever + * at trivial cost while the wide aux capture stays capped and unchanged. + * + * WHY NOT EXTEND [Spo2Sample] (the v3 table). Wrong shape, and extending it would corrupt a live reader: + * `spo2Sample` is `(deviceId, ts, red, ir)`, two NOT NULL raw ADC channels off the WHOOP 4.0 v24 layout, + * and [com.noop.analytics.AnalyticsEngine.nightlySpo2RawMeans] averages them into `spo2Red`/`spo2Ir`. + * We have a computed PERCENTAGE and no red/IR at all, so every insert would have to fabricate zeros and + * drag those means toward zero. Two different physical quantities from two device generations. + * + * [pct] is the RAW in-band byte ([SPO2_CANDIDATE_IN_BAND], 70..100), stored verbatim. Run grouping and + * acquisition-ramp trimming are READ-TIME policies (see + * [com.noop.analytics.AnalyticsEngine.nightlySpo2Pct]) — they have already changed once on the cloud + * reader, and a store that baked the old policy in would have destroyed the evidence needed to change + * them again. The only thing applied on the way in is the [SPO2_CANDIDATE_IN_BAND] demultiplex, which is + * not a policy but the definition of which values on `@82` are percentages at all. + * + * NOT PRUNED, by design — the same "durable decoded biometric history" class as `hrSample` and + * `ppgWaveformSample`. The `v18AuxSample` rolling sweep deletes only `FROM v18AuxSample`, so this table + * is untouched by it. PK (deviceId, ts) and field order (deviceId, ts, pct) mirror the GRDB schema, so a + * `.noopbak` round-trips. + */ +@Entity(tableName = "spo2PctSample", primaryKeys = ["deviceId", "ts"]) +data class Spo2PctSampleEntity( + val deviceId: String, + val ts: Long, + /** The raw in-band `@82` byte, 70..100. Never a sentinel, never a diagnostic code, never 0. */ + val pct: Int, +) diff --git a/android/app/src/main/java/com/noop/data/V18AuxCodec.kt b/android/app/src/main/java/com/noop/data/V18AuxCodec.kt index 1e012ec2e1..c9f7285e7d 100644 --- a/android/app/src/main/java/com/noop/data/V18AuxCodec.kt +++ b/android/app/src/main/java/com/noop/data/V18AuxCodec.kt @@ -182,6 +182,42 @@ data class V18AuxRow( } } +/** + * The in-band window for the `@82` SpO2 candidate byte: a value in `70..100` is a MEASUREMENT, and + * everything else on that byte is something else entirely. + * + * `@82` is MULTIPLEXED and separable BY VALUE — this is the separation, established over 626,725 + * production records: + * - `70..100` a real percentage. Distribution modes at 97; per-night medians 94-97; sleep-gated; + * emitted as a run of ~30 one-second samples once per ~1,200 s. + * - bit 7 set status sentinel (`0x80`, `0x88`, `0x90`, `0xA0`, `0xA8` are the observed values). + * - `1..69` diagnostic codes, not a saturation of anything. + * - `0` the strap emitted nothing this second (awake, or between runs). + * - `101..127` never observed — the gap is what makes the bit-7 sentinels and the measurements + * unambiguously separable without a side channel. + * + * This is the SAME window the decoder applies when it emits `spo2_candidate_82`, the same one + * [com.noop.analytics.AnalyticsEngine.nightlySpo2CandidateMean] applies, and the same one + * [WhoopRepository.insert] applies when forking [Spo2PctSampleEntity] rows out of the aux stream. + * Named here so the three agree structurally rather than by three matching literals. Byte-identical + * twin of the Swift `spo2CandidateInBand` (WhoopProtocol/Streams.swift). + */ +val SPO2_CANDIDATE_IN_BAND: IntRange = 70..100 + +/** + * One in-band `@82` SpO2 percentage as it travels through a decoded batch, before it is banked as a + * [Spo2PctSampleEntity]. Twin of the Swift `Spo2PctSample`. + * + * LOSSLESS BY CONSTRUCTION: [pct] is the raw in-band byte, verbatim. No run grouping, no ramp trimming, + * no smoothing, no dedupe beyond the (deviceId, ts) primary key — see [Spo2PctSampleEntity] for why. + */ +data class Spo2PctRow( + /** Wall-clock unix seconds — one per strap-second that carried a reading. */ + val ts: Long, + /** The raw in-band `@82` byte, 70..100. */ + val pct: Int, +) + /** * Storage codec for the 5/MG v18 auxiliary-field stream (`v18AuxSample.fields`). * diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 3b3ea1109c..d1d42ecf19 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -99,6 +99,18 @@ interface WhoopDao : DeviceRegistryDao { @Insert(onConflict = OnConflictStrategy.IGNORE) suspend fun insertV18Aux(rows: List): List + /** + * The durable in-band `@82` SpO2 percentages (v34 / MIGRATION_25_26), forked out of the SAME v18 aux + * records `insertV18Aux` banks. IGNORE keeps the FIRST-seen value for a second, matching every other + * per-second stream's dedupe rule and making a re-offload of the same window idempotent. Swift twin: + * the `ON CONFLICT(deviceId, ts) DO NOTHING` insert in `StreamStore.insert`. + * + * There is deliberately NO prune query beside this one: unlike `v18AuxSample` this table is never + * swept, which is the entire reason it exists. + */ + @Insert(onConflict = OnConflictStrategy.IGNORE) + suspend fun insertSpo2Pct(rows: List): List + /** * Bound the v18 aux table to the newest [keep] rows for [deviceId] (rolling retention, v31). Same * shape as [pruneRawImu] — this is instrumentation nothing reads yet, so it is capped rather than @@ -320,6 +332,20 @@ interface WhoopDao : DeviceRegistryDao { suspend fun v18AuxSamples(deviceId: String, from: Long, to: Long, limit: Int): List + /** + * The durable in-band `@82` SpO2 percentages in [from, to] (ascending). Unlike [v18AuxSamples] this + * reaches back over the WHOLE recorded history — `spo2PctSample` is never pruned, while the aux table + * it is forked from is capped at ~7 days. Empty on a WHOOP 4.0 (no v18 aux stream) and for any window + * offloaded before MIGRATION_25_26. Values are RAW in-band bytes; run grouping and acquisition-ramp + * trimming are the CALLER's policy. Swift twin: `WhoopStore.spo2PctSamples`. + */ + @Query( + "SELECT * FROM spo2PctSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + + "ORDER BY ts ASC LIMIT :limit" + ) + suspend fun spo2PctSamples(deviceId: String, from: Long, to: Long, limit: Int): + List + /** Aggregate HR over a window (one indexed (deviceId,ts) range scan — no row materialisation, * no [hrSamples] LIMIT truncation). Backs the imported-workout HR fallback (#77). * @@ -876,6 +902,13 @@ interface WhoopDao : DeviceRegistryDao { @Query("DELETE FROM v18AuxSample WHERE ts < :minTs OR ts > :maxTs") suspend fun pruneV18AuxByTs(minTs: Long, maxTs: Long): Int + // The durable SpO2 percentages are keyed by the SAME type-47 `ts` as every stream above and are + // forked from the same records, so a bad-clock strap garbages them identically. They are never + // pruned on age, so an unhealed out-of-bounds row here would persist FOREVER rather than rolling off + // — which makes the heal more important here, not less. Swift twin: TimestampHeal.rawTables. + @Query("DELETE FROM spo2PctSample WHERE ts < :minTs OR ts > :maxTs") + suspend fun pruneSpo2PctByTs(minTs: Long, maxTs: Long): Int + @Query("DELETE FROM event WHERE ts < :minTs OR ts > :maxTs") suspend fun pruneEventByTs(minTs: Long, maxTs: Long): Int diff --git a/android/app/src/main/java/com/noop/data/WhoopDatabase.kt b/android/app/src/main/java/com/noop/data/WhoopDatabase.kt index f51bf44f1e..52739f9ade 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDatabase.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDatabase.kt @@ -51,8 +51,9 @@ import androidx.sqlite.db.SupportSQLiteDatabase PpgWaveformSampleEntity::class, RawImuSampleEntity::class, V18AuxSampleEntity::class, + Spo2PctSampleEntity::class, ], - version = 25, + version = 26, // #775: ON so Room's KSP processor writes the generated schema (every table's exact `CREATE TABLE`, // columns in declaration order with affinity/NOT NULL/default, PK and indices) as JSON. That export // is what lets a plain JVM test — no device, no Robolectric — read Android's REAL schema and compare @@ -691,6 +692,53 @@ abstract class WhoopDatabase : RoomDatabase() { } } + /** + * Retain the WHOOP 5/MG's own computed SpO2 percentage forever instead of losing it after seven + * days. Twin of the Swift GRDB migration `v34-spo2-pct-durable`. + * + * THE PROBLEM THIS FIXES. The 5/MG emits its own SpO2 % on `@82` of the v18 record. That byte has + * been banked since MIGRATION_24_25 — but only inside `v18AuxSample.fields`, a table CAPPED at + * [WhoopRepository.V18_AUX_RETENTION_ROWS] (604,800 rows/device, ~7 days at 1 Hz) because the aux + * blob costs ~30 MB/day. So every reading rolled off after a week and was gone permanently: the + * strap trims its own history the moment NOOP acks the offload, so there is no second copy + * anywhere. Raising the aux cap is the WRONG fix — it buys a year of SpO2 at the price of ~11 GB + * of mostly-unrelated aux bytes. The in-band signal is only ~2,000 samples/week (~100k rows/year), + * so it gets a narrow sibling that is NEVER pruned while the wide aux capture stays capped and + * unchanged. See [Spo2PctSampleEntity] for why this is not a column on `spo2Sample`. + * + * ADDITIVE ONLY: one CREATE TABLE, no rebuild, no row touched, no existing column changed — the + * MIGRATION_9_10 form (a brand-new table). No destructive fallback (see the class doc): a silent + * rebuild would lose already-acked, non-resendable strap history. + * + * `pct` is NOT NULL because a row is written only when the byte carried an in-band measurement; + * absence is encoded as "no row", never as a fabricated 0. That matters here more than usual: `0` + * is a real value ON THE WIRE ("the strap emitted nothing this second"), so a NULLable column + * holding 0 would be indistinguishable from a genuine sentinel. + * + * VERSION SEQUENCING. Room 25 -> 26 pairs with GRDB `v34`, not `v32`/`v33`: GRDB's identifiers and + * Room's integer versions are independent counters that have drifted apart since the fork's + * iOS-only migrations (see `grdbMigrationLineages` in `schema_oracle.json`). Room's version is a + * dense integer sequence and MUST be the next free one; GRDB's is a string and was numbered ahead + * of upstream on purpose. Both are pinned by SchemaOracleTest. + * + * The CREATE TABLE column order matches [Spo2PctSampleEntity]'s field order and the GRDB schema, + * so a migrated schema, a freshly-created one, and a `.noopbak` from iOS all agree. + * + * Exposed as [SPO2_PCT_DURABLE_MIGRATION_SQL] so a plain-JVM unit test can assert its shape + * without an emulator, like the migrations above. + */ + internal val SPO2_PCT_DURABLE_MIGRATION_SQL: List = listOf( + "CREATE TABLE IF NOT EXISTS `spo2PctSample` (`deviceId` TEXT NOT NULL, " + + "`ts` INTEGER NOT NULL, `pct` INTEGER NOT NULL, " + + "PRIMARY KEY(`deviceId`, `ts`))", + ) + + internal val MIGRATION_25_26 = object : Migration(25, 26) { + override fun migrate(db: SupportSQLiteDatabase) { + for (stmt in SPO2_PCT_DURABLE_MIGRATION_SQL) db.execSQL(stmt) + } + } + private fun build(appContext: Context): WhoopDatabase = Room.databaseBuilder(appContext, WhoopDatabase::class.java, DB_NAME) // #1014: replace ONLY the corruption handling of the default open-helper. The @@ -707,7 +755,7 @@ abstract class WhoopDatabase : RoomDatabase() { MIGRATION_10_11, MIGRATION_11_12, MIGRATION_12_13, MIGRATION_13_14, MIGRATION_14_15, MIGRATION_15_16, MIGRATION_16_17, MIGRATION_17_18, MIGRATION_18_19, MIGRATION_19_20, MIGRATION_20_21, MIGRATION_21_22, - MIGRATION_22_23, MIGRATION_23_24, MIGRATION_24_25, + MIGRATION_22_23, MIGRATION_23_24, MIGRATION_24_25, MIGRATION_25_26, ) // #1037: a FRESH install builds the schema straight at the current version and runs NO // migrations, so the MIGRATION_7_8 "my-whoop" registry seed never fires and the WHOOP, diff --git a/android/app/src/main/java/com/noop/data/WhoopRepository.kt b/android/app/src/main/java/com/noop/data/WhoopRepository.kt index 1ae11814b4..6b0f09e011 100644 --- a/android/app/src/main/java/com/noop/data/WhoopRepository.kt +++ b/android/app/src/main/java/com/noop/data/WhoopRepository.kt @@ -492,6 +492,44 @@ class WhoopRepository(private val dao: WhoopDao) { } } + // DURABLE SpO2 (v34 / MIGRATION_25_26) — the in-band `@82` percentages, forked out of the SAME + // v18 aux samples banked just above into a narrow table that is NEVER pruned. The aux blob is + // capped at ~7 days (the sweep above); this is the copy that survives, and without it every + // reading older than a week is destroyed with no second copy anywhere (the strap trims its own + // history as soon as an offload is acked). + // + // READ OFF THE DECODED ROW, NOT THE BLOB. `it.auxByte82` is the slot value; the alternative — + // indexing byte 23 of the packed `fields` blob, which is what the cloud reader does — is correct + // ONLY while every slot ahead of [V18AuxSlot.AUX_BYTE_82] is present, because [V18AuxCodec] is a + // presence-bitmap format that omits absent slots from the body entirely. With all slots present + // the arithmetic happens to land on 23 (5-byte header + 4+1+1+1+1+2+1+1+2+2+2 = 18 body bytes + // ahead of it), which is why `fields[23]` yields physiologically valid values in production — but + // a single absent earlier slot silently shifts it onto a neighbouring byte and the read becomes + // garbage that STILL looks like a plausible percentage. `V18AuxCodecOffsetTest` pins both halves. + // Here the decoded row is already in hand, so the fragile path is not merely avoided, it is + // unreachable. + // + // Separate loop rather than a branch inside the one above, so the empty-blob skip there can never + // silently drop an SpO2 reading. (In practice a present `auxByte82` guarantees a non-empty blob, + // but coupling the two would make that a load-bearing invariant nobody stated.) + // + // Gate: [SPO2_CANDIDATE_IN_BAND] only. `@82` is multiplexed — bit-7 sentinels (0x80/0x88/0x90/ + // 0xA0/0xA8), sub-70 diagnostic codes, and 0-for-not-emitted share the byte with real percentages, + // and 101..127 is an empty band that keeps the two separable. Nothing else is applied: no run + // grouping, no acquisition-ramp trim, no smoothing. Those are read-time policies (they have + // already changed once on the cloud reader) and baking one in here would destroy the evidence + // needed to change it again. Swift twin: the `spo2PctSample` insert in `StreamStore.insert`. + if (streams.v18Aux.isNotEmpty()) { + val spo2PctRows = streams.v18Aux.mapNotNull { + val v = it.auxByte82 ?: return@mapNotNull null + if (v.toInt() !in SPO2_CANDIDATE_IN_BAND) null + else Spo2PctSampleEntity(deviceId, it.ts, v.toInt()) + } + // IGNORE keeps the FIRST value banked for a second, matching every other per-second stream's + // dedupe rule and making a re-offload of the same window idempotent. + if (spo2PctRows.isNotEmpty()) dao.insertSpo2Pct(spo2PctRows) + } + // OnConflictStrategy.IGNORE returns -1 for skipped (already-present) rows; count the inserts. return InsertCounts( hr = hrIds.countInserted() + ppgHrIds.countInserted(), @@ -669,6 +707,7 @@ class WhoopRepository(private val dao: WhoopDao) { deleted += dao.prunePpgWaveformByTs(minTs, maxTs) deleted += dao.pruneRawImuByTs(minTs, maxTs) deleted += dao.pruneV18AuxByTs(minTs, maxTs) + deleted += dao.pruneSpo2PctByTs(minTs, maxTs) deleted += dao.pruneEventByTs(minTs, maxTs) deleted += dao.pruneBatteryByTs(minTs, maxTs) // (b) computed daily metrics (by day key) + sleep sessions (by startTs). The prune queries apply @@ -862,6 +901,26 @@ class WhoopRepository(private val dao: WhoopDao) { dao.v18AuxSamples(deviceId, from, to, limit) .map { V18AuxCodec.unpack(it.fields, it.ts) } + /** + * Durable in-band `@82` SpO2 percentages (v34 / MIGRATION_25_26) in [from, to] for one device, + * ascending by ts. + * + * Unlike [v18AuxSamples], this reaches back over the WHOLE recorded history: `spo2PctSample` is never + * pruned, while the aux table it is forked from is capped at ~7 days. Empty for a WHOOP 4.0 (no v18 + * aux stream) and for any window offloaded before MIGRATION_25_26 — the older aux rows still on the + * device DO carry the byte, but only for whatever slice of the last seven days survives, and this + * reader deliberately does NOT fall back to them: a series that silently changed source partway + * through would be unusable as evidence. Backfilling those survivors is a one-shot migration's job, + * not a read path's. + * + * Values are RAW in-band bytes. Run grouping and acquisition-ramp trimming are the CALLER's policy — + * see [com.noop.analytics.AnalyticsEngine.nightlySpo2Pct] for the nightly one. Kotlin twin of the + * Swift `WhoopStore.spo2PctSamples(deviceId:from:to:)`. + */ + suspend fun spo2PctSamples(deviceId: String, from: Long, to: Long, limit: Int = DEFAULT_LIMIT): + List = + dao.spo2PctSamples(deviceId, from, to, limit).map { Spo2PctRow(it.ts, it.pct) } + /** #423: persist a batch of decoded 5/MG raw-IMU offload buffers (one row per strap-second, packed i16 * BLOB), then bound the table to the newest [RAW_IMU_RETENTION_ROWS] for the device. Comes from the * deep-buffer capture seam, not the normal stream path, so it inserts directly (idempotent by ts). */ diff --git a/android/app/src/main/java/com/noop/ingest/HealthConnectWriter.kt b/android/app/src/main/java/com/noop/ingest/HealthConnectWriter.kt index 72382afb95..e5d9ac5b82 100644 --- a/android/app/src/main/java/com/noop/ingest/HealthConnectWriter.kt +++ b/android/app/src/main/java/com/noop/ingest/HealthConnectWriter.kt @@ -74,6 +74,34 @@ object HealthConnectWriter { /** How far back to write. Recomputation only ever touches recent nights; 60 days is generous. */ private const val WINDOW_DAYS = 60L + /** + * Whether a strap-derived nightly SpO2 is written back to Health Connect as an + * [OxygenSaturationRecord]. Twin of the Swift `HealthKitBridge.exportsSpo2ToHealthKit`. + * + * FALSE, deliberately, and it is not an oversight to be tidied away. + * + * This export loop reads only the COMPUTED NOOP device rows — imported rows never pass through it — + * so before v34 populated `spo2Pct` for WHOOP nights this branch never fired even once. Turning the + * field on therefore does not "resume" an export; it STARTS one, and what it would start writing is a + * number nobody has checked against a reference oximeter. + * + * Why that bar is higher here than for the in-app card. A value in the app is read in context, next + * to its own provenance, by the person who built it. A Health Connect record is different in kind: it + * lands in the system health record under Android's own Blood Oxygen heading, sits alongside + * clinically-sourced readings with nothing distinguishing it, and is readable by every other app the + * user has authorised. Deleting it later is fiddly and does not recall what already propagated. + * + * The evidence genuinely does not reach that bar yet. The decode is well supported — distribution, + * run structure, sleep gating, night-to-night stability — but ALL of it says "this byte is a real + * oxygen measurement", none of it says "97 means 97". Upstream declines to promote the same metric + * for the same reason: two straps were checked against the WHOOP app and moved in OPPOSITE directions. + * + * Flip this to `true` (in lockstep with the Swift constant, or the platforms disagree about what + * leaves the device) once paired reference-oximeter nights exist and the bias is characterised — and, + * if the bias is a fixed offset, correct for it before writing rather than here. + */ + const val EXPORTS_SPO2_TO_HEALTH_CONNECT = false + private val WRITE_RECORDS: List> = listOf( RestingHeartRateRecord::class, HeartRateVariabilityRmssdRecord::class, @@ -139,7 +167,8 @@ object HealthConnectWriter { metadata = meta("hrv", d.day, version), )) } - d.spo2Pct?.let { + // DELIBERATELY NOT EXPORTED while [EXPORTS_SPO2_TO_HEALTH_CONNECT] is false — see the constant. + d.spo2Pct?.takeIf { EXPORTS_SPO2_TO_HEALTH_CONNECT }?.let { records.add(OxygenSaturationRecord( time = instant, zoneOffset = offset, percentage = Percentage(it), metadata = meta("spo2", d.day, version), diff --git a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt index 30f7ca7bb9..09279aaab4 100644 --- a/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt +++ b/android/app/src/main/java/com/noop/protocol/HistoricalStreams.kt @@ -293,6 +293,40 @@ fun decodeHistorical(frame: ByteArray, family: DeviceFamily = DeviceFamily.WHOOP return out } +/** + * The WHOOP 5/MG type-47 `hist_version` values THIS platform has a real field map for. Every other + * version reaches no decoder at all: [decodeWhoop5Historical] returns null and nothing but the raw bytes + * of the record can preserve it. + * + * This is the single source of truth for "does NOOP-Android understand this layout": the version gates in + * [decodeWhoop5Historical] and the v26 branch of [extractHistoricalStreams] read nothing else, and + * [rejectedHistoricalRecords] uses it to decide which records must be archived raw. Keeping one set means + * the two can never disagree — the failure mode this replaced was a record from an unmapped layout that + * happened to decode a plausible `unix` and `gravity_x`/`heart_rate` slipping past the archive filter, so + * its bytes were kept nowhere at all and the strap freed them on the very next trim ack. + * + * ⚠️ DELIBERATELY NARROWER THAN THE SWIFT `mappedWhoop5HistoricalVersions`, which is `[18, 20, 21, 26]`. + * Swift's `decodeWhoop5Historical` dispatches v20 (raw optical block) and v21 (raw 6-axis IMU) through + * `decodeWhoop5HistoricalV2021`; the Kotlin historical path has no such branch — `Whoop5RawOptical` and + * `Whoop5RawImu` exist here but are wired to the LIVE deep-buffer route in `WhoopBleClient`, not to the + * type-47 historical dispatch. Claiming 20/21 here would assert a field map Android does not have, which + * is precisely the "it holds by accident" class of bug this constant was introduced to close. The archive + * OUTCOME is unchanged either way today: an unmapped 5/MG version already decodes to null on Android and + * so was already archived by the decode-outcome route. Add a version here only when a decoder for it + * lands on this side. Pinned by `UnmappedHistoricalLayoutTest`. + */ +val MAPPED_WHOOP5_HISTORICAL_VERSIONS: Set = setOf(18, 26) + +/** + * True when [frame] is a WHOOP 5/MG type-47 record whose layout version has NO field map on this + * platform — the records that are never re-derivable from anything NOOP stores. Non-type-47 and + * too-short frames are false. Twin of the Swift `isUnmappedWhoop5HistoricalRecord`. + */ +fun isUnmappedWhoop5HistoricalRecord(frame: ByteArray): Boolean { + if (frame.size <= 9 || (frame[8].toInt() and 0xFF) != PacketType.HISTORICAL_DATA.rawValue) return false + return (frame[9].toInt() and 0xFF) !in MAPPED_WHOOP5_HISTORICAL_VERSIONS +} + /** * WHOOP 5.0/MG type-47 "v18" historical decode. The puffin envelope is longer, so the record starts at * byte 8 (type@8, version@9) and fields sit at their WHOOP5-ABSOLUTE offsets — NOT the WHOOP4 V24 layout @@ -307,6 +341,11 @@ fun decodeHistorical(frame: ByteArray, family: DeviceFamily = DeviceFamily.WHOOP private fun decodeWhoop5Historical(frame: ByteArray): Map? { if (frame.histU8(8) != PacketType.HISTORICAL_DATA.rawValue) return null val version = frame.histU8(9) ?: return null + // One gate, one list: 18 is the only version this function maps, and v26 (handled by + // [decodeWhoop5HistoricalV26] from [extractHistoricalStreams]) is the only other one this platform + // maps at all — so [MAPPED_WHOOP5_HISTORICAL_VERSIONS] must contain exactly {18, 26}. That set is + // what decides whether a record gets archived raw, so the two cannot be allowed to drift; + // `UnmappedHistoricalLayoutTest` pins the lockstep against the decoder's actual behaviour. if (version != 18) return null val out = LinkedHashMap() @@ -477,10 +516,12 @@ private fun decodeWhoop5HistoricalV26(frame: ByteArray): V26Record? { } /** - * The HISTORICAL_DATA (type-47) record frames in [rawFrames] that genuinely FAIL to decode — a CRC - * failure, or an unmapped firmware layout whose v24 plausibility gate (see [decodeHistorical]) also - * rejects it. These are exactly the record frames [extractHistoricalStreams] silently drops: their - * biometric payload would otherwise be lost forever once the strap trims the acked history. + * The HISTORICAL_DATA (type-47) record frames in [rawFrames] that NOOP cannot turn into rows — a CRC + * failure, an unmapped firmware layout (5/MG: any `hist_version` outside + * [MAPPED_WHOOP5_HISTORICAL_VERSIONS]), or a mapped layout whose envelope parsed but whose v24 + * plausibility gate (see [decodeHistorical]) rejected it. These are the record frames + * [extractHistoricalStreams] silently drops: their biometric payload would otherwise be lost forever once + * the strap trims the acked history. * * EXCLUDED (decode to zero rows BY DESIGN, never "lost" data — must NOT be counted): * - CONSOLE_LOGS (type-50) frames — the strap's own diagnostics text channel. On WHOOP 4.0 the @@ -507,6 +548,24 @@ fun rejectedHistoricalRecords( if (t != PacketType.HISTORICAL_DATA.rawValue) return@filter false // type-50 console / metadata / etc. // WHOOP 5/MG v26 = raw PPG block, deliberately not stored — known-skipped, not lost data. if (family == DeviceFamily.WHOOP5 && frame.histU8(9) == 26) return@filter false + // UNMAPPED LAYOUT (5/MG) — archive UNCONDITIONALLY, whatever it decoded. + // + // The decode-outcome test below is the wrong QUESTION for a layout NOOP has no field map for. + // Today it happens to give the right ANSWER, because [decodeWhoop5Historical] returns null for + // every version outside [MAPPED_WHOOP5_HISTORICAL_VERSIONS] — but that is an accident of the + // decoder's current shape, not a property of the format. One static schema field for type 47, or + // one partially-mapped new version, and a record that yielded a plausible `unix` plus a + // `heart_rate`/`gravity_x` would pass the screen and be kept NOWHERE at all, its bytes freed by + // the very next trim ack. That is exactly the shape a novel record (an on-demand MG ECG capture, + // a new firmware's rollup) can take. Deciding on the VERSION makes it structural. + // + // This cannot flood the archive with records we already understand: the mapped versions never + // reach here. Retention for the records that DO ([RawHistoryArchive.evictLines]) evicts + // entirely-zero-payload frames first, so a firmware that banks empty placeholder records at 1 Hz + // cannot push out the one informative frame either. Twin of the Swift `rejectedHistoricalRecords` + // layout clause. WHOOP 4 is untouched: its unmapped versions go through the v24 fallback below, + // which keeps a record only when it validates physiologically. + if (family == DeviceFamily.WHOOP5 && isUnmappedWhoop5HistoricalRecord(frame)) return@filter true // A type-47 record that [decodeHistorical] cannot turn into usable biometrics — CRC failure or // an unmapped layout the v24-fallback plausibility gate rejected. This is precisely what // [extractHistoricalStreams] drops (`decodeHistorical(...) ?: continue`), so the rejected set diff --git a/android/app/src/main/java/com/noop/protocol/Whoop5Ecg.kt b/android/app/src/main/java/com/noop/protocol/Whoop5Ecg.kt index f1435a5cf8..5fb9390153 100644 --- a/android/app/src/main/java/com/noop/protocol/Whoop5Ecg.kt +++ b/android/app/src/main/java/com/noop/protocol/Whoop5Ecg.kt @@ -16,7 +16,11 @@ package com.noop.protocol // reimplemented here in NOOP's own code — facts with attribution, never copied expression. // // Deliberately NOT asserted: the packet TYPE byte these records arrive under (no capture exists, and the -// PacketType table has no Labrador entry), the WristSelection raw values (right-first is an inference), +// PacketType table has no Labrador entry — an app layer finds it empirically, running +// filteredBytesPerSampleCandidates over unclassified frames and censusing EVERY one of them by type byte, +// hits and misses alike, because a heuristic that logs only its own hits destroys the evidence for its own +// misses), the filtered stream's bytes-per-sample (decodeFiltered implements 2; the triage admits 2, 3 and +// 4 — see FILTERED_WIDTH_CANDIDATES), the WristSelection raw values (right-first is an inference), // the heartKeyProgress "timed out" sentinel, and any clinical meaning at all. The arrhythmia result is // computed on-strap by an embedded third-party classifier; NOOP decodes the byte. NOOP is not a medical // device and this value is not a diagnosis — see DISCLAIMER.md. @@ -112,10 +116,27 @@ data class FilteredLabradorPacket( * * The raw blob is opaque: its bytes-per-sample is `rawECGDataRaw.size / numberOfECGSamples`, which means * the blob's LENGTH is not itself on the wire (see [Whoop5Ecg.rawBytesPerSampleCandidates]). + * + * BOTH variable-length-looking regions are really FIXED-SIZE containers with a count that says how many + * leading slots are valid: the sample region (see [unusedSampleBytes] and [Whoop5Ecg.decodeRaw]) and the + * leads-off block ([Whoop5Ecg.LEADS_OFF_SLOT_COUNT] slots for I and the same again for Q, with + * [numberOfLeadsOffSamples] selecting how many are valid). In both, the unused slots are dropped rather + * than carried, so a packet's arrays only ever hold values the record says are real. */ data class RawLabradorPacket( val header: EcgStatusHeader, + /** + * Exactly `numberOfECGSamples * bytesPerSample` bytes — the VALID part of the sample region, which on + * a partly-filled record is shorter than the region itself. + */ val rawECGDataRaw: List, + /** + * Bytes of the fixed sample region that `numberOfECGSamples` did not fill. Zero for a full record. + * + * The strap zero-fills them, and the decoder requires that. They are counted, not carried, so + * `HEADER_LENGTH + rawECGDataRaw.size + unusedSampleBytes` is where the leads-off block begins. + */ + val unusedSampleBytes: Int, val numberOfLeadsOffSamples: Int, val leadsOffIRaw: List, val leadsOffQRaw: List, @@ -124,6 +145,10 @@ data class RawLabradorPacket( /** Bytes per raw sample, or null when the packet carried no samples to divide by. */ val bytesPerSample: Int? get() = if (header.numberOfECGSamples > 0) rawECGDataRaw.size / header.numberOfECGSamples else null + + /** The full sample region the record reserved, valid part plus unused capacity. */ + val sampleRegionBytes: Int + get() = rawECGDataRaw.size + unusedSampleBytes } object Whoop5Ecg { @@ -137,6 +162,41 @@ object Whoop5Ecg { /** Trailing bytes tolerated after the last decoded field (the puffin pad4 budget). */ const val DEFAULT_MAX_PADDING = 3 + /** + * Slots in the raw record's leads-off diagnostic block, per array. + * + * MEASURED FROM HARDWARE, not attested: two complete type-47 layout-16 flash records captured + * 2026-08-06 (embedded verbatim in the Swift `Whoop5EcgRawHardwareTests`) show the block is + * FIXED-SIZE — the count byte is followed by eleven i16 I slots and then eleven i16 Q slots, present + * in full whether or not `numberOfLeadsOffSamples` fills them, with the unused tail slots zeroed. + * Both counts seen in the capture (10 and 11) place the Q array at the same offset and leave the same + * single trailing byte, which is what identifies the block as fixed rather than packed. + * + * Reading it as PACKED — `count` elements each — was the original assumption, and it is wrong in two + * compounding ways whenever `count < 11`: Q is read two bytes early per missing slot (so it gains a + * spurious leading value and loses its last real one), and the misplaced end-of-record leaves a + * remainder over [DEFAULT_MAX_PADDING], which discarded 227 of the capture's 351 populated records + * before any field was read. + */ + const val LEADS_OFF_SLOT_COUNT = 11 + + /** + * The bytes-per-sample widths the FILTERED triage admits, in the order it reports them. + * + * 2 leads because it is the width [decodeFiltered] implements and the width the filtered layout is + * documented with, so the triage's earlier behaviour is a strict subset of the widened one. + * + * 3 and 4 are here because the 2-byte assumption was, on this repo's own evidence, unsafe: a populated + * RAW flash record read off hardware carried `numberOfECGSamples = 500` against a 1500-byte sample + * blob — 3 bytes per sample. Nothing attests that the FILTERED stream uses the same width and nothing + * rules it out; what a hardcoded 2 did was make the difference unobservable, because a 3-byte frame + * failed the length agreement and was discarded before anyone could look at it. + * + * Only the WIDTH is loosened — the four Bool-typed bytes, the two classifier enums, the + * signal-quality range and `n > 0` are unchanged. + */ + val FILTERED_WIDTH_CANDIDATES = listOf(2, 3, 4) + // Commands. All four share the shape {revision, arg, padding}; `revision` is the leading inner byte // the 5/MG command family already uses (CLIENT_HELLO, SET_CONFIG), and the struct's trailing padding // is exactly what the puffin pad4 supplies. @@ -295,54 +355,101 @@ object Whoop5Ecg { // Decode — raw + /** + * Bytes the raw record's leads-off block occupies: the count byte, then a full I array, then a full Q + * array. Fixed — see [LEADS_OFF_SLOT_COUNT]. + */ + const val LEADS_OFF_BLOCK_LENGTH = 1 + LEADS_OFF_SLOT_COUNT * 4 + /** * Decode a raw packet from the inner record's PAYLOAD with an explicit sample width. * * The width has to be supplied because the raw blob's length is NOT on the wire. + * + * **The sample region is a fixed-size container, not `n * bytesPerSample` bytes.** MEASURED FROM + * HARDWARE, on the same capture and by the same argument as [LEADS_OFF_SLOT_COUNT]: the capture's one + * partly-filled record (`numberOfECGSamples == 245`, embedded in the Swift `Whoop5EcgRawHardwareTests` + * and its Kotlin twin) is the same 1584-byte frame as every full 500-sample record, its sample data + * stops after `245 * 3` bytes, the rest of the region is zeroed, and its leads-off block sits at the + * SAME frame offset as every other record's. So `numberOfECGSamples` says how many of the region's + * slots are valid in exactly the way `numberOfLeadsOffSamples` says how many leads-off slots are. + * + * That is why the block is located from the END of the payload rather than from + * `HEADER_LENGTH + n * bytesPerSample`: the region's LENGTH is no more on the wire than the width is, + * and a decoder that assumes the region is exactly full lands the count byte inside the zero fill. + * No region length is hardcoded; the block's own fixed size anchors it. + * + * Candidate ends are tried closest-to-the-end first, so the record claims the least padding it can — + * the puffin pad4 filler is minimal by construction. That ordering is a tie-break, not a proof: an + * all-zero tail can also validate a block placed a byte or two earlier, and the tightest placement is + * the one that explains the most bytes. Two things then have to hold: the region must be long enough + * for the declared samples, and every byte between them and the block must be ZERO. The zero-fill + * check is what keeps the width enumeration honest — a width that under-reads the samples leaves real + * sample bytes in the unused span and is rejected there. */ - fun decodeRaw(payload: List, bytesPerSample: Int): RawLabradorPacket? { - if (bytesPerSample <= 0) return null + fun decodeRaw( + payload: List, + bytesPerSample: Int, + maxPadding: Int = DEFAULT_MAX_PADDING, + ): RawLabradorPacket? { + if (bytesPerSample <= 0 || maxPadding < 0) return null val header = decodeHeader(payload) ?: return null // `bytesPerSample` is caller-supplied and `numberOfECGSamples` comes off the wire, so the product // is checked rather than assumed: a Kotlin Int overflow wraps silently to a NEGATIVE index, which // would throw on the subscript below. A decode failure is the correct outcome, not an exception. val blobLength = header.numberOfECGSamples.toLong() * bytesPerSample.toLong() if (blobLength > Int.MAX_VALUE - HEADER_LENGTH) return null - val rawEnd = HEADER_LENGTH + blobLength.toInt() - if (rawEnd < HEADER_LENGTH || rawEnd >= payload.size) return null // the leads-off count byte must fit - // Same domain check as decodeHeader: a negative count would make qEnd negative, slip past the - // `qEnd > size` bound, and throw on subList. On the wire this byte is always 0..255. - val leadsOffCount = payload[rawEnd] - if (leadsOffCount !in 0..255) return null - val iStart = rawEnd + 1 - val qStart = iStart + leadsOffCount * 2 - val qEnd = qStart + leadsOffCount * 2 - if (qEnd > payload.size) return null - - val leadsOffI = (0 until leadsOffCount).map { payload[iStart + it * 2] or (payload[iStart + it * 2 + 1] shl 8) } - val leadsOffQ = (0 until leadsOffCount).map { payload[qStart + it * 2] or (payload[qStart + it * 2 + 1] shl 8) } - return RawLabradorPacket( - header = header, - rawECGDataRaw = payload.subList(HEADER_LENGTH, rawEnd).toList(), - numberOfLeadsOffSamples = leadsOffCount, - leadsOffIRaw = leadsOffI, - leadsOffQRaw = leadsOffQ, - padding = payload.subList(qEnd, payload.size).toList(), - ) + val sampleEnd = HEADER_LENGTH + blobLength.toInt() + if (sampleEnd < HEADER_LENGTH || sampleEnd > payload.size) return null + + for (pad in 0..maxPadding) { + val blockEnd = payload.size - pad + val countIndex = blockEnd - LEADS_OFF_BLOCK_LENGTH + // The block must sit entirely after the declared samples. This also bounds `countIndex` from + // below, since `sampleEnd >= HEADER_LENGTH >= 0`. + if (countIndex < sampleEnd) continue + // Unused sample capacity is zero-filled on the wire. A non-zero byte here means the block is + // not at this offset — including the case where it holds samples a wider width would read. + if ((sampleEnd until countIndex).any { payload[it] != 0 }) continue + // Same domain check the Swift twin makes: on the wire this byte is 0..255, and the block holds + // only LEADS_OFF_SLOT_COUNT slots — a count above that is not a record this layout can + // describe, so it fails closed rather than reading past the block. + val leadsOffCount = payload[countIndex] + if (leadsOffCount !in 0..LEADS_OFF_SLOT_COUNT) continue + // Both arrays are FIXED-SIZE and always fully present — see [LEADS_OFF_SLOT_COUNT]. The count + // byte selects how many leading slots are VALID; it does not size the block. + val iStart = countIndex + 1 + val qStart = iStart + LEADS_OFF_SLOT_COUNT * 2 + + val leadsOffI = (0 until leadsOffCount).map { payload[iStart + it * 2] or (payload[iStart + it * 2 + 1] shl 8) } + val leadsOffQ = (0 until leadsOffCount).map { payload[qStart + it * 2] or (payload[qStart + it * 2 + 1] shl 8) } + return RawLabradorPacket( + header = header, + rawECGDataRaw = payload.subList(HEADER_LENGTH, sampleEnd).toList(), + unusedSampleBytes = countIndex - sampleEnd, + numberOfLeadsOffSamples = leadsOffCount, + leadsOffIRaw = leadsOffI, + leadsOffQRaw = leadsOffQ, + padding = payload.subList(blockEnd, payload.size).toList(), + ) + } + return null } /** * Every sample width in [widths] that yields a structurally consistent record leaving at most * [maxPadding] trailing bytes. A DISAMBIGUATION helper, not a claim. + * + * A PARTLY-FILLED record is genuinely ambiguous, and that is a fact about the record rather than a gap + * here: 245 samples inside a 1500-byte region fits 3 bytes per sample and 4 alike, so the capture's + * short record reports `[3, 4]` and the single-candidate [decodeRaw] below declines it. The width is a + * property of the STREAM — the 350 full records in the same capture resolve to `[3]` on their own. */ fun rawBytesPerSampleCandidates( payload: List, widths: List = listOf(1, 2, 3, 4), maxPadding: Int = DEFAULT_MAX_PADDING, - ): List = widths.filter { width -> - val packet = decodeRaw(payload, width) - packet != null && packet.padding.size <= maxPadding - } + ): List = widths.filter { width -> decodeRaw(payload, width, maxPadding) != null } /** Decode a raw record only when the buffer admits exactly ONE width; ambiguity returns null. */ fun decodeRaw( @@ -351,12 +458,17 @@ object Whoop5Ecg { maxPadding: Int = DEFAULT_MAX_PADDING, ): RawLabradorPacket? { val candidates = rawBytesPerSampleCandidates(payload, widths, maxPadding) - return if (candidates.size == 1) decodeRaw(payload, candidates[0]) else null + return if (candidates.size == 1) decodeRaw(payload, candidates[0], maxPadding) else null } /** CRC-gated raw decode straight off a complete 5/MG frame, with an explicit sample width. */ - fun decodeRawFrame(frame: ByteArray, bytesPerSample: Int, payloadStart: Int = PUFFIN_PAYLOAD_START): RawLabradorPacket? = - innerPayload(frame, payloadStart)?.let { decodeRaw(it, bytesPerSample) } + fun decodeRawFrame( + frame: ByteArray, + bytesPerSample: Int, + payloadStart: Int = PUFFIN_PAYLOAD_START, + maxPadding: Int = DEFAULT_MAX_PADDING, + ): RawLabradorPacket? = + innerPayload(frame, payloadStart)?.let { decodeRaw(it, bytesPerSample, maxPadding) } // Discovery @@ -380,17 +492,51 @@ object Whoop5Ecg { * armed, every unclassified 5/MG frame is run through this and the hits are logged with their type. It * is a HEURISTIC — four booleans, three enum ranges and a length agreement — not a classifier, and * nothing downstream may treat a hit as proof. + * + * A pass under ANY width in [FILTERED_WIDTH_CANDIDATES] is a pass; callers that need to know WHICH + * width agreed call [filteredBytesPerSampleCandidates] instead of re-deriving it. + */ + fun plausibleFilteredPayload(payload: List, maxPadding: Int = DEFAULT_MAX_PADDING): Boolean = + filteredBytesPerSampleCandidates(payload, maxPadding = maxPadding).isNotEmpty() + + /** + * Every width in [widths] under which [payload] could be a filtered Labrador payload — the widths + * whose length agreement holds once the buffer has passed the field guards. + * + * Empty is the triage's rejection. More than one width means the buffer genuinely does not determine + * the answer; like [rawBytesPerSampleCandidates] this REPORTS the ambiguity rather than picking one. */ - fun plausibleFilteredPayload(payload: List, maxPadding: Int = DEFAULT_MAX_PADDING): Boolean { - val header = decodeHeader(payload) ?: return false - if (header.signalQualityRaw > 3) return false - if (header.heartKeyArrhythmiaCheckResult == null) return false - if (header.heartKeyArrhythmiaCheckStatus == null) return false - if (payload[2] > 1 || payload[3] > 1 || payload[4] > 1 || payload[5] > 1) return false + fun filteredBytesPerSampleCandidates( + payload: List, + widths: List = FILTERED_WIDTH_CANDIDATES, + maxPadding: Int = DEFAULT_MAX_PADDING, + ): List { + val header = decodeHeader(payload) ?: return emptyList() + if (header.signalQualityRaw > 3) return emptyList() + if (header.heartKeyArrhythmiaCheckResult == null) return emptyList() + if (header.heartKeyArrhythmiaCheckStatus == null) return emptyList() + if (payload[2] > 1 || payload[3] > 1 || payload[4] > 1 || payload[5] > 1) return emptyList() val n = header.numberOfECGSamples - if (n <= 0) return false - val end = HEADER_LENGTH + n * 2 - return end <= payload.size && payload.size - end <= maxPadding + if (n <= 0) return emptyList() + return widths.filter { width -> + // `widths` is caller-supplied and `n` comes off the wire, so the offset math is done in Long + // and range-checked: a Kotlin Int overflow wraps silently NEGATIVE and would slip past the + // bounds test below. Same discipline as decodeRaw. + if (width <= 0) return@filter false + val end = HEADER_LENGTH + n.toLong() * width.toLong() + end <= payload.size && payload.size - end <= maxPadding + } + } + + /** The frame-level form of [filteredBytesPerSampleCandidates], CRC-gated. */ + fun filteredBytesPerSampleCandidates( + frame: ByteArray, + payloadStart: Int = PUFFIN_PAYLOAD_START, + widths: List = FILTERED_WIDTH_CANDIDATES, + maxPadding: Int = DEFAULT_MAX_PADDING, + ): List { + val payload = innerPayload(frame, payloadStart) ?: return emptyList() + return filteredBytesPerSampleCandidates(payload, widths, maxPadding) } /** diff --git a/android/app/src/main/java/com/noop/protocol/Whoop5EcgProbe.kt b/android/app/src/main/java/com/noop/protocol/Whoop5EcgProbe.kt index 8fe948f106..3f5ffc29e1 100644 --- a/android/app/src/main/java/com/noop/protocol/Whoop5EcgProbe.kt +++ b/android/app/src/main/java/com/noop/protocol/Whoop5EcgProbe.kt @@ -106,17 +106,184 @@ object Whoop5EcgProbe { * [requestsRealtimeData] deliberately has NO default: every construction site must state it, because * the verdicts that read silence as evidence turn on this flag and a silently-omitted `true` is * exactly the bug this field exists to prevent. + * + * [sentArgument] is the ARGUMENT byte the command was sent with, when it is known. [label] carries + * only the opcode, because it is also the key an app layer matches a COMMAND_RESPONSE against and + * the reply echoes no argument — so the argument cannot live in the label without breaking that + * match. It is recorded separately because two runs of the same opcode can differ only in this byte: + * `mainControlECGDataGeneration` takes `stop`/`start`/`restart` (0/1/2), so a start run and a restart + * run render identically without it and a future reader of a copied report cannot tell which one + * produced the verdict. Reported as the RAW NUMBER, never a token name, so it cannot drift from the + * bytes actually put on the wire. `null` is "not known" — the honest value for an UNSOLICITED reply. + * It is display-only and feeds no verdict, so unlike [requestsRealtimeData] a default is safe: an + * omission loses an annotation, never changes a claim. */ data class Step( val label: String, val outcome: CommandOutcome, val requestsRealtimeData: Boolean, + val sentArgument: Int? = null, val replyHex: String? = null, ) { /** How the report annotates the step, so a reader can see WHY the verdict is what it is. */ val roleNote: String get() = if (requestsRealtimeData) "asks for realtime ECG data" else "cannot produce ECG data (configuration or OFF)" + + /** The [label], plus the argument when it is known — what every report line leads with. */ + val labelWithArgument: String + get() = if (sentArgument == null) label else "$label arg=$sentArgument" + } + + /** + * A bounded census of EVERY unclassified 5/MG frame seen while a probe run is armed — the ones the + * structural triage rejected as well as the ones it accepted. + * + * ## Why the rejects are the point + * + * The packet TYPE byte the Labrador records arrive under is NOT attested, so the probe hunts for it + * with a shape heuristic. Logging only the heuristic's HITS makes that hunt unfalsifiable: when the + * heuristic is wrong, the frame it was wrong about is discarded, the report truthfully says "no frame + * passed the structural triage", and the bytes that would have shown the mistake are gone. That is + * exactly what a hardcoded 2-bytes-per-sample length agreement did to every 3-byte frame before + * [Whoop5Ecg.FILTERED_WIDTH_CANDIDATES] widened it. + * + * It makes no claim about anything it records: it is a census, and every line in it is a frame the + * probe could not classify. + * + * ## Bounds + * + * At most [MAX_SAMPLES_PER_TYPE] recorded frames per distinct type byte and [MAX_TYPES] distinct type + * bytes. Everything past a cap is COUNTED, never silently dropped. + * + * The Kotlin twin of the Swift `Whoop5EcgProbe.FrameCensus`. Android has no ECG app layer to feed it — + * nothing on this platform sends the Labrador commands — so this exists for parity and is covered by + * its own tests, exactly like the rest of this file. + */ + class FrameCensus { + + /** One recorded frame. Nothing here is interpreted — these are measurements of the buffer. */ + data class Sample( + val frameLength: Int, + /** Inner payload length, or null when the frame yielded none (failed a CRC / too short). */ + val payloadLength: Int?, + /** + * `numberOfECGSamples` as the status header would read it, or null when the payload is too + * short to hold a header. Reading it does NOT claim this frame is an ECG record. + */ + val numberOfECGSamples: Int?, + /** + * The bytes-per-sample widths the payload's length agreed with. Empty means the triage + * rejected the frame, which is the case this census exists to preserve. + */ + val widths: List, + /** The frame's first [HEAD_BYTES] bytes, lowercase hex. */ + val headHex: String, + ) { + val line: String + get() { + val payload = payloadLength?.toString() ?: "?" + val samples = numberOfECGSamples?.toString() ?: "?" + val widthText = if (widths.isEmpty()) "none" else widths.joinToString(",") + return "len=$frameLength payload=$payload samples=$samples widths=$widthText head=$headHex" + } + } + + /** Everything seen under one type byte. */ + data class Bucket(val typeByte: Int, var count: Int, val samples: MutableList) + + private val bucketList = mutableListOf() + + /** Buckets in FIRST-SEEN order, which is the order the report prints them in. */ + val buckets: List get() = bucketList + + /** Every frame offered to the census, including those past a cap. */ + var framesSeen = 0 + private set + + /** Frames dropped because their type byte was past the [MAX_TYPES]th distinct one. */ + var framesBeyondTypeCap = 0 + private set + + val isEmpty: Boolean get() = framesSeen == 0 + + /** + * Fold one frame into the census. + * + * The caller CRC-gates the frame first; this reads the type byte directly and gets the payload + * through [Whoop5Ecg.innerPayload], which re-checks both CRCs itself, so a frame that somehow + * arrives unverified yields a null payload length rather than a decoded field. Frames with no type + * byte at all (fewer than 9 bytes) are not records and are ignored. + */ + fun record( + frame: ByteArray, + payloadStart: Int = Whoop5Ecg.PUFFIN_PAYLOAD_START, + maxPadding: Int = Whoop5Ecg.DEFAULT_MAX_PADDING, + ) { + if (frame.size <= 8) return + framesSeen++ + val typeByte = frame[8].toInt() and 0xFF + val bucket = bucketList.firstOrNull { it.typeByte == typeByte } + if (bucket == null) { + if (bucketList.size >= MAX_TYPES) { + framesBeyondTypeCap++ + return + } + bucketList.add( + Bucket(typeByte, 1, mutableListOf(sample(frame, payloadStart, maxPadding))) + ) + return + } + bucket.count++ + if (bucket.samples.size >= MAX_SAMPLES_PER_TYPE) return + bucket.samples.add(sample(frame, payloadStart, maxPadding)) + } + + /** The census as report lines, already indented for [report]. */ + val lines: List + get() { + val out = mutableListOf() + for (bucket in bucketList) { + out.add(" type=0x%02x frames=%d".format(bucket.typeByte, bucket.count)) + for (s in bucket.samples) out.add(" ${s.line}") + if (bucket.count > bucket.samples.size) { + out.add(" (+${bucket.count - bucket.samples.size} more of this type, not recorded)") + } + } + if (framesBeyondTypeCap > 0) { + out.add( + " ($framesBeyondTypeCap frame(s) of further type bytes past the " + + "$MAX_TYPES-type cap, counted only)" + ) + } + return out + } + + companion object { + /** Recorded frames per distinct type byte. Three examples answer "what does this look like". */ + const val MAX_SAMPLES_PER_TYPE = 3 + + /** Distinct type bytes tracked. Well past what a live link shows, and keeps the report readable. */ + const val MAX_TYPES = 16 + + /** Leading bytes of each recorded frame rendered as hex: the envelope, the header, first samples. */ + const val HEAD_BYTES = 64 + + /** Measure one frame. Public so a test can build a [Sample] without a census. */ + fun sample( + frame: ByteArray, + payloadStart: Int = Whoop5Ecg.PUFFIN_PAYLOAD_START, + maxPadding: Int = Whoop5Ecg.DEFAULT_MAX_PADDING, + ): Sample { + val payload = Whoop5Ecg.innerPayload(frame, payloadStart) + val widths = payload?.let { + Whoop5Ecg.filteredBytesPerSampleCandidates(it, maxPadding = maxPadding) + } ?: emptyList() + val samples = payload?.let { Whoop5Ecg.decodeHeader(it)?.numberOfECGSamples } + val head = frame.take(HEAD_BYTES).joinToString("") { "%02x".format(it.toInt() and 0xFF) } + return Sample(frame.size, payload?.size, samples, widths, head) + } + } } /** The gate verdict, kept separate from the report text so it is assertable in a test. */ @@ -221,11 +388,11 @@ object Whoop5EcgProbe { val failures = steps.filter { it.outcome is CommandOutcome.Failure } if (failures.isNotEmpty()) { // A refusal is only evidence about the BLOCK question when what was refused asked for data. - val dataFailures = failures.filter { it.requestsRealtimeData }.map { it.label } + val dataFailures = failures.filter { it.requestsRealtimeData }.map { it.labelWithArgument } if (dataFailures.isNotEmpty()) return Verdict.DataRequestRefused(dataFailures) - return Verdict.CommandRefused(failures.map { it.label }) + return Verdict.CommandRefused(failures.map { it.labelWithArgument }) } - val unsupported = steps.filter { it.outcome is CommandOutcome.Unsupported }.map { it.label } + val unsupported = steps.filter { it.outcome is CommandOutcome.Unsupported }.map { it.labelWithArgument } if (unsupported.isNotEmpty()) return Verdict.OpcodeUnsupported(unsupported) if (ecgPacketsSeen > 0) return Verdict.EcgCandidatesArrived(ecgPacketsSeen) if (steps.isEmpty()) return Verdict.NoReplies @@ -233,9 +400,9 @@ object Whoop5EcgProbe { // Past this point the verdict INTERPRETS SILENCE, which only carries information about the ECG // data path when the run asked that path for something and the strap said yes. val requests = steps.filter { it.requestsRealtimeData } - if (requests.isEmpty()) return Verdict.NoDataRequested(steps.map { it.label }) + if (requests.isEmpty()) return Verdict.NoDataRequested(steps.map { it.labelWithArgument }) if (requests.none { it.outcome is CommandOutcome.Success }) { - return Verdict.DataRequestNotAccepted(requests.map { it.label }) + return Verdict.DataRequestNotAccepted(requests.map { it.labelWithArgument }) } if (steps.all { it.outcome is CommandOutcome.Success }) { return Verdict.AcceptedButSilent(windowSeconds) @@ -248,12 +415,17 @@ object Whoop5EcgProbe { * * [candidateFrames] are the type/length lines for frames that passed the structural triage in * [Whoop5Ecg.plausibleFilteredPayload]. + * + * [census] is the same question asked WITHOUT the heuristic: every unclassified frame the run saw, + * bucketed by type byte, hits and misses alike. It defaults to empty so a caller that has none still + * renders — see [FrameCensus] for why the misses matter more than the hits. */ fun report( steps: List, ecgPacketsSeen: Int, candidateFrames: List, windowSeconds: Int, + census: FrameCensus = FrameCensus(), ): String { val sb = StringBuilder() sb.append("WHOOP MG ECG (Labrador) TURN-ON PROBE\n") @@ -264,7 +436,7 @@ object Whoop5EcgProbe { } else { // Each step carries WHY it does or does not bear on the block question, so the verdict above // can be checked against its own inputs without reading the source. - for (step in steps) sb.append(" ${step.label}: ${step.outcome.token} — ${step.roleNote}\n") + for (step in steps) sb.append(" ${step.labelWithArgument}: ${step.outcome.token} — ${step.roleNote}\n") } sb.append("\nECG-shaped packets seen in ${windowSeconds}s: $ecgPacketsSeen\n") if (steps.isNotEmpty() && steps.none { it.requestsRealtimeData }) { @@ -294,7 +466,27 @@ object Whoop5EcgProbe { sb.append("Candidate packet types (structural triage only, NOT a confirmed mapping):\n") for (line in candidateFrames) sb.append(" $line\n") } - val replies = steps.mapNotNull { step -> step.replyHex?.let { " ${step.label}: $it" } } + // The census goes IMMEDIATELY under the triage result, because it is the check on it: "no frame + // passed the structural triage" is only informative next to what the frames actually were. + // Twin of the Swift `Whoop5EcgProbe.report` block; the wording is byte-identical. + if (census.isEmpty) { + sb.append( + "Unclassified-frame census: no unclassified frame arrived at all — the triage had " + + "nothing to reject.\n" + ) + } else { + sb.append( + "Unclassified-frame census — EVERY frame this run could not classify, triage hits and " + + "misses alike, bucketed by the inner record's type byte. The ECG type byte is not " + + "attested, so this, not the triage, is the record of what arrived. widths= are the " + + "bytes-per-sample the payload length agreed with; none = the triage rejected it. " + + "Recorded: ${census.framesSeen} frame(s), first ${FrameCensus.MAX_SAMPLES_PER_TYPE} per " + + "type byte, first ${FrameCensus.MAX_TYPES} type bytes, first ${FrameCensus.HEAD_BYTES} " + + "bytes each:\n" + ) + for (line in census.lines) sb.append("$line\n") + } + val replies = steps.mapNotNull { step -> step.replyHex?.let { " ${step.labelWithArgument}: $it" } } if (replies.isNotEmpty()) { sb.append("\nRaw replies:\n") sb.append(replies.joinToString("\n") + "\n") diff --git a/android/app/src/main/java/com/noop/ui/AppViewModel.kt b/android/app/src/main/java/com/noop/ui/AppViewModel.kt index 12954c02f4..a4006e0c17 100644 --- a/android/app/src/main/java/com/noop/ui/AppViewModel.kt +++ b/android/app/src/main/java/com/noop/ui/AppViewModel.kt @@ -943,6 +943,8 @@ class AppViewModel(app: Application) : AndroidViewModel(app) { useExperimentalSleepV2 = PuffinExperiment.from(appContext).experimentalSleepV2, // Opt-in motion-aware wake refinement (#364 follow-up) — same Context-free threading. useMotionAwareWake = PuffinExperiment.from(appContext).motionAwareWake, + // Wake-gap bridging (experimental, default OFF) — same Context-free threading. + useWakeBridge = PuffinExperiment.from(appContext).sleepWakeBridge, // Sleep & Rest test mode (Test Centre E5): when the SLEEP domain is on, route the // per-day sleep gate trace into the SAME shareable strap log, tagged .sleep so it // lands under the profile in the export. Zero-cost when off: the gate is one @@ -1562,6 +1564,8 @@ class AppViewModel(app: Application) : AndroidViewModel(app) { useExperimentalSleepV2 = PuffinExperiment.from(appContext).experimentalSleepV2, // Opt-in motion-aware wake refinement (#364 follow-up) — same flag the 15-min loop reads. useMotionAwareWake = PuffinExperiment.from(appContext).motionAwareWake, + // Wake-gap bridging (experimental, default OFF) — same flag the 15-min loop reads. + useWakeBridge = PuffinExperiment.from(appContext).sleepWakeBridge, ) }.onFailure { if (it is kotlin.coroutines.cancellation.CancellationException) throw it } } diff --git a/android/app/src/main/java/com/noop/ui/SettingsScreen.kt b/android/app/src/main/java/com/noop/ui/SettingsScreen.kt index 469a2f735b..d05cc578cc 100644 --- a/android/app/src/main/java/com/noop/ui/SettingsScreen.kt +++ b/android/app/src/main/java/com/noop/ui/SettingsScreen.kt @@ -502,6 +502,9 @@ fun SettingsScreen( // "Motion-aware wake refinement" (#364 follow-up) — OFF by default. Self-gates on observed gravity + // step density, so it is a no-op on a sparse (e.g. WHOOP 4.0) night regardless of this switch. var motionAwareWake by remember { mutableStateOf(puffinExperiment.motionAwareWake) } + // "Bridge brief wakes" — OFF by default. Bridges short (≤45 min) wake gaps into one detected + // session with the wake staged inside; see PuffinExperiment.sleepWakeBridge. + var sleepWakeBridge by remember { mutableStateOf(puffinExperiment.sleepWakeBridge) } // Whether to surface the WHOOP 5/MG-only probes (puffin / R22 / broadcast-HR / frame-capture). Gated // so a confident 4.0 owner never sees 5/MG controls that can't touch their strap (#22). The model @@ -2204,6 +2207,47 @@ fun SettingsScreen( color = Palette.textTertiary, ) + // --- Bridge brief wakes — OFF by default. --- + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(16.dp), + ) { + Text( + "Bridge brief wakes", + style = NoopType.subhead, + color = Palette.textPrimary, + modifier = Modifier.weight(1f), + ) + Switch( + checked = sleepWakeBridge, + onCheckedChange = { + sleepWakeBridge = it + puffinExperiment.sleepWakeBridge = it + }, + colors = SwitchDefaults.colors( + checkedThumbColor = Palette.surfaceBase, + checkedTrackColor = Palette.accent, + uncheckedThumbColor = Palette.textSecondary, + uncheckedTrackColor = Palette.surfaceInset, + uncheckedBorderColor = Palette.hairline, + ), + modifier = Modifier.semantics { + contentDescription = "Bridge brief wakes" + }, + ) + } + Text( + "Keeps a night interrupted by a brief get-up (under ~45 minutes) as one sleep, " + + "with the wake scored inside it, instead of splitting it into separate sleeps " + + "-- and short sleep stretches between such wakes are no longer discarded for " + + "being under an hour on their own. Worth trying if your nights show up split " + + "into two or three pieces around bathroom trips or brief stirs. Off by " + + "default; takes effect on the next nights scored.", + style = NoopType.caption, + color = Palette.textTertiary, + ) + // Diagnostics: dump the decoded per-sample sensor streams (last 24h) to one long-format // CSV so power users / external devs can prototype sleep/activity/VBT algorithms on real // data without a BLE stream (#308/#276/#322). On-device only; plain text, no BLE hex. diff --git a/android/app/src/test/java/com/noop/analytics/RegistryDayOwnerSourceTest.kt b/android/app/src/test/java/com/noop/analytics/RegistryDayOwnerSourceTest.kt index 661dff1727..76f07a7f8d 100644 --- a/android/app/src/test/java/com/noop/analytics/RegistryDayOwnerSourceTest.kt +++ b/android/app/src/test/java/com/noop/analytics/RegistryDayOwnerSourceTest.kt @@ -62,6 +62,7 @@ class RegistryDayOwnerSourceTest { override suspend fun deletePpgWaveformFor(deviceId: String) {} override suspend fun deleteRawImuFor(deviceId: String) {} override suspend fun deleteV18AuxFor(deviceId: String) {} + override suspend fun deleteSpo2PctFor(deviceId: String) {} override suspend fun deleteEventsFor(deviceId: String) {} override suspend fun deleteBatteryFor(deviceId: String) {} override suspend fun deleteDailyMetricsFor(deviceId: String) {} @@ -92,6 +93,7 @@ class RegistryDayOwnerSourceTest { override suspend fun reKeyPpgWaveform(from: String, to: String) {} override suspend fun reKeyRawImu(from: String, to: String) {} override suspend fun reKeyV18Aux(from: String, to: String) {} + override suspend fun reKeySpo2Pct(from: String, to: String) {} override suspend fun reKeyEvents(from: String, to: String) {} override suspend fun reKeyBattery(from: String, to: String) {} override suspend fun reKeyDailyMetrics(from: String, to: String) {} diff --git a/android/app/src/test/java/com/noop/analytics/SleepStagerWakeBridgeTest.kt b/android/app/src/test/java/com/noop/analytics/SleepStagerWakeBridgeTest.kt new file mode 100644 index 0000000000..76d5c29d73 --- /dev/null +++ b/android/app/src/test/java/com/noop/analytics/SleepStagerWakeBridgeTest.kt @@ -0,0 +1,202 @@ +package com.noop.analytics + +import com.noop.data.GravitySample +import com.noop.data.HrSample +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.math.abs + +/** + * Wake-gap bridging (experimental, default OFF): assemble adjacent sleep runs across ≤ 45-min wake + * gaps into ONE candidate run, so a brief get-up stays inside one session as staged wake. + * + * Includes a REAL-DATA pin: the exact seam layout of a reporting wearer's stored fragmented nights + * (Aug 2026 mirror), asserting precisely which seams the bridge merges and which it leaves split. + * An HR-led rescue of restless-but-asleep nights was prototyped alongside this bridge and WITHDRAWN + * after a real-data replay — see the Swift history note above `SleepStager.wakeBridgeMaxMin`; the + * restless-night test below pins that the bridge deliberately does NOT fix that failure. + * Faithful Kotlin mirror of SleepStagerWakeBridgeTests.swift: same scenarios, same numbers. + */ +class SleepStagerWakeBridgeTest { + + private val dev = "test" + + /** 2026-06-10 00:00:00 UTC — same fixed midnight the Swift twin uses. */ + private val refMidnight = 1_749_513_600L + + private fun at(hourUTC: Int, min: Int = 0): Long = refMidnight + hourUTC * 3_600L + min * 60L + + private fun stillGravity(start: Long, durationS: Int): List = + (0 until durationS).map { GravitySample(deviceId = dev, ts = start + it, x = 0.0, y = 0.0, z = 1.0) } + + /** Oscillating orientation (0.5 g jumps per sample) — clearly "moving" to the stillness spine. */ + private fun restlessGravity(start: Long, durationS: Int): List = + (0 until durationS).map { i -> + val phase = (i % 2) * 0.5 + GravitySample(deviceId = dev, ts = start + i, x = phase, y = 0.0, z = 1.0) + } + + private fun hrStream(start: Long, durationS: Int, bpm: Int): List = + (0 until durationS).map { HrSample(deviceId = dev, ts = start + it, bpm = bpm) } + + // ── Wake-gap bridging (the SPLIT-night failure) ────────────────────────── + + @Test + fun briefGetUpBridgedIntoOneSession() { + // A 16-min get-up (real movement + HR 80) mid-night. The centered rolling window smears it + // past mergeMin, so OFF stores TWO sessions; ON bridges them into ONE. + val grav = stillGravity(at(23), 4 * 3_600) + // 23:00–03:00 + restlessGravity(at(27), 16 * 60) + // 03:00–03:16 get-up + stillGravity(at(27, 16), 3 * 3_600 + 44 * 60) // 03:16–07:00 + val hr = hrStream(at(21), 2 * 3_600, 75) + + hrStream(at(23), 4 * 3_600, 52) + + hrStream(at(27), 16 * 60, 80) + + hrStream(at(27, 16), 3 * 3_600 + 44 * 60, 52) + + hrStream(at(31), 2 * 3_600, 75) + + assertEquals("OFF: the smeared get-up splits the night", + 2, SleepStager.detectSleep(hr = hr, gravity = grav).size) + + val on = SleepStager.detectSleep(hr = hr, gravity = grav, wakeBridge = true) + assertEquals("ON: one session bridging the get-up", 1, on.size) + assertTrue(abs(on[0].start - at(23)) <= 120) + assertTrue(abs(on[0].end - at(31)) <= 120) + } + + @Test + fun longWakeGapStaysSplit() { + // An 80-min genuinely-awake block (elevated HR, real movement) exceeds the 45-min bridge. + val grav = stillGravity(at(22), 3 * 3_600) + // 22:00–01:00 + restlessGravity(at(25), 80 * 60) + // 01:00–02:20 awake + stillGravity(at(26, 20), 3 * 3_600 + 40 * 60) // 02:20–06:00 + val hr = hrStream(at(20), 2 * 3_600, 75) + + hrStream(at(22), 3 * 3_600, 52) + + hrStream(at(25), 80 * 60, 78) + + hrStream(at(26, 20), 3 * 3_600 + 40 * 60, 52) + + hrStream(at(30), 2 * 3_600, 75) + + assertEquals("an over-threshold wake gap must not be bridged", + 2, SleepStager.detectSleep(hr = hr, gravity = grav, wakeBridge = true).size) + } + + @Test + fun restlessNightIsNotRescuedByBridge() { + // The withdrawn-rescue pin: a restless night (motion "active", HR at clear sleep levels) is + // NOT rescued by the bridge — ON must change nothing versus OFF. The bridge only assembles + // runs the unchanged stillness spine already accepted. + val grav = restlessGravity(at(22), 6 * 3_600 + 1_800) + // 22:00–04:30 + stillGravity(at(28, 30), 90 * 60) // 04:30–06:00 + val hr = hrStream(at(20), 2 * 3_600, 75) + + hrStream(at(22), 8 * 3_600, 52) + + hrStream(at(30), 2 * 3_600, 75) + + val off = SleepStager.detectSleep(hr = hr, gravity = grav) + val on = SleepStager.detectSleep(hr = hr, gravity = grav, wakeBridge = true) + assertEquals("the bridge must not reclassify restless stretches", off, on) + assertEquals("only the still morning fragment scores either way", 1, on.size) + assertTrue(on[0].start >= at(28, 25)) + } + + // ── Guards still hold on assembled runs ────────────────────────────────── + + @Test + fun daytimeGuardStillAppliesToBridgedRuns() { + // Two still DAYTIME stretches bridged across a 30-min gap (flat in-band HR, no cardiac dip) + // must still be dropped by the daytime false-sleep guard on the ASSEMBLED window. + val grav = stillGravity(at(12), 2 * 3_600) + // 12:00–14:00 + stillGravity(at(14, 30), 2 * 3_600) // 14:30–16:30 + val hr = hrStream(at(10), 8 * 3_600, 60) // flat, no dip + + assertEquals("no resting-HR dip: the daytime guard must reject the bridged window", + 0, SleepStager.detectSleep(hr = hr, gravity = grav, wakeBridge = true).size) + } + + @Test + fun defaultOffIsByteIdentical() { + val grav = stillGravity(at(23), 4 * 3_600) + + restlessGravity(at(27), 16 * 60) + + stillGravity(at(27, 16), 3 * 3_600 + 44 * 60) + val hr = hrStream(at(23), 8 * 3_600, 52) + hrStream(at(31), 4 * 3_600, 75) + + val implicitDefault = SleepStager.detectSleep(hr = hr, gravity = grav) + val explicitFalse = SleepStager.detectSleep(hr = hr, gravity = grav, wakeBridge = false) + assertEquals(implicitDefault, explicitFalse) + } + + // ── Pure helper ────────────────────────────────────────────────────────── + + @Test + fun bridgeWakeGapsMergesOnlyUnderThreshold() { + val a = SleepStager.Period(stage = "sleep", start = 0L, end = 3_600L) + val gap = SleepStager.Period(stage = "active", start = 3_600L, end = 3_600L + 40 * 60) + val b = SleepStager.Period(stage = "sleep", start = 3_600L + 40 * 60, end = 3 * 3_600L) + val merged = SleepStager.bridgeWakeGaps(listOf(a, gap, b)) + assertEquals(1, merged.size) + assertEquals(0L, merged[0].start) + assertEquals(3 * 3_600L, merged[0].end) + + val farGap = SleepStager.Period(stage = "active", start = 3_600L, end = 3_600L + 46 * 60) + val c = SleepStager.Period(stage = "sleep", start = 3_600L + 46 * 60, end = 4 * 3_600L) + assertEquals("a 46-min gap is over wakeBridgeMaxMin (45) and must not merge", + 3, SleepStager.bridgeWakeGaps(listOf(a, farGap, c)).size) + } + + @Test + fun bridgeRefusesOverSpanCapMerge() { + // H4 span guard: a merge whose ASSEMBLED span would exceed maxMainSleepSpanS (16 h) is + // refused, so a pathological chain can never assemble and then be dropped whole by the + // ladder's cap — the exact failure the withdrawn HR rescue exhibited on real data. + val a = SleepStager.Period(stage = "sleep", start = 0L, end = 8 * 3_600L) + val b = SleepStager.Period(stage = "sleep", start = 8 * 3_600L + 30 * 60, end = 17 * 3_600L) + assertEquals("a merge assembling a >16 h span must be refused", + 2, SleepStager.bridgeWakeGaps(listOf(a, b)).size) + } + + // ── Real-data seam pin (Aug 2026 mirror, reporting wearer's stored fragments) ── + + @Test + fun realNightSeamsBridgeExactly() { + // The stored fragment layouts of five REAL split nights, as (start, end) unix pairs from + // the wearer's server mirror. Asserts exactly which seams the 45-min bridge merges + // (16/12/28/25+19-min arousal seams) and which it leaves split (2.2–3.9 h genuine wake + // gaps, and a 90-min morning re-sleep gap — over the threshold, left to the selector's + // #861 night-tail bridge). Same pairs as the Swift twin. + fun sleeps(spans: List>): List = + spans.map { SleepStager.Period(stage = "sleep", start = it.first, end = it.second) } + + // Aug 11→12 ET: 4 fragments; only the 16-min bathroom seam (06:17→06:33 ET) merges → 3 blocks. + val aug12 = sleeps(listOf(1786491049L to 1786496628L, 1786510824L to 1786514588L, + 1786522512L to 1786529853L, 1786530794L to 1786541952L)) + assertEquals(listOf(1786491049L to 1786496628L, 1786510824L to 1786514588L, + 1786522512L to 1786541952L), + SleepStager.bridgeWakeGaps(aug12).map { it.start to it.end }) + + // Aug 17→18 ET: 3 fragments; the 12-min seam merges, the 3.8 h gap stays → 2 blocks. + val aug18 = sleeps(listOf(1787032890L to 1787037056L, 1787050829L to 1787054649L, + 1787055394L to 1787063445L)) + assertEquals(listOf(1787032890L to 1787037056L, 1787050829L to 1787063445L), + SleepStager.bridgeWakeGaps(aug18).map { it.start to it.end }) + + // Aug 18→19 ET: 2 fragments at a 28-min seam → 1 block. + val aug19 = sleeps(listOf(1787126248L to 1787132602L, 1787134284L to 1787141425L)) + assertEquals(listOf(1787126248L to 1787141425L), + SleepStager.bridgeWakeGaps(aug19).map { it.start to it.end }) + + // Aug 20→21 ET: 3 fragments at 25-min and 19-min seams → 1 block (11.4 h < the 16 h cap). + // NOTE: the mirror's correction pass (an HR-evidence-led edit session, not a human label) + // kept the third block (a post-wake morning doze) as a SEPARATE doze; the 45-min rule cannot + // distinguish "brief wake, more sleep" from "woke, dozed again", so the bridge deliberately + // folds it in (wake staged inside). Pinned as-is. + val aug21 = sleeps(listOf(1787287220L to 1787297404L, 1787298936L to 1787322337L, + 1787323440L to 1787328166L)) + assertEquals(listOf(1787287220L to 1787328166L), + SleepStager.bridgeWakeGaps(aug21).map { it.start to it.end }) + + // Aug 23→24 ET: 2 fragments at a 90-min gap → stays split (over the 45-min bridge). The + // mirror's correction pass filled this gap as SLEEP on HR evidence — it is the + // withdrawn-rescue (restless dropping) class, not a seam; the bridge must not guess it. + val aug24 = sleeps(listOf(1787546768L to 1787551261L, 1787556701L to 1787575851L)) + assertEquals(2, SleepStager.bridgeWakeGaps(aug24).size) + } +} diff --git a/android/app/src/test/java/com/noop/analytics/Spo2PctNightlyTest.kt b/android/app/src/test/java/com/noop/analytics/Spo2PctNightlyTest.kt new file mode 100644 index 0000000000..dec9000cec --- /dev/null +++ b/android/app/src/test/java/com/noop/analytics/Spo2PctNightlyTest.kt @@ -0,0 +1,203 @@ +package com.noop.analytics + +import com.noop.data.Spo2PctRow +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Test + +/** + * v34 / MIGRATION_25_26 — the nightly SpO2 percentage banked on `DailyMetric.spo2Pct` for a 5/MG night. + * + * The strap samples SpO2 in RUNS: ~30 one-second samples roughly once per ~1,200 s while asleep. The + * first samples of each run are the optical front end settling and they read LOW. That makes a naive mean + * over the night's in-band samples biased downward — and biased *more* on nights made of many short runs, + * which is the opposite of when you would think to check. These pin the two corrections (per-run ramp + * trim, then median) and, more usefully, pin the size of the bias they remove. + * + * Byte-parity twin of the Swift `Spo2PctNightlyTests`: same fixtures, same expected values. + */ +class Spo2PctNightlyTest { + + private fun session(start: Long, durSec: Long) = DetectedSleep( + start = start, end = start + durSec, efficiency = 0.9, + stages = emptyList(), restingHR = 50, avgHRV = 60.0, + ) + + /** + * A measurement run: [count] consecutive one-second samples starting at [ts], the first [rampCount] + * of them at the depressed [rampValue]. + */ + private fun run( + ts: Long, + count: Int, + settled: Int, + rampCount: Int = 0, + rampValue: Int = 88, + ): List = (0 until count).map { Spo2PctRow(ts + it, if (it < rampCount) rampValue else settled) } + + // MARK: - The bias this exists to remove + + /** + * THE MOTIVATING CASE. One 30-sample run whose first 5 are the acquisition ramp at 88 and whose + * settled 25 are 97. A naive mean lands at 95.5 — a full 1.5 points below what the strap actually + * settled on, entirely from a one-sided artifact. The ramp trim removes exactly those 5 samples and + * the median returns the settled value. + */ + @Test fun theAcquisitionRampBiasesANaiveMeanAndTheTrimRemovesIt() { + val samples = run(1_000, count = 30, settled = 97, rampCount = 5) + val naiveMean = samples.sumOf { it.pct }.toDouble() / samples.size + assertEquals("the bias is real and this is its size", 95.5, naiveMean, 0.001) + + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples) + assertEquals(97.0, r!!.first, 0.001) + assertEquals("the five ramp samples are dropped, the settled 25 are kept", 25, r.second) + } + + // MARK: - Run segmentation + + /** + * Runs are separated by a gap far larger than the 1 s spacing inside one, so EACH run gets its own + * ramp trim. Trimming only the night's first five samples would leave every later run's ramp in. + */ + @Test fun eachRunIsTrimmedSeparately() { + val samples = run(1_000, count = 20, settled = 97, rampCount = 5) + + run(3_000, count = 20, settled = 97, rampCount = 5) + + run(5_000, count = 20, settled = 97, rampCount = 5) + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 5_000)), samples) + assertEquals("3 runs x (20 - 5) survivors", 45, r!!.second) + assertEquals(97.0, r.first, 0.001) + } + + /** + * A dropped second or two inside a run must NOT split it — the gap threshold sits in the gulf between + * 1 s (within a run) and ~1,200 s (between runs), so a small hole is tolerated. + */ + @Test fun aSmallHoleDoesNotSplitARun() { + // 10 samples, then a 20 s hole, then 10 more: still ONE run, so ONE ramp trim. + val samples = run(1_000, count = 10, settled = 97, rampCount = 5) + run(1_030, count = 10, settled = 97) + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples) + assertEquals("one run: 20 samples less one 5-sample ramp", 15, r!!.second) + } + + @Test fun theRunGapThresholdIsTheDocumentedOne() { + assertEquals(60L, AnalyticsEngine.SPO2_RUN_GAP_SECONDS) + assertEquals(5, AnalyticsEngine.SPO2_RAMP_SAMPLES) + } + + // MARK: - The half-run cap + + /** + * The trim never eats more than half a run. A short run must contribute its settled tail rather than + * vanishing — otherwise the night would be silently reweighted toward whichever runs happened to be + * long, which is a selection effect nobody would see in the output. + */ + @Test fun aShortRunKeepsItsTailInsteadOfVanishing() { + // 4 samples: drop min(5, 2) = 2, keep 2. + val r = AnalyticsEngine.nightlySpo2Pct( + listOf(session(900, 600)), + listOf(Spo2PctRow(1_000, 88), Spo2PctRow(1_001, 90), Spo2PctRow(1_002, 96), Spo2PctRow(1_003, 96)), + ) + assertEquals(2, r!!.second) + assertEquals(96.0, r.first, 0.001) + } + + /** A single-sample run keeps its one sample (drop = min(5, 0) = 0) rather than being erased. */ + @Test fun aSingleSampleRunSurvives() { + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), listOf(Spo2PctRow(1_000, 95))) + assertEquals(1, r!!.second) + assertEquals(95.0, r.first, 0.001) + } + + // MARK: - The statistic + + /** + * The median is the statistic, matching what the cloud reader reports for the same rows — a phone that + * disagreed with the server on identical data would poison every future cross-check. + */ + @Test fun medianNotMeanOverTheSurvivors() { + // Survivors after the ramp trim: 90, 97, 97, 97, 97 — median 97, mean 95.6. + val pcts = listOf(90, 90, 90, 90, 90, 90, 97, 97, 97, 97) + val samples = pcts.mapIndexed { i, v -> Spo2PctRow(1_000L + i, v) } + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples) + assertEquals(5, r!!.second) + assertEquals("a single low survivor must not drag the answer the way a mean would", + 97.0, r.first, 0.001) + } + + /** An even survivor count averages the two middle values, so the result can land on a half. */ + @Test fun evenSurvivorCountAveragesTheTwoMiddleValues() { + // A 4-sample run: the half-run cap drops 2, leaving survivors 94 and 97 -> (94 + 97) / 2 = 95.5. + val samples = listOf(88, 88, 94, 97).mapIndexed { i, v -> Spo2PctRow(1_000L + i, v) } + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples) + assertNotNull(r) + assertEquals(2, r!!.second) + assertEquals(95.5, r.first, 0.001) + } + + /** + * The half-run cap governs whenever a run is shorter than twice the ramp, so the survivor count is + * `count - min(5, count/2)` and not simply `count - 5`. Pinned across the crossover at 10. + */ + @Test fun survivorCountFollowsTheCappedTrim() { + for ((count, want) in listOf(1 to 1, 2 to 1, 4 to 2, 6 to 3, 9 to 5, 10 to 5, 11 to 6, 30 to 25)) { + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), run(1_000, count, settled = 97)) + assertEquals("a $count-sample run must keep $want", want, r!!.second) + } + } + + // MARK: - Session gating and absence + + /** Only in-bed samples count — a daytime reading is not part of the night's saturation. */ + @Test fun samplesOutsideEverySessionAreExcluded() { + val samples = run(1_000, count = 10, settled = 97) + run(9_000, count = 10, settled = 80) + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples) + assertEquals("only the in-bed run contributes", 5, r!!.second) + assertEquals(97.0, r.first, 0.001) + } + + /** + * Gap size is the ONLY run signal — a session edge is not a second one. A brief out-of-bed moment + * mid-run leaves the run intact and singly-trimmed, because the acquisition ramp is a property of the + * SENSOR restarting and the sensor does not restart because the sleep detector drew a line. + */ + @Test fun aBriefSessionEdgeDoesNotSplitARun() { + val samples = run(1_000, count = 20, settled = 97) + // Sessions carve out [1000..1004] and [1010..1019] — 15 in-bed samples with only a 6 s hole, + // which is under the threshold, so this stays ONE run and is trimmed ONCE. + val r = AnalyticsEngine.nightlySpo2Pct(listOf(session(1_000, 4), session(1_010, 9)), samples) + assertEquals("one run of 15, less one 5-sample ramp", 10, r!!.second) + } + + /** + * When the excluded stretch IS longer than the gap threshold, the two halves are genuinely separate + * runs and each pays its own ramp trim. + */ + @Test fun aWideSessionGapDoesSplitARun() { + val samples = run(1_000, count = 10, settled = 97) + run(1_200, count = 10, settled = 97) + // 200 s apart — comfortably over the 60 s threshold, so two runs: (10-5) + (10-5). + assertEquals(10, AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), samples)!!.second) + } + + /** Absence stays absence — nothing here may invent a percentage. */ + @Test fun nullWhenThereIsNothingToAverage() { + assertNull(AnalyticsEngine.nightlySpo2Pct(emptyList(), listOf(Spo2PctRow(1_000, 97)))) + assertNull(AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), emptyList())) + // In-band samples exist, but none of them fall in a session. + assertNull(AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 100)), listOf(Spo2PctRow(9_999, 97)))) + } + + /** + * Unsorted input must not change the answer — run boundaries are a property of the timestamps, not of + * the order the store happened to hand them over in. + */ + @Test fun inputOrderDoesNotMatter() { + val ordered = run(1_000, count = 20, settled = 97, rampCount = 5) + val shuffled = ordered.reversed() + assertEquals( + AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), ordered)!!.second, + AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), shuffled)!!.second, + ) + assertEquals(97.0, AnalyticsEngine.nightlySpo2Pct(listOf(session(900, 600)), shuffled)!!.first, 0.001) + } +} diff --git a/android/app/src/test/java/com/noop/ble/BackfillContinuationTest.kt b/android/app/src/test/java/com/noop/ble/BackfillContinuationTest.kt index 6cd2574274..958b9edb2a 100644 --- a/android/app/src/test/java/com/noop/ble/BackfillContinuationTest.kt +++ b/android/app/src/test/java/com/noop/ble/BackfillContinuationTest.kt @@ -295,6 +295,15 @@ class BackfillContinuationTest { passes += 1 assertTrue("auto-continue must be bounded", passes <= WhoopBleClient.MAX_AUTO_CONTINUES + 1) } + // It stopped because it CAUGHT UP — the strap is no longer ahead of our frontier — not because a cap + // cut it off mid-recovery, and not because it silently stalled at pass 1. + // + // This used to assert `count == MAX_AUTO_CONTINUES`, which passed only because the cap was 6 and + // catching up needs 7 passes: the drain was ALWAYS severed one day short of done, and the test + // enshrined that as correct. The cap is a runaway backstop, not a drain schedule — a strap that is + // genuinely behind and genuinely handing over data must be allowed to finish. The cap's bounding job + // is pinned by the HISTORY_COMPLETE-slice test, which uses a strap that never catches up; that is the + // case the cap exists for. Mirrors Swift testMultiPassDrainUntilCaughtUpOrCapped. assertEquals(WhoopBleClient.MAX_AUTO_CONTINUES, count) } diff --git a/android/app/src/test/java/com/noop/ble/RawHistoryArchiveEvictionTest.kt b/android/app/src/test/java/com/noop/ble/RawHistoryArchiveEvictionTest.kt index 4afdc46daa..a54988f13a 100644 --- a/android/app/src/test/java/com/noop/ble/RawHistoryArchiveEvictionTest.kt +++ b/android/app/src/test/java/com/noop/ble/RawHistoryArchiveEvictionTest.kt @@ -1,6 +1,7 @@ package com.noop.ble import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -53,6 +54,119 @@ class RawHistoryArchiveEvictionTest { assertEquals(lines, RawHistoryArchive.evictLines(lines, maxBytes = 1_000_000, floor = 2)) } + // MARK: - zero-payload frames lose retention priority + + /** + * A synthetic WHOOP 5/MG type-47 record: a non-zero 21-byte header (type @8 = 47, hist_version @9), + * then [payloadBytes] of payload, then a 4-byte CRC trailer. + * + * [payloadByte] = 0 reproduces the measured `enable_raw_data_w_ecg` placeholder — the strap banks one + * such record per second and every byte from offset 21 to the CRC is zero. [marker] writes a single + * distinguishing byte into the payload, which is also what makes a frame informative. + */ + private fun whoop5Frame( + version: Int, + payloadByte: Int, + payloadBytes: Int = 64, + marker: Int? = null, + ): ByteArray { + val f = ByteArray(21) { 0x11 } // non-zero header (seq/ts/const bytes) + f[0] = 0xAA.toByte(); f[8] = 47; f[9] = version.toByte() + val payload = ByteArray(payloadBytes) { payloadByte.toByte() } + if (marker != null) payload[0] = marker.toByte() + return f + payload + byteArrayOf(0xDE.toByte(), 0xAD.toByte(), 0xBE.toByte(), 0xEF.toByte()) + } + + /** One archived JSONL line for [frame], in the exact shape `append` writes. */ + private fun archiveLine(frame: ByteArray, family: String = "whoop5"): String { + val hex = frame.joinToString("") { "%02x".format(it) } + return """{"capturedAtMs":1,"trim":1,"family":"$family","frameHex":"$hex"}""" + "\n" + } + + /** The strict test: entirely-zero payload region -> yes; ONE non-zero byte anywhere -> no. */ + @Test fun zeroPayloadDetectionIsStrictAndHeaderAware() { + val empty = whoop5Frame(22, payloadByte = 0) + assertTrue( + "the header and CRC bytes are non-zero but the PAYLOAD region is all zero", + RawHistoryArchive.hasZeroPayload(empty, com.noop.protocol.DeviceFamily.WHOOP5), + ) + // A single populated byte in an otherwise-empty buffer is exactly what this archive is for. + val oneByte = empty.copyOf().also { it[21 + 40] = 1 } + assertFalse( + "'mostly zero' must NOT count as empty — one non-zero byte makes it informative", + RawHistoryArchive.hasZeroPayload(oneByte, com.noop.protocol.DeviceFamily.WHOOP5), + ) + // The last payload byte, immediately before the CRC trailer, is inside the region. + val lastByte = empty.copyOf().also { it[it.size - 5] = 1 } + assertFalse(RawHistoryArchive.hasZeroPayload(lastByte, com.noop.protocol.DeviceFamily.WHOOP5)) + // A payload region too short to judge is never called empty. + assertFalse(RawHistoryArchive.hasZeroPayload( + whoop5Frame(22, payloadByte = 0, payloadBytes = 8), com.noop.protocol.DeviceFamily.WHOOP5)) + assertFalse(RawHistoryArchive.hasZeroPayload( + byteArrayOf(0xAA.toByte(), 1), com.noop.protocol.DeviceFamily.WHOOP5)) + // The header length is family-aware: the puffin envelope is 4 bytes longer. + assertEquals(21, RawHistoryArchive.payloadStart(com.noop.protocol.DeviceFamily.WHOOP5)) + assertEquals(17, RawHistoryArchive.payloadStart(com.noop.protocol.DeviceFamily.WHOOP4)) + } + + /** + * THE DEFECT. With `enable_raw_data_w_ecg` armed the strap banks one all-zero record per second, so a + * purely age-ordered archive evicts a real contact-window record within the half hour. An informative + * frame must outrank an all-zero one of the SAME layout version, even when it is the OLDEST line in + * the file and the empties are the newest — so no floor can be what saves it. + */ + @Test fun evictLinesEvictsZeroPayloadBeforeInformative() { + val informative = archiveLine(whoop5Frame(22, payloadByte = 0, marker = 0xE7)) + val lines = ArrayList() + lines.add(informative) + repeat(200) { lines.add(archiveLine(whoop5Frame(22, payloadByte = 0))) } + + val kept = RawHistoryArchive.evictLines(lines, maxBytes = 4_096, floor = 2, zeroFloor = 2) + assertTrue(kept.sumOf { it.toByteArray(Charsets.UTF_8).size } <= 4_096) + assertTrue( + "the one informative line must survive a flood of all-zero-payload lines", + kept.contains(informative), + ) + assertTrue("the zero-payload surplus must actually have been evicted", kept.size < 200) + } + + /** + * A version whose records are ALL empty still keeps [zeroFloor] dated samples — enough to prove the + * artefact exists without letting it own the archive. + */ + @Test fun allZeroVersionKeepsItsSmallFloor() { + val lines = ArrayList() + repeat(200) { lines.add(archiveLine(whoop5Frame(22, payloadByte = 0))) } + repeat(200) { i -> lines.add(archiveLine(whoop5Frame(18, payloadByte = 0, marker = (i and 0xFF).coerceAtLeast(1)))) } + + val kept = RawHistoryArchive.evictLines(lines, maxBytes = 4_096, floor = 2, zeroFloor = 2) + val keptZeroVersion = kept.count { it.contains("2f16") } // 0x2f = type 47, 0x16 = v22 + assertTrue( + "an all-empty version must keep its small floor, not vanish (kept $keptZeroVersion)", + keptZeroVersion >= 2, + ) + } + + /** The eviction target is a low-water mark below the cap, not the cap itself — see LOW_WATER_DIVISOR. */ + @Test fun evictionLeavesHeadroomSoAOneHzStreamDoesNotRewriteEveryRecord() { + val lines = (0 until 400).map { jsonl(18, filler = "%02x".format(it and 0xFF)) } + val kept = RawHistoryArchive.evictLines(lines, maxBytes = 4_096, floor = 2) + val bytes = kept.sumOf { it.toByteArray(Charsets.UTF_8).size } + assertTrue("must fit the cap", bytes <= 4_096) + assertTrue( + "must evict PAST the cap to 7/8 of it, or every subsequent batch rewrites the whole file", + bytes <= 4_096 - 4_096 / RawHistoryArchive.LOW_WATER_DIVISOR, + ) + } + + /** The shipped floors are the documented ones, and the zero floor is far smaller than the real one. */ + @Test fun shippedFloors() { + assertEquals(64, RawHistoryArchive.PER_VERSION_FLOOR) + assertEquals(8, RawHistoryArchive.ZERO_PAYLOAD_FLOOR) + assertTrue(RawHistoryArchive.ZERO_PAYLOAD_FLOOR < RawHistoryArchive.PER_VERSION_FLOOR) + assertEquals(5L * 1024 * 1024, RawHistoryArchive.REJECTED_ARCHIVE_MAX_BYTES) + } + @Test fun versionByteReadsTheRightIndexPerFamily() { // WHOOP 4 = frame[5]; WHOOP 5 = frame[9] (puffin envelope 4 B longer). val w4 = byteArrayOf(0xAA.toByte(), 1, 0, 0, 47, 19, 0, 0) diff --git a/android/app/src/test/java/com/noop/ble/SourceCoordinatorAdoptionTest.kt b/android/app/src/test/java/com/noop/ble/SourceCoordinatorAdoptionTest.kt index cc23195f99..b946a8daa7 100644 --- a/android/app/src/test/java/com/noop/ble/SourceCoordinatorAdoptionTest.kt +++ b/android/app/src/test/java/com/noop/ble/SourceCoordinatorAdoptionTest.kt @@ -76,6 +76,7 @@ class SourceCoordinatorAdoptionTest { override suspend fun deletePpgWaveformFor(deviceId: String) {} override suspend fun deleteRawImuFor(deviceId: String) {} override suspend fun deleteV18AuxFor(deviceId: String) {} + override suspend fun deleteSpo2PctFor(deviceId: String) {} override suspend fun deleteEventsFor(deviceId: String) {} override suspend fun deleteBatteryFor(deviceId: String) {} override suspend fun deleteDailyMetricsFor(deviceId: String) {} @@ -108,6 +109,7 @@ class SourceCoordinatorAdoptionTest { override suspend fun reKeyPpgWaveform(from: String, to: String) {} override suspend fun reKeyRawImu(from: String, to: String) {} override suspend fun reKeyV18Aux(from: String, to: String) {} + override suspend fun reKeySpo2Pct(from: String, to: String) {} override suspend fun reKeyEvents(from: String, to: String) {} override suspend fun reKeyBattery(from: String, to: String) {} override suspend fun reKeyDailyMetrics(from: String, to: String) {} diff --git a/android/app/src/test/java/com/noop/data/DeviceRegistryTest.kt b/android/app/src/test/java/com/noop/data/DeviceRegistryTest.kt index d1203f8b9a..3210fe57b5 100644 --- a/android/app/src/test/java/com/noop/data/DeviceRegistryTest.kt +++ b/android/app/src/test/java/com/noop/data/DeviceRegistryTest.kt @@ -91,6 +91,7 @@ class DeviceRegistryTest { override suspend fun deletePpgWaveformFor(deviceId: String) { deletedTables += "ppgWaveformSample" to deviceId } override suspend fun deleteRawImuFor(deviceId: String) { deletedTables += "rawImuSample" to deviceId } override suspend fun deleteV18AuxFor(deviceId: String) { deletedTables += "v18AuxSample" to deviceId } + override suspend fun deleteSpo2PctFor(deviceId: String) { deletedTables += "spo2PctSample" to deviceId } override suspend fun deleteEventsFor(deviceId: String) { deletedTables += "event" to deviceId } override suspend fun deleteBatteryFor(deviceId: String) { deletedTables += "battery" to deviceId } override suspend fun deleteDailyMetricsFor(deviceId: String) { deletedTables += "dailyMetric" to deviceId } @@ -127,6 +128,7 @@ class DeviceRegistryTest { override suspend fun reKeyPpgWaveform(from: String, to: String) {} override suspend fun reKeyRawImu(from: String, to: String) {} override suspend fun reKeyV18Aux(from: String, to: String) {} + override suspend fun reKeySpo2Pct(from: String, to: String) {} override suspend fun reKeyEvents(from: String, to: String) {} override suspend fun reKeyBattery(from: String, to: String) {} override suspend fun reKeyDailyMetrics(from: String, to: String) {} @@ -282,6 +284,9 @@ class DeviceRegistryTest { val expectedTables = setOf( "hrSample", "rrInterval", "spo2Sample", "skinTempSample", "respSample", "gravitySample", "stepSample", "ppgHrSample", "ppgWaveformSample", "rawImuSample", "v18AuxSample", + // v34/MIGRATION_25_26: never pruned, so a wipe that missed it would leave YEARS of + // blood-oxygen readings behind after the user asked for them to be gone. + "spo2PctSample", "event", "battery", "dailyMetric", "sleepSession", "journal", "workout", "appleDaily", "metricSeries", "dayOwnership", "scoreInputProvenance", diff --git a/android/app/src/test/java/com/noop/data/SchemaOracleTest.kt b/android/app/src/test/java/com/noop/data/SchemaOracleTest.kt index 14aec7c2f4..a482a41d4b 100644 --- a/android/app/src/test/java/com/noop/data/SchemaOracleTest.kt +++ b/android/app/src/test/java/com/noop/data/SchemaOracleTest.kt @@ -238,25 +238,67 @@ class SchemaOracleTest { * migrations by NAME and applies them in registration order; Room keys by integer version, so the * only thing that can be compared mechanically is the RESULTING schema — which is what the tests * above do. Pinning the two identifier spaces is what forces them to be re-checked together. + * + * The one sanctioned exception is a DECLARED lineage. A fork that numbered its own migrations before + * upstream reached those numbers cannot renumber its way out — GRDB keys by identifier string, so + * renaming one makes it read as un-applied on a device that already ran it. Such a fork lists its + * identifiers under `grdbMigrationLineages`; everything NOT listed is the baseline lineage and is + * still held to exactly v1..vN. An UNDECLARED collision still fails, which is the #369-vs-#475 case. + * Mirrors `testGrdbMigrationIdentifiersAreUniqueAndSequential` in the Swift half. */ @Test fun pinnedMigrationIdentifiersAreCoherent() { val oracle = loadOracle() val grdb = oracle.getJSONArray("grdbMigrations").strings() assertEquals("duplicate GRDB migration identifier in schema_oracle.json", grdb.size, grdb.toSet().size) - grdb.forEachIndexed { i, id -> - val n = id.removePrefix("v").takeWhile { it.isDigit() }.toIntOrNull() + + // A declared lineage must be true (every member pinned) and self-consistent (its own vN strictly + // increasing), so the ledger can only shrink deliberately — same rule the divergence reasons live by. + val lineages = oracle.optJSONObject("grdbMigrationLineages") ?: JSONObject() + val claimedBy = HashMap() + for (name in lineages.keys().asSequence().sorted()) { + val migrations = lineages.getJSONObject(name).getJSONArray("migrations").strings() + assertTrue("lineage '$name' declares no migrations — delete it", migrations.isNotEmpty()) + val numbers = migrations.map { id -> + val other = claimedBy.put(id, name) + assertTrue("'$id' is claimed by both lineage '$other' and '$name'", other == null) + assertTrue( + "lineage '$name' declares '$id', which is not in grdbMigrations — a lineage entry that " + + "stopped being true must be deleted, not left to rot", + grdb.contains(id), + ) + val n = migrationVersion(id) + assertNotNull("lineage '$name' member '$id' is not of the form v[-slug]", n) + n!! + } + numbers.zipWithNext { a, b -> + assertTrue( + "lineage '$name' goes v$a then v$b — a lineage numbers its own migrations, so its vN " + + "must strictly increase", + b > a, + ) + } + } + + grdb.filterNot { claimedBy.containsKey(it) }.forEachIndexed { i, id -> assertEquals( - "GRDB migration '$id' claims v$n but is #${i + 1} in registration order — two migrations " + - "claiming the same vN, or a gap, makes the GRDB-name <-> Room-version mapping ambiguous.", + "GRDB migration '$id' claims v${migrationVersion(id)} but is #${i + 1} in the BASELINE " + + "lineage (grdbMigrations minus every lineage grdbMigrationLineages declares) — two " + + "migrations claiming the same vN, or a gap, makes the GRDB-name <-> Room-version " + + "mapping ambiguous. Renumber before merging, or, if the identifier already shipped and " + + "renaming it would wedge a live database, declare it in grdbMigrationLineages.", i + 1, - n, + migrationVersion(id), ) } // loadRoomSchema asserts the exported version equals this; call it so the check is not vacuous. loadRoomSchema(oracle.getInt("roomVersion")) } + /** The `N` of a `v[-slug]` migration identifier, or null if it is not of that form. */ + private fun migrationVersion(id: String): Int? = + id.removePrefix("v").takeWhile { it.isDigit() }.toIntOrNull() + /** * The Android and Swift copies of the oracle MUST be byte-identical, so neither platform can edit its * fixture without the other. Skips gracefully if the Swift tree isn't present. diff --git a/android/app/src/test/java/com/noop/data/Spo2PctDurableMigrationTest.kt b/android/app/src/test/java/com/noop/data/Spo2PctDurableMigrationTest.kt new file mode 100644 index 0000000000..699366a249 --- /dev/null +++ b/android/app/src/test/java/com/noop/data/Spo2PctDurableMigrationTest.kt @@ -0,0 +1,210 @@ +package com.noop.data + +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.lang.reflect.Proxy + +/** + * MIGRATION_25_26 — the durable `@82` SpO2 percentage table. Twin of the Swift GRDB migration + * `v34-spo2-pct-durable` and its `Spo2PctDurableTests`. + * + * The 5/MG emits its own computed SpO2 % on `@82` of the v18 record. That byte has been banked since + * MIGRATION_24_25 — inside `v18AuxSample.fields`, a table CAPPED at 604,800 rows/device (~7 days at 1 Hz) + * because the aux blob costs ~30 MB/day. So every reading rolled off after a week and was gone + * permanently: the strap trims its own history the moment an offload is acked, leaving no second copy + * anywhere. `spo2PctSample` is the narrow, never-pruned sibling that fixes that. + * + * The SQL here is hand-written, which is exactly where a mistake is possible, so this test guards its + * shape; the write path is exercised through a Proxy DAO (the DeepCaptureMigrationTest pattern — there is + * no SQLite driver on this unit-test classpath) and the DELETE/upsert semantics are proved end-to-end on + * the Swift twin, which CI runs. + */ +class Spo2PctDurableMigrationTest { + + @Test + fun migrationCreatesTheDurableTableAndTouchesNothingElse() { + val sql = WhoopDatabase.SPO2_PCT_DURABLE_MIGRATION_SQL + assertEquals("one statement: a brand-new table, nothing altered", 1, sql.size) + + // The column order must match Spo2PctSampleEntity's field order (deviceId, ts, pct) and the GRDB + // schema, or a fresh install, a migrated install and an iOS `.noopbak` disagree. + assertEquals( + "CREATE TABLE IF NOT EXISTS `spo2PctSample` (`deviceId` TEXT NOT NULL, " + + "`ts` INTEGER NOT NULL, `pct` INTEGER NOT NULL, " + + "PRIMARY KEY(`deviceId`, `ts`))", + sql[0], + ) + + // Nothing in this migration may touch an existing row or an existing table. + for (stmt in sql) { + val upper = stmt.uppercase() + for (banned in listOf("DROP ", "DELETE ", "UPDATE ", "INSERT ", "ALTER ")) { + assertFalse("migration must not contain $banned", upper.contains(banned)) + } + } + assertEquals(25, WhoopDatabase.MIGRATION_25_26.startVersion) + assertEquals(26, WhoopDatabase.MIGRATION_25_26.endVersion) + } + + /** + * The v3 `spo2Sample` table is NOT the home for this and must be left exactly as it was: its + * `red`/`ir` are NOT NULL raw ADC channels off the WHOOP 4.0 v24 layout, and + * `AnalyticsEngine.nightlySpo2RawMeans` averages them into a live tile. Storing percentages there + * would have meant fabricating red/ir zeros on every insert and dragging that mean toward zero. + */ + @Test + fun theWhoop4RawTableIsUntouched() { + assertFalse( + "the percentage and the raw ADC pair never share a table", + WhoopDatabase.SPO2_PCT_DURABLE_MIGRATION_SQL.any { it.contains("spo2Sample") }, + ) + // Its own entity, unchanged: a percentage has no home among these columns. + val raw = Spo2Sample("d", 1L, red = 100, ir = 200) + assertEquals(100, raw.red) + assertEquals(200, raw.ir) + } + + // MARK: - the in-band demultiplex, exercised through the real insert path + + /** Capture every `spo2PctSample` row `WhoopRepository.insert` would write for [aux]. */ + private fun bankedFor(aux: List): List = runBlocking { + var rows: List = emptyList() + val dao = Proxy.newProxyInstance( + WhoopDao::class.java.classLoader, + arrayOf(WhoopDao::class.java), + ) { _, method, args -> + when (method.name) { + "insertV18Aux" -> listOf(1L) + "pruneV18Aux" -> Unit + "insertSpo2Pct" -> { + @Suppress("UNCHECKED_CAST") + rows = args[0] as List + rows.map { 1L } + } + else -> throw UnsupportedOperationException("spo2-pct insert must not call ${method.name}") + } + } as WhoopDao + WhoopRepository(dao).insert(StreamBatch(v18Aux = aux), "my-whoop") + rows + } + + @Test + fun inBandValuesPersistVerbatim() { + val banked = bankedFor( + listOf( + V18AuxRow(ts = 100, auxByte82 = 97), + V18AuxRow(ts = 101, auxByte82 = 94), + V18AuxRow(ts = 102, auxByte82 = 88), + ), + ) + assertEquals(listOf(100L, 101L, 102L), banked.map { it.ts }) + assertEquals(listOf(97, 94, 88), banked.map { it.pct }) + assertEquals(listOf("my-whoop", "my-whoop", "my-whoop"), banked.map { it.deviceId }) + } + + /** + * The band is INCLUSIVE at both ends — 70 and 100 are real readings, and a gate that quietly dropped + * the endpoints would bias every night's median upward at exactly the readings that matter most. + */ + @Test + fun bandBoundariesAreInclusive() { + val banked = bankedFor(listOf(V18AuxRow(ts = 200, auxByte82 = 70), V18AuxRow(ts = 201, auxByte82 = 100))) + assertEquals(listOf(70, 100), banked.map { it.pct }) + assertEquals(70..100, SPO2_CANDIDATE_IN_BAND) + } + + /** + * `@82` is MULTIPLEXED: the same byte carries measurements, bit-7 status sentinels, sub-70 diagnostic + * codes, and 0 for "not emitted". Only 70..100 is a percentage of anything. A sentinel banked as a + * percentage would not merely be wrong, it would be *plausible* — 0x88 is 136, which no reader would + * flag, and 0x08 is 8, which reads as catastrophic hypoxia. Rejection has to happen at the boundary. + */ + @Test + fun statusSentinelsAndDiagnosticCodesAreRejected() { + val rejected = listOf(0x00L, 0x08L, 0x45L, 0x80L, 0x88L, 0x90L, 0xA0L, 0xA8L, 0xFFL) + val banked = bankedFor(rejected.mapIndexed { i, v -> V18AuxRow(ts = 300L + i, auxByte82 = v) }) + assertTrue("no sentinel or diagnostic code may be banked as a percentage", banked.isEmpty()) + } + + /** + * 101..127 is the EMPTY BAND — nothing has ever been observed there across 626,725 production + * records. It is what makes measurements and bit-7 sentinels separable by value alone, with no side + * channel. A value appearing there would mean the demultiplex assumption has broken, so it must not be + * admitted on the strength of merely being "close to 100". + */ + @Test + fun theEmptyBandAbove100IsRejected() { + val banked = bankedFor((101..127).mapIndexed { i, v -> V18AuxRow(ts = 400L + i, auxByte82 = v.toLong()) }) + assertTrue(banked.isEmpty()) + } + + /** An aux record with no `@82` slot at all banks no percentage — absence stays absence. */ + @Test + fun absentSlotBanksNothing() { + assertTrue(bankedFor(listOf(V18AuxRow(ts = 500, statusWord = 7))).isEmpty()) + } + + /** + * The durable rows are forked from the SAME aux samples, but through their OWN loop — the empty-blob + * skip on the aux side must never be able to drop an SpO2 reading, and the aux write must still happen. + */ + @Test + fun theAuxRowAndThePercentageAreBothBanked(): Unit = runBlocking { + var auxRows = 0 + var pctRows = 0 + val dao = Proxy.newProxyInstance( + WhoopDao::class.java.classLoader, + arrayOf(WhoopDao::class.java), + ) { _, method, args -> + when (method.name) { + "insertV18Aux" -> { auxRows = (args[0] as List<*>).size; listOf(1L) } + "pruneV18Aux" -> Unit + "insertSpo2Pct" -> { pctRows = (args[0] as List<*>).size; listOf(1L) } + else -> throw UnsupportedOperationException("must not call ${method.name}") + } + } as WhoopDao + WhoopRepository(dao).insert( + StreamBatch(v18Aux = listOf(V18AuxRow(ts = 600, auxByte82 = 96))), + "my-whoop", + ) + assertEquals("the raw byte is still captured in the aux table — this gate demultiplexes, " + + "it does not discard", 1, auxRows) + assertEquals(1, pctRows) + } + + /** + * THE LOAD-BEARING PROPERTY. The aux sweep must not reach this table: `pruneV18Aux` deletes only + * `FROM v18AuxSample`, and there is deliberately no prune query for `spo2PctSample` at all. This is + * the entire reason MIGRATION_25_26 exists — before it, seven-day-old SpO2 was destroyed with no + * recoverable copy. + */ + @Test + fun thereIsNoPruneForTheDurableTable() { + val prunes = WhoopDao::class.java.methods.filter { it.name.startsWith("prune") }.map { it.name } + assertTrue("v18AuxSample is the capped one", prunes.contains("pruneV18Aux")) + assertTrue( + "spo2PctSample must have NO rolling prune — it is retained forever, by design. " + + "Found: $prunes", + prunes.none { it.startsWith("pruneSpo2Pct") && it != "pruneSpo2PctByTs" }, + ) + // The timestamp heal is the one exception, and it deletes only implausible-clock rows: those are + // never pruned on AGE, so an unhealed out-of-bounds row here would persist forever. + assertTrue(prunes.contains("pruneSpo2PctByTs")) + } + + /** + * A never-pruned table makes delete-means-gone MORE important, not less: without this the wipe would + * leave years of blood-oxygen readings behind after the user asked for them to be erased. The + * whole-set guard is `DeviceRegistryTest.deleteDeviceDataCallsEveryDaoDeleteMethod`; this pins that + * the DAO method exists at all. + */ + @Test + fun theTableIsDeviceScopedForDeleteAllData() { + val names = DeviceRegistryDao::class.java.methods.map { it.name } + assertTrue(names.contains("deleteSpo2PctFor")) + assertTrue("adopt-serial must re-key it too", names.contains("reKeySpo2Pct")) + } +} diff --git a/android/app/src/test/java/com/noop/data/V18AuxCodecOffsetTest.kt b/android/app/src/test/java/com/noop/data/V18AuxCodecOffsetTest.kt new file mode 100644 index 0000000000..c30a7d109e --- /dev/null +++ b/android/app/src/test/java/com/noop/data/V18AuxCodecOffsetTest.kt @@ -0,0 +1,82 @@ +package com.noop.data + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The blob-offset question, pinned. Twin of the Swift `V18AuxCodecOffsetTests`. + * + * The cloud reader takes `fields[23]` of the STORED `v18AuxSample.fields` blob and gets physiologically + * valid SpO2 out of it. That works — but it is an ARITHMETIC COINCIDENCE of a fully-populated record, not + * a property of the format. [V18AuxCodec] is a presence-bitmap codec: absent slots are omitted from the + * body entirely, so every slot's byte offset depends on which slots ahead of it were present. + * + * These tests establish both halves, because only the pair is decisive. The first alone would license the + * fragile read; the second is what says the device write path must not use it — which is why + * `WhoopRepository.insert` forks `spo2PctSample` rows off `V18AuxRow.auxByte82`, the DECODED value, and + * never indexes the blob. + */ +class V18AuxCodecOffsetTest { + + /** A record carrying every slot, with distinguishable values. */ + private fun fullRow(): V18AuxRow = V18AuxRow( + ts = 1L, + recordIndex = 0x11223344L, rrCount = 4L, cardiacFlags = 0x21L, hrQualityFlags = 0x82L, + heartRateAlt = 58L, rrPacked = 0x0505L, cardiacStatus = 0x31L, stepCadence = 0x41L, + statusWord = 0x0606L, statusWord1 = 0x0707L, statusWord2 = 0x0808L, auxByte82 = 93L, + opticalBaselineA = 0x51L, opticalBaselineB = 0x52L, opticalAmpA = 0x53L, opticalAmpB = 0x54L, + unknownF32Bits = 0x55667788L, + ) + + private fun ByteArray.u8(i: Int): Int = this[i].toInt() and 0xFF + + /** + * A record carrying every slot: `auxByte82` lands at blob byte 23. Header is 5 bytes (version + u32 + * bitmap), then the eleven slots ahead of it occupy 4+1+1+1+1+2+1+1+2+2+2 = 18 bytes, so it sits at + * 5 + 18 = 23 and the whole blob is 32 bytes. THIS is why the cloud's `fields[23]` reads real + * percentages: in production every one of those slots is present. + */ + @Test fun fullyPopulatedRecordPutsAuxByte82AtBlobByte23() { + val blob = V18AuxCodec.pack(fullRow()) + + assertEquals("5-byte header + 27 body bytes", 32, blob.size) + assertEquals("byte 23 of a fully-populated blob IS @82", 93, blob.u8(23)) + // And it agrees with the decoded row, which is the only contract that actually holds. + assertEquals(93L, V18AuxCodec.unpack(blob, 1L).auxByte82) + // Derive the offset from the slot table rather than trusting the literal, so a future slot + // inserted ahead of @82 fails HERE with a readable number instead of silently moving the cloud's + // read onto a neighbouring byte. + val ahead = V18AuxSlot.entries + .filter { it.index < V18AuxSlot.AUX_BYTE_82.index } + .sumOf { it.width } + assertEquals(23, V18AuxCodec.HEADER_BYTES + ahead) + } + + /** + * ONE absent slot ahead of it and byte 23 is a different field entirely — while the decoded value is + * still correct. The blob read does not degrade loudly; it returns a neighbouring byte that can look + * exactly like a plausible saturation. + */ + @Test fun oneAbsentEarlierSlotMovesAuxByte82OffByte23() { + // Identical to the record above except `recordIndex` (4 bytes, slot 0) is absent. + val blob = V18AuxCodec.pack(fullRow().copy(recordIndex = null)) + + assertEquals("four fewer body bytes without recordIndex", 28, blob.size) + // @82 has slid four bytes earlier... + assertEquals(93, blob.u8(19)) + // ...and byte 23 now holds opticalAmpB, four slots further along. Reading it as a percentage would + // yield 0x54 = 84: in band, unremarkable, and completely fabricated. Note it does NOT fail loudly + // — an optical amplitude byte and a saturation percentage occupy the same numeric range, so the + // bad read is indistinguishable from a good one at the call site. + assertEquals(0x54, blob.u8(23)) + assertNotEquals(93, blob.u8(23)) + assertTrue( + "the wrong byte is still 'in band' — which is exactly why this read is unsafe", + blob.u8(23) in SPO2_CANDIDATE_IN_BAND, + ) + // The decoded read is unaffected — it is driven by the bitmap, not by an offset. + assertEquals(93L, V18AuxCodec.unpack(blob, 1L).auxByte82) + } +} diff --git a/android/app/src/test/java/com/noop/protocol/UnmappedHistoricalLayoutTest.kt b/android/app/src/test/java/com/noop/protocol/UnmappedHistoricalLayoutTest.kt new file mode 100644 index 0000000000..78ade014b7 --- /dev/null +++ b/android/app/src/test/java/com/noop/protocol/UnmappedHistoricalLayoutTest.kt @@ -0,0 +1,154 @@ +package com.noop.protocol + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The archive decision for a WHOOP 5/MG type-47 record must be made on its LAYOUT VERSION, not on what + * the record happened to decode to. Twin of the Swift `UnmappedHistoricalLayoutTests`. + * + * [rejectedHistoricalRecords] used to ask only "did [decodeHistorical] produce a record?". That screen is + * the wrong QUESTION for a layout NOOP has no field map for: a record from an unmapped version that + * answered yes would be kept NOWHERE — not as rows (nothing mapped it) and not as bytes (it passed the + * archive filter) — and the strap frees it on the very next trim ack. An on-demand capture (the MG's ~30 s + * ECG record, banked to flash and pulled back in a later offload) is exactly the shape of record that can + * look well-formed enough to pass. + * + * The screen survives in practice only because of an accident: [decodeWhoop5Historical] returns null for + * every version but 18 (`unmappedLayoutsDecodeNothing` pins that this is true right now). One added + * static field for type 47, or one partially-mapped new version, would silently reopen the hole. These + * pin the version-based decision instead, so the archive no longer depends on that accident holding. + */ +class UnmappedHistoricalLayoutTest { + + private fun bytes(s: String): ByteArray = s.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + + // A real WHOOP 5/MG type-47 v18 record (from Whoop5HistoricalDecodeTest): decodes a real unix, a + // plausible heart rate and a ~1 g gravity vector — i.e. it PASSES the old decode-outcome screen. + private val whoop5V18Hex = + "aa01740001003fb12f1280733d8401b69f266a66460066025a0265020000000000007b0a8d656463ff0012163cf6a439bf2924fd3ed763fe3e3200aa000000000000000000f7000901f10b0007010c020c00000000000000000000000000000000000000000000000100656f1e1e0000009d61a7c00000003e862817" + + /** + * Re-stamp the WHOOP 5 CRC32 trailer after mutating the version byte, so the frame stays CRC-VALID. + * Load-bearing for every test here: a CRC failure is archived on its own, which would make the + * version-based path untestable. + */ + private fun whoop5FrameWithVersion(version: Int): ByteArray { + val f = bytes(whoop5V18Hex) + f[9] = version.toByte() + val payloadEnd = f.size - 4 + val c = Crc.crc32(f, 8, payloadEnd) + for (b in 0 until 4) f[payloadEnd + b] = ((c shr (8 * b)) and 0xFF).toByte() + return f + } + + // MARK: - defect: an unmapped layout that decodes real biometrics must still be archived + + /** + * The headline case. The frame's BYTES are a real v18 record — at the v18 offsets they carry a valid + * unix, a plausible heart rate and a ~1 g gravity vector, the exact combination that made the old + * screen say "decodable, nothing lost". Only the layout-version byte differs. Whatever such a record + * decodes to, its bytes must reach the archive. + */ + @Test fun unmappedLayoutIsArchivedEvenThoughItsBytesDecodeCleanlyAsV18() { + // Precondition: read as its true version, this record decodes everything the old screen wanted. + val asV18 = decodeHistorical(bytes(whoop5V18Hex), DeviceFamily.WHOOP5)!! + assertEquals("precondition: these bytes DO decode a unix", 1780916150, asV18["unix"]) + assertEquals("precondition: these bytes DO decode a heart rate", 102, asV18["heart_rate"]) + assertTrue("precondition: these bytes DO decode a gravity vector", asV18["gravity_x"] is Double) + + val unmapped = whoop5FrameWithVersion(22) + assertEquals( + "precondition: the record is CRC-VALID, so only the layout decision can archive it", + true, Framing.parseFrame(unmapped, DeviceFamily.WHOOP5).crcOk, + ) + assertTrue(isUnmappedWhoop5HistoricalRecord(unmapped)) + assertEquals( + "a record from a layout NOOP cannot map must be archived whatever it decoded", + listOf(unmapped.toList()), + rejectedHistoricalRecords(listOf(unmapped), DeviceFamily.WHOOP5).map { it.toList() }, + ) + } + + /** Every version outside [MAPPED_WHOOP5_HISTORICAL_VERSIONS] is archived — no gaps, no lucky values. */ + @Test fun everyUnmappedVersionIsArchived() { + for (v in 0..255) { + if (v in MAPPED_WHOOP5_HISTORICAL_VERSIONS) continue + val f = whoop5FrameWithVersion(v) + assertEquals( + "hist_version $v has no field map, so its bytes must be archived", + 1, rejectedHistoricalRecords(listOf(f), DeviceFamily.WHOOP5).size, + ) + } + } + + // MARK: - lockstep: the version set and the decoder's dispatch cannot drift apart + + /** + * [MAPPED_WHOOP5_HISTORICAL_VERSIONS] is the single source of truth for the archive decision, so it + * must not claim more (or fewer) versions than the decoder actually maps. Proved from the decoder's + * own behaviour: EVERY version outside the set decodes to nothing at all. + */ + @Test fun unmappedLayoutsDecodeNothing() { + for (v in 0..255) { + if (v in MAPPED_WHOOP5_HISTORICAL_VERSIONS) continue + assertNull( + "v$v is outside the mapped set but the decoder produced a record for it — the set and " + + "the version gate in decodeWhoop5Historical have drifted", + decodeHistorical(whoop5FrameWithVersion(v), DeviceFamily.WHOOP5), + ) + } + } + + /** + * The mapped versions ARE recognised as mapped (the other direction of the lockstep). + * + * ⚠️ This set is {18, 26}, NOT the Swift `[18, 20, 21, 26]`. Swift dispatches v20 (raw optical) and + * v21 (raw 6-axis IMU) through `decodeWhoop5HistoricalV2021`; the Kotlin historical path has no such + * branch — `Whoop5RawOptical`/`Whoop5RawImu` are wired to the LIVE deep-buffer route, not to the + * type-47 historical dispatch. Claiming 20/21 here would assert a field map Android does not have. + * Update BOTH this assertion and the constant when a v20/v21 historical decoder lands on this side. + */ + @Test fun mappedVersionsAreNotTreatedAsUnmapped() { + assertEquals(setOf(18, 26), MAPPED_WHOOP5_HISTORICAL_VERSIONS) + for (v in MAPPED_WHOOP5_HISTORICAL_VERSIONS) { + assertFalse( + "v$v has a field map and must not be archived on layout grounds", + isUnmappedWhoop5HistoricalRecord(whoop5FrameWithVersion(v)), + ) + } + } + + /** + * A cleanly-decoding v18 record is still NOT archived — widening the screen must not start archiving + * the layouts NOOP already understands wholesale. v26 keeps its own exclusion (raw PPG has a durable + * `ppgWaveformSample` stream; it is known-skipped, not lost). + */ + @Test fun mappedRecordsAreStillNotArchived() { + assertTrue(rejectedHistoricalRecords(listOf(bytes(whoop5V18Hex)), DeviceFamily.WHOOP5).isEmpty()) + assertTrue(rejectedHistoricalRecords(listOf(whoop5FrameWithVersion(26)), DeviceFamily.WHOOP5).isEmpty()) + } + + /** + * The layout rule is WHOOP 5-only: it keys off `frame[9]`, which on a WHOOP 4 frame is a payload byte, + * not a version. WHOOP 4's unmapped versions go through the validated v24 fallback in + * [decodeHistorical], which keeps a record only when it decodes to a ~1 g gravity vector and a + * plausible HR and otherwise drops the biometrics — so those records reach the archive by the + * decode-outcome route and must not be dragged in by this one. + */ + @Test fun whoop4FramesAreUnaffectedByTheWhoop5LayoutRule() { + // A synthetic WHOOP 4 V24 type-47 record (HR=63) that decodes cleanly. + val v24Hex = + "aa5a008e2f18000000000000f153650000000000003f0152030000000000000000dc053075" + + "000000cdcc4c3dcdcccc3d5a657e3f00000040cdcc4c3dcdcccc3d5a657e3f504668428403" + + "200364006400b80bb80b000000000000c25c1a88" + assertTrue(rejectedHistoricalRecords(listOf(bytes(v24Hex)), DeviceFamily.WHOOP4).isEmpty()) + assertFalse( + "a WHOOP 4 frame has no type-47 byte at index 8 — the rule must not fire on it", + isUnmappedWhoop5HistoricalRecord(bytes(v24Hex)), + ) + } +} diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5EcgProbeTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5EcgProbeTest.kt index ab1f288dba..b795a1f8ca 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5EcgProbeTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5EcgProbeTest.kt @@ -2,6 +2,7 @@ package com.noop.protocol import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotEquals import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test @@ -36,6 +37,7 @@ class Whoop5EcgProbeTest { label = "$name($cmd)", outcome = outcome, requestsRealtimeData = Whoop5Ecg.requestsRealtimeData(cmd, arg), + sentArgument = arg, replyHex = replyHex, ) } @@ -103,11 +105,11 @@ class Whoop5EcgProbeTest { @Test fun attestedResultCodesOutrankTheShapeHeuristic() { assertEquals( - Whoop5EcgProbe.Verdict.DataRequestRefused(listOf("TOGGLE_LABRADOR_DATA_GENERATION(124)")), + Whoop5EcgProbe.Verdict.DataRequestRefused(listOf("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1")), Whoop5EcgProbe.verdict(listOf(sent(124, 1, Whoop5EcgProbe.CommandOutcome.Failure)), 12, 30), ) assertEquals( - Whoop5EcgProbe.Verdict.OpcodeUnsupported(listOf("TOGGLE_LABRADOR_FILTERED(139)")), + Whoop5EcgProbe.Verdict.OpcodeUnsupported(listOf("TOGGLE_LABRADOR_FILTERED(139) arg=1")), Whoop5EcgProbe.verdict(listOf(sent(139, 1, Whoop5EcgProbe.CommandOutcome.Unsupported)), 12, 30), ) assertEquals( @@ -159,7 +161,7 @@ class Whoop5EcgProbeTest { ), ) assertEquals( - Whoop5EcgProbe.Verdict.NoDataRequested(listOf("SELECT_WRIST(123)")), + Whoop5EcgProbe.Verdict.NoDataRequested(listOf("SELECT_WRIST(123) arg=1")), Whoop5EcgProbe.verdict(steps, 0, 30), ) val text = Whoop5EcgProbe.report(steps, 0, emptyList(), 30) @@ -183,7 +185,7 @@ class Whoop5EcgProbeTest { ), ) assertEquals( - Whoop5EcgProbe.Verdict.CommandRefused(listOf("SELECT_WRIST(123)")), + Whoop5EcgProbe.Verdict.CommandRefused(listOf("SELECT_WRIST(123) arg=0")), Whoop5EcgProbe.verdict(steps, 0, 30), ) val text = Whoop5EcgProbe.report(steps, 0, emptyList(), 30) @@ -203,9 +205,9 @@ class Whoop5EcgProbeTest { assertEquals( Whoop5EcgProbe.Verdict.NoDataRequested( listOf( - "TOGGLE_LABRADOR_DATA_GENERATION(124)", - "TOGGLE_LABRADOR_RAW_SAVE(125)", - "TOGGLE_LABRADOR_FILTERED(139)", + "TOGGLE_LABRADOR_DATA_GENERATION(124) arg=0", + "TOGGLE_LABRADOR_RAW_SAVE(125) arg=0", + "TOGGLE_LABRADOR_FILTERED(139) arg=0", ), ), Whoop5EcgProbe.verdict(steps, 0, 30), @@ -217,7 +219,7 @@ class Whoop5EcgProbeTest { // RAW_SAVE names flash. A realtime listen window observes nothing from it even on total success, // so a raw-save-only run cannot be read as "accepted and then silent" (#891 hypothesis (b)). assertEquals( - Whoop5EcgProbe.Verdict.NoDataRequested(listOf("TOGGLE_LABRADOR_RAW_SAVE(125)")), + Whoop5EcgProbe.Verdict.NoDataRequested(listOf("TOGGLE_LABRADOR_RAW_SAVE(125) arg=1")), Whoop5EcgProbe.verdict(listOf(sent(125, 1, Whoop5EcgProbe.CommandOutcome.Success)), 0, 30), ) } @@ -232,7 +234,7 @@ class Whoop5EcgProbeTest { ) assertEquals( Whoop5EcgProbe.Verdict.DataRequestNotAccepted( - listOf("TOGGLE_LABRADOR_FILTERED(139)", "TOGGLE_LABRADOR_DATA_GENERATION(124)"), + listOf("TOGGLE_LABRADOR_FILTERED(139) arg=1", "TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1"), ), Whoop5EcgProbe.verdict(steps, 0, 30), ) @@ -273,7 +275,7 @@ class Whoop5EcgProbeTest { @Test fun verdictUnsupportedIsReportedAsItselfNotAsABlock() { assertEquals( - Whoop5EcgProbe.Verdict.OpcodeUnsupported(listOf("TOGGLE_LABRADOR_FILTERED(139)")), + Whoop5EcgProbe.Verdict.OpcodeUnsupported(listOf("TOGGLE_LABRADOR_FILTERED(139) arg=1")), Whoop5EcgProbe.verdict(listOf(sent(139, 1, Whoop5EcgProbe.CommandOutcome.Unsupported)), 0, 30), ) } @@ -348,13 +350,56 @@ class Whoop5EcgProbeTest { ) val text = Whoop5EcgProbe.report(steps, 0, listOf("type=0x28 len=220"), 30) assertTrue(text.contains("DATA REQUEST REFUSED")) - assertTrue(text.contains("SELECT_WRIST(123): SUCCESS(1)")) - assertTrue(text.contains("TOGGLE_LABRADOR_DATA_GENERATION(124): FAILURE(0)")) + assertTrue(text.contains("SELECT_WRIST(123) arg=1: SUCCESS(1)")) + assertTrue(text.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1: FAILURE(0)")) assertTrue(text.contains("type=0x28 len=220")) assertTrue(text.contains("aabb")) assertTrue(text.contains("not a medical measurement or a diagnosis")) } + /** + * Twin of Swift `testStartAndRestartRunsRenderDistinguishably`. A START run and a RESTART run send + * the SAME three opcodes and differ in exactly one byte, so without the argument annotation their + * reports are character-for-character identical — and a report is the artefact that gets copied out + * of an app and pasted into an issue, long after the log that recorded the payload bytes is gone. + */ + @Test + fun startAndRestartRunsRenderDistinguishably() { + fun run(control: Whoop5Ecg.ControlSignal): String = Whoop5EcgProbe.report( + listOf( + sent(139, 1, Whoop5EcgProbe.CommandOutcome.Success), + sent(125, 1, Whoop5EcgProbe.CommandOutcome.Success), + sent(124, control.raw, Whoop5EcgProbe.CommandOutcome.Success), + ), + 0, emptyList(), 30, + ) + val startText = run(Whoop5Ecg.ControlSignal.START) + val restartText = run(Whoop5Ecg.ControlSignal.RESTART) + assertTrue(startText.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=1: SUCCESS(1)")) + assertTrue(restartText.contains("TOGGLE_LABRADOR_DATA_GENERATION(124) arg=2: SUCCESS(1)")) + assertNotEquals(startText, restartText) + // Both are still the SAME verdict: restart asks for realtime data exactly like start does, so + // the annotation records what was sent without changing what the run is read as. + assertTrue(startText.contains("Accepted but SILENT")) + assertTrue(restartText.contains("Accepted but SILENT")) + } + + /** + * An UNSOLICITED reply has no known argument — nothing in an app layer sent it — and the report must + * say nothing rather than invent a value. + */ + @Test + fun anUnknownArgumentIsOmittedRatherThanGuessed() { + val step = Whoop5EcgProbe.Step( + label = "TOGGLE_LABRADOR_DATA_GENERATION(124)", + outcome = Whoop5EcgProbe.CommandOutcome.Success, + requestsRealtimeData = false, + ) + assertNull(step.sentArgument) + assertEquals("TOGGLE_LABRADOR_DATA_GENERATION(124)", step.labelWithArgument) + assertFalse(Whoop5EcgProbe.report(listOf(step), 0, emptyList(), 30).contains("arg=")) + } + /** * REGRESSION (#891), twin of Swift `testNoVerdictMentionsDeviceFlagsAtAll`. No verdict may name * `blockedByDeviceFlags` or a "device-flag block" AT ALL — not to assert it, and not to deny it. @@ -410,4 +455,133 @@ class Whoop5EcgProbeTest { assertFalse("report must not name $token", text.lowercase().contains(token.lowercase())) } } + + // MARK: - Unclassified-frame census + + private fun header(samples: Int): List = listOf( + 2, 0x05, 1, 1, 0, 1, 0, 1, 42, 0, 61, 63, 812 and 0xFF, (812 shr 8) and 0xFF, 17, + samples and 0xFF, (samples shr 8) and 0xFF, + ) + + private fun puffinFrame(type: Int, payload: List): ByteArray = + Framing.puffinCommandFrame( + cmd = 0x00, seq = 0x01, + payload = payload.map { it.toByte() }.toByteArray(), + type = type, + ) + + /** + * The census exists for the frames the triage says NO to. A heuristic that logs only its own hits + * destroys the evidence for its own misses: the 2-bytes-per-sample assumption discarded every 3-byte + * frame, and the report then said, truthfully, that nothing passed. + */ + @Test + fun censusRecordsAFrameTheTriageRejects() { + val frame = puffinFrame(0x1A, List(20) { 0xEE }) + assertFalse("fixture must be a triage MISS", Whoop5Ecg.plausibleFilteredFrame(frame)) + + val census = Whoop5EcgProbe.FrameCensus() + census.record(frame) + assertEquals(1, census.framesSeen) + assertEquals(1, census.buckets.size) + assertEquals(0x1A, census.buckets[0].typeByte) + assertEquals(1, census.buckets[0].count) + + val sample = census.buckets[0].samples[0] + assertEquals(frame.size, sample.frameLength) + assertEquals(21, sample.payloadLength) // 20 + the envelope's pad4 byte + assertEquals(0xEEEE, sample.numberOfECGSamples) // read, not believed + assertTrue(sample.widths.isEmpty()) + assertTrue(sample.headHex.startsWith("aa")) + assertEquals(" type=0x1a frames=1", census.lines[0]) + assertTrue(census.lines[1].contains("widths=none")) + assertTrue(census.lines[1].contains("payload=21")) + } + + /** A HIT is censused too — the census is the whole record of the window, not a rejects bin. */ + @Test + fun censusRecordsTriageHitsWithTheWidthThatAgreed() { + val frame = puffinFrame(0x28, header(samples = 4) + List(12) { 0x11 }) + assertTrue(Whoop5Ecg.plausibleFilteredFrame(frame)) + val census = Whoop5EcgProbe.FrameCensus() + census.record(frame) + assertEquals(listOf(3), census.buckets[0].samples[0].widths) + assertEquals(4, census.buckets[0].samples[0].numberOfECGSamples) + assertTrue(census.lines[1].contains("widths=3")) + } + + /** A frame whose CRC does not check out yields no decoded field — only its shape and its bytes. */ + @Test + fun censusReadsNoFieldOutOfAnUnverifiedFrame() { + val frame = puffinFrame(0x28, header(samples = 3) + listOf(1, 0, 2, 0, 3, 0)) + frame[frame.size - 1] = (frame[frame.size - 1].toInt() xor 0xFF).toByte() + val census = Whoop5EcgProbe.FrameCensus() + census.record(frame) + assertNull(census.buckets[0].samples[0].payloadLength) + assertNull(census.buckets[0].samples[0].numberOfECGSamples) + assertTrue(census.buckets[0].samples[0].widths.isEmpty()) + assertTrue(census.lines[1].contains("payload=? samples=? widths=none")) + } + + /** A 1 Hz stream must not grow the report without bound — and nothing may be dropped in silence. */ + @Test + fun censusCapsSamplesPerTypeButKeepsCounting() { + val frame = puffinFrame(0x30, List(20) { 0xEE }) + val census = Whoop5EcgProbe.FrameCensus() + repeat(50) { census.record(frame) } + assertEquals(50, census.framesSeen) + assertEquals(1, census.buckets.size) + assertEquals(50, census.buckets[0].count) + assertEquals(Whoop5EcgProbe.FrameCensus.MAX_SAMPLES_PER_TYPE, census.buckets[0].samples.size) + assertTrue(census.lines.any { it.contains("+47 more of this type, not recorded") }) + } + + @Test + fun censusCapsDistinctTypesAndCountsTheOverflow() { + val census = Whoop5EcgProbe.FrameCensus() + for (type in 0 until Whoop5EcgProbe.FrameCensus.MAX_TYPES + 4) { + census.record(puffinFrame(type, List(20) { 0xEE })) + } + assertEquals(Whoop5EcgProbe.FrameCensus.MAX_TYPES, census.buckets.size) + assertEquals(4, census.framesBeyondTypeCap) + assertEquals(Whoop5EcgProbe.FrameCensus.MAX_TYPES + 4, census.framesSeen) + assertTrue(census.lines.any { it.contains("4 frame(s) of further type bytes past the") }) + } + + @Test + fun censusIgnoresBuffersTooShortToCarryATypeByte() { + val census = Whoop5EcgProbe.FrameCensus() + census.record(ByteArray(0)) + census.record(ByteArray(8) { 0xAA.toByte() }) + assertTrue(census.isEmpty) + assertTrue(census.buckets.isEmpty()) + } + + /** The census is only useful if the operator can COPY it: it belongs in the report sheet. */ + @Test + fun reportCarriesTheCensusBesideTheTriageResult() { + val census = Whoop5EcgProbe.FrameCensus() + census.record(puffinFrame(0x1A, List(20) { 0xEE })) + val text = Whoop5EcgProbe.report( + steps = listOf( + sent(139, 1, Whoop5EcgProbe.CommandOutcome.Success), + sent(124, Whoop5Ecg.ControlSignal.START.raw, Whoop5EcgProbe.CommandOutcome.Success), + ), + ecgPacketsSeen = 0, + candidateFrames = emptyList(), + windowSeconds = 30, + census = census, + ) + assertTrue(text.contains("Candidate packet types: none — no frame passed the structural triage.")) + assertTrue(text.contains("Unclassified-frame census")) + assertTrue(text.contains("type=0x1a frames=1")) + assertTrue(text.contains("widths=none")) + assertTrue(text.contains(census.buckets[0].samples[0].headHex)) + } + + @Test + fun reportSaysSoWhenNoUnclassifiedFrameArrivedAtAll() { + val text = Whoop5EcgProbe.report(emptyList(), 0, emptyList(), 30) + assertTrue(text.contains("no unclassified frame arrived at all")) + } } diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5EcgRawHardwareTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5EcgRawHardwareTest.kt new file mode 100644 index 0000000000..3b3069621a --- /dev/null +++ b/android/app/src/test/java/com/noop/protocol/Whoop5EcgRawHardwareTest.kt @@ -0,0 +1,302 @@ +package com.noop.protocol + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The Kotlin twin of WhoopProtocolTests/Whoop5EcgRawHardwareTests.swift, against the SAME real bytes. + * + * The three complete 1584-byte type-47 layout-16 WHOOP MG flash records embedded here came off hardware + * on 2026-08-06 while the clasp electrodes were held; they are byte-identical to the Swift fixtures. The + * cross-platform parity contract says the two decoders must agree on stored values, and a synthetic + * fixture can only prove they agree about an assumption — these bytes prove they agree about the strap. + * + * Only the load-bearing assertions are mirrored: the two fixed-size containers (the leads-off block and + * the sample region), the resolved sample width, and that a populated record is not discarded. The full + * characterisation (including the header misalignment that is still open, and the 18-bit continuity + * argument) lives on the Swift side. + * + * No scale or unit is claimed for the sample values, and none is applied. + */ +class Whoop5EcgRawHardwareTest { + + /** The payload offset at which the record's numberOfECGSamples lands. NOT PUFFIN_PAYLOAD_START. */ + private val ecgPayloadStart = 17 + + /** unix 1786031627 — numberOfLeadsOffSamples == 11, the block exactly full. */ + private val hex11 = + "aa0128060100cde02f10036a61cd000bae746aeb51010a000100000000ffff00f401834ae78349e98345d5834703834c" + + "53834814834916834a0d83478e834d3e83510a8351bb834fda834a718347d383447c8344f3834ec6834c158347848348" + + "87834785834a2d834978834c33834d39834c07834b178352dc83537e8351fb8352968350db835319835048834540834a" + + "b3834de2834f1b834f0f835133834f31834f1a83515d834e178351f18351a38352628352558352fd834d4c834a698350" + + "de835762834e80834b218349338349e6835095835228834c578351b6834f53834ff58351f4834dcb834a6f8350dc8352" + + "658350d9834b2f834ba8834d50834e73834cd2834d3c8356698355e08356508357a48359bb835af9835a5f8353ae8352" + + "78835bde835c928358c08353a3835693835bae835b7a835c4a835e89835c8a835bb0835f2583608b83610f835ec9835f" + + "ee83611283606f8363e68361cd836cf583649b835d0783637983654d8360c0835f2883608f835a2c8352e78350f98349" + + "f18340e5833e90833c878338e0833677832ec9833094833bd28333be833941833e328343d7834d4e83570c8359228362" + + "0383622283647f83618c8365ce837351836e3e8367e28368fb836cb3836cd5836de9836d8383717a836f7a836dc4836f" + + "8783726783723c8370828375d0837291836de9836f4583740883785c8373eb837d748376af8374b283745f8371ad8373" + + "9a83763e8373eb8375d58375b48374f1837c6e838486837fc0837d3b8379cd837402837747837f4e83765783755b837a" + + "ab8386ab83876b837c6d8375ef8376cc837d41837b33837f4a837f37837c5583811a837d95837b77837cc1837dc88381" + + "11837c56837883837a32837da8837f2d8381ec837c388382cb838541837fef8378bd8374ac83731f837914837d9f837b" + + "93837f35837e22838094838290837e6b837b12837dac837fdc838048837fda83808f8380018381ee83822e8384ed8387" + + "7883836b837f858382428378e3837bf38382e683865183850583810883839283886583840c8388568389b4838af6838e" + + "148388b783875983893583861883865c838e6283915d838a58838d748391e583904b8393288397618391038391e98390" + + "e383905f838c82838b178390f683947e83963883963383981d839973839b94839974839894839b0f83952b839453839b" + + "78839d39839b708399028394bc839038838f648395be839a4a839ad0839f44839c18839b008399a98398ee839bbe839c" + + "6983967a8392c38396b28396088395218395d1839a16839fe3839e5f83997483948f83974c839afc8396a2839b40839e" + + "48839aec839903839a8f8399ef839f09839aa58397dc839c9483a4f383a2bd83a3e683a197839a968399c4839821839b" + + "8c839dd58398488398528394eb839806839b0d839ae2839a388396f1839521838de2839230839a1983a05083a3e58399" + + "fe83981b83982b8396be83a224839df2839a04839ecc8399c9839671839a6d83953a8396c1839a6583a05483a22e83a1" + + "3483a2e983a60083997983984883990d839e6b839ee783a030839dc183a0b983a68583a0c483a40583a0c6839f1d839c" + + "5683a05e83a0ac839d93839bd1839ebc83a53083aa8b83a71083a49883a0ff83a11f83a22f83a5ac83a69983a54383a3" + + "2b839b30839e0d839e2f83a040839fae839f99839b5b83a93f83aeb383a61783a78683a7d383a88383abf683aa6e83a6" + + "0383a79983a4cf83a60983a5dd83a7d083a48b83a61683a8c883a88f83a6f683a80a83a8ef83a5b083a73483aa4683a9" + + "1b83aca583b07883a4be839ed383a1f483a30783a3ea83a79c83a67d83a50383a70383b0c083a64283abfd83aed883a9" + + "1583ac5783aeea83ad7b83ae2683a99283a64183abca83ac8783aca783b38083b0b183a53183a42a83a7d383a69283a8" + + "e083ab2983aba083b1d283ad2883a4cd83a62183ab2c83a9fe83a478839d8983a4cb83aca483a393839ede83a2e483ab" + + "c383a92f83a46083a49983ad4c83ad5083a7a883a53683a59883aa4783a8db83a45583a47183a59383a6ae83ac7c83b0" + + "1d83ae9783ae7383aba783a80483aaf483b04c83af0c83a95083a77d83a61183a61b83a48383ad7183aed083a8bc0b43" + + "004300430043004300430043004300430043004300f8fff8fff8fff8fff8fff8fff8fff7fff7fff7fff7ff00313774b4" + + /** unix 1786031642 — numberOfLeadsOffSamples == 10, one slot unused. */ + private val hex10 = + "aa0128060100cde02f10037961cd001aae746aeb51030a000124000000ffff00f40183f47c83f61983f31b83f42a83f5" + + "6083f8bc83f3a683f10a83f15883f2d583f71683f45783f58783f26e83f18983efdb83efc483f51583f5e683f6f283f4" + + "7d83f59e83f76283f59883f66383f1d583f4d783fbbc83f59183f5a083f4f483f5c683f7ab83eed783ecbf83f19083f0" + + "5a83f1bd83efa383f30b83f48083f00483f37583f3dd83f1e283f0d983f1a983f2f183f0f283f07283ef9983ef5d83f2" + + "7c83f3ec83f39d83f2e083f01583eece83ed8b83f06883f10383f19683f42983f08b83f35783f30383edae83ec4083ee" + + "3583f05e83f16083f3f383ee9b83ed5083f03883ecec83ee8d83f31b83eee883eb4883e87883e0e283e9f483f0d183f0" + + "7f83effe83ed8983eef183e9dd83ea0b83ed9783ef3083eb8883e9ad83f03983f6d983f33c83ee9583eb6a83ee9483f3" + + "7f83f25483ee0083ee1283ebf383eda583ef0583f05683f3ab83ef1a83f2cb83f45583f48383f0c083eacf83ee5583f2" + + "9483f16e83ec5c83eaa683ecb583ecd283f0cf83f38983f39183e9c583e7dd83e9f783e8a983e41783e34b83e1cd83df" + + "1083dde183d17283d22283c94183bfe583c02483bd9d83bfa683c39483c36183bde583c4c783c7ae83d37d83de9783e3" + + "0683e62283e8bd83e78d83eb4983e8e683e99283eaac83ecac83ed1a83efe083efdc83ecdb83f35483f20983ef6783f2" + + "8183f56983f26183f40283f82583f44d83eb8d83eb0f83eb3283edc283f80f83fae583f5c183f4f383fc3783fae583f3" + + "1a83eeea83ff7883fe3583f5e883fac783ff5983f9ae83f60383f7b883fb8d83f9f083feb783fee18003788009ce8006" + + "f48002a38005f28006ad800522800cb5800f30800ca7800dda8009c68005f18004e180031180093e8010658010918009" + + "558005f98008da801274800e0e8002648004ce8003d9800b73800dd4800d2680133b800b908004e783f8dc83fabe8004" + + "a6800c9a80180c8016b280180b80170b801340801a21801d1e801947801acd801ba480177c8020b68022a2802cba802c" + + "ec802a6b8030688036da8039fd8040a68043f4804868804b7b804463804cec804fdd8058308055be8055618050d5804f" + + "85804ebc80556e8062e9805ce18058c1805908805632805eca806349805ee6805a13805b86805dd2806925806655805c" + + "4780587480620580629b8062358062a4806d0c8063c3805b65805f0d805e4e806b00806a8680656c805fbb805851805b" + + "4b8057f780596a8061a1806570805e02805e7f8063e58065d48061e680613e805f8b805e5e80579b80536c80543c8053" + + "e4805d83805aee80569880577f80567e805b73805074804fdd8050ec804f74805355804ef2804f46804f52804d718050" + + "1f804f9b8058a28050758044c5804ccd804f49804f3a804b0e804bd5804da380451580467a804b6a804961804959804a" + + "26804c118041fc8041458042e38042918045b7804de88047b28044e68041498035ff80359d8038a3803b81803de78042" + + "8a803b1c8033b68034c6803919803992802fc880334e8036a780350c80397b8041d1803998803d6c8032a68025428033" + + "74803db9803761803f7980386c802dc0802f2c8035b08033b8803496802935801bac8021d6802e9b802d30802fbe802b" + + "eb803439802d85802bfe802eb0802fb4802f938030ec802bd0802932802c4680339f802ef5802b368024cc8025e5802a" + + "ba802a29802b80802f3780362c8034b38031e28033e2802eda80261b8022908024cd8029438023448027d3802e948027" + + "5a8024088022928029038033dc80330f80290b802fbe8023858016cb8021fe80274f802304801ed68022ff802df38031" + + "4380318a802c9680240780264a802766802a47802fa8802d808029958028d0801eeb8017d8801f00802854802ab28034" + + "3b802ff78022b2801e018020fd8029f28023ac8023208019a68019c08021cc8028858025ed801afc800c5b801e09801f" + + "fa801df4801df08016aa8012598017e2801c9480233a801f6f801d2f8011d080080e8014a2801d018022a98016218005" + + "0b800589800dde8019cf8020168020a2800e2b8003ac800ba6801956801838800ebe800c68800d218009ee800e4b0a43" + + "004300430043004300420042004200420042000000f7fff7fff7fff7fff7fff7fff6fff6fff6fff6ff000000fe104255" + + /** + * unix 1786031625 — the capture's one PARTLY FILLED record, numberOfECGSamples == 245. + * + * The first record of the second reading: the empty record before it is at 1786031624 and the first + * full 500-sample one at 1786031626, so the strap began the reading part-way into this record's + * second and wrote 245 of the region's 500 slots. Same 1584-byte frame, same leads-off offsets. + */ + private val hexPartial245 = + "aa0128060100cde02f10036861cd0009ae746aeb51010a000100000000ffff00f50083ae1e8259e48210008210008210" + + "008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210" + + "008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210" + + "008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210008210" + + "00821000821000821000821000821000821000821000821000821000821000821000821000c21000c21000c21000c210" + + "00c21085c25696c328a5c39fc1c3cda0c3e169c3e9d9c3ecc2c3ec8bc3ec7dc3efd1c3f2d9c3f434c3f49ac3f4c6c3f5" + + "b0c3f6f2c3f847c3f716c3f7dac3f776c3f66ec3f67fc3f8a3c3f980c3f8cac3f79dc3f85dc3f839c3f747c3f8b7c3f8" + + "fac3f829c3f964c3f923c3f7cec3fa53c3f88ac3f78dc3f8c2c3f8f0c3f73fc3f763c3f867c3f7d5c3f816c3f99bc3f7" + + "dfc3f7f2c3f8fbc3f830c3f777c3f800c3f7b3c3f864c3f6fdc3f79ec3f882c3f77ac3f748c3f88ec3f907c3f980c3f7" + + "f4c3f924c3f8c8c3f856c3f802c3f823c3f903c3f90fc3f950c3fab5c3fa82c3f98ec3f91ac3f8bdc3fb13c3fb7ec3fb" + + "3fc3fbd5c3fa08c3faf3c3fc74c3fa42c3fa90c3fbd5c3fc54c3fc01c3fcf6c3fb4dc3fc34c3fd5ac3fcccc3fdfac3fd" + + "40c3fbacc3fb30c3fc65c3fbf4c3fb99c3fb7dc3fbc4c3fc6fc3fcf4c3fbadc3fbcec3fb46c3fb58c3fc58c3fb0bc3f9" + + "45c3fa54c3fa75c3fa37c3fab2c3fa6cc3faf0c3fab2c3fad5c3fb46c3fb1ac3fc9dc3fbd2c3faa7c3fa98c3fb85c3fd" + + "b0c3fc78c3fb95c3fab7c3fc94c3fe67c3fe15c3fcffc3fa3bc3fd33c3ff86c00062c001dac0013cc3ff22c3fefcc3ff" + + "78c3fef1c00077c00148c000a1c00088c0011dc00263c0017ec000d6c002d9c0023cc001c3c00287c00322c00340c003" + + "18c00460c002eec003e1c0059bc00561c00479c0056cc00640c005e2c00670c0063bc006444005d440043a4002eb4002" + + "a10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a45" + + "004400450045004500450045004500460047000000f9fff9fffafffafffafffafffafffafffbfffcff0000009b973d34" + + private fun bytes(h: String): ByteArray = + ByteArray(h.length / 2) { ((h[it * 2].digitToInt(16) shl 4) or h[it * 2 + 1].digitToInt(16)).toByte() } + + private val frame11 get() = bytes(hex11) + private val frame10 get() = bytes(hex10) + private val framePartial get() = bytes(hexPartial245) + + /** 24-bit big-endian field, low 18 bits signed. Measured from these bytes; not a vendor fact. */ + private fun sample18(blob: List, i: Int): Int { + val o = i * 3 + var v = ((blob[o] and 0x03) shl 16) or (blob[o + 1] shl 8) or blob[o + 2] + if (v and 0x20000 != 0) v -= 0x40000 + return v + } + + /** + * Regression: a populated real record is NOT discarded, and resolves to width 3. Under the packed + * reading the count=10 record left a 5-byte remainder against DEFAULT_MAX_PADDING and was rejected + * outright — 227 of the capture's 351 populated records. + */ + @Test + fun populatedRealRecordIsNotDiscarded() { + for (frame in listOf(frame11, frame10)) { + assertEquals(1584, frame.size) + assertEquals(47, frame[8].toInt() and 0xFF) + assertEquals(16, frame[9].toInt() and 0xFF) + val payload = Whoop5Ecg.innerPayload(frame, ecgPayloadStart) + assertNotNull("real capture must pass both CRCs", payload) + assertEquals(1563, payload!!.size) + assertEquals(listOf(3), Whoop5Ecg.rawBytesPerSampleCandidates(payload)) + val packet = Whoop5Ecg.decodeRaw(payload) + assertNotNull(packet) + assertEquals(500, packet!!.header.numberOfECGSamples) + assertEquals(1500, packet.rawECGDataRaw.size) + assertEquals(3, packet.bytesPerSample) + assertEquals(listOf(0x00), packet.padding) + } + } + + /** The full block: eleven valid I and eleven valid Q. */ + @Test + fun countElevenRecordDecodesElevenOfEach() { + val payload = Whoop5Ecg.innerPayload(frame11, ecgPayloadStart)!! + val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 3)!! + assertEquals(11, packet.numberOfLeadsOffSamples) + assertEquals(List(11) { 67 }, packet.leadsOffIRaw) + assertEquals(List(7) { 65528 } + List(4) { 65527 }, packet.leadsOffQRaw) + assertEquals( + listOf(-46361, -46615, -47659, -47357, -45997, -47084, -46826, -46579, -47218, -45762, -44790, -44613), + (0 until 12).map { sample18(packet.rawECGDataRaw, it) }, + ) + } + + /** + * The partially filled block: ten valid values off the FRONT of each fixed array. The packed reading + * started Q two bytes early, gaining a spurious leading 0 and losing the tenth real Q value. + */ + @Test + fun countTenRecordDecodesTenOfEachWithQAligned() { + val payload = Whoop5Ecg.innerPayload(frame10, ecgPayloadStart)!! + val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 3)!! + assertEquals(10, packet.numberOfLeadsOffSamples) + assertEquals(listOf(67, 67, 67, 67, 67, 66, 66, 66, 66, 66), packet.leadsOffIRaw) + assertEquals( + listOf(65527, 65527, 65527, 65527, 65527, 65527, 65526, 65526, 65526, 65526), + packet.leadsOffQRaw, + ) + assertEquals(65526, packet.leadsOffQRaw.last()) // previously lost into the "padding" + assertEquals(listOf(0x00), packet.padding) + assertEquals( + listOf(-2948, -2535, -3301, -3030, -2720, -1860, -3162, -3830, -3752, -3371, -2282, -2985), + (0 until 12).map { sample18(packet.rawECGDataRaw, it) }, + ) + // 17 + 1500 + 0 + 1 + 22 + 22 + 1 = 1563, with the block charged at its FIXED size. + assertEquals(0, packet.unusedSampleBytes) + val accounted = Whoop5Ecg.HEADER_LENGTH + packet.rawECGDataRaw.size + packet.unusedSampleBytes + + 1 + Whoop5Ecg.LEADS_OFF_SLOT_COUNT * 2 + Whoop5Ecg.LEADS_OFF_SLOT_COUNT * 2 + + packet.padding.size + assertEquals(payload.size, accounted) + } + + /** + * The SAMPLE region is fixed-size too. The partly-filled record is byte-for-byte the same size as a + * full one, its sample data stops at frame[769] = 34 + 245 * 3, everything from there to the + * leads-off block is zero, and the block is at the SAME frame offsets as both full records'. Reading + * the region as exactly `n * bytesPerSample` landed the count byte on a zero at payload[752], read an + * empty block and left 766 trailing bytes — the last populated record the decoder discarded. + */ + @Test + fun partlyFilledRecordDecodesAtTheStreamWidth() { + assertEquals(1584, framePartial.size) + assertTrue((769 until 1534).all { framePartial[it].toInt() == 0 }) + assertEquals(10, framePartial[1534].toInt() and 0xFF) + + val payload = Whoop5Ecg.innerPayload(framePartial, ecgPayloadStart)!! + assertEquals(1563, payload.size) + assertEquals(0, payload[752]) + + val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 3)!! + assertEquals(245, packet.header.numberOfECGSamples) + assertEquals(735, packet.rawECGDataRaw.size) // only the VALID slots are carried + assertEquals(3, packet.bytesPerSample) + assertEquals(765, packet.unusedSampleBytes) + assertEquals(1500, packet.sampleRegionBytes) // the region every full record carries + assertEquals(10, packet.numberOfLeadsOffSamples) + assertEquals(listOf(69, 68, 69, 69, 69, 69, 69, 69, 70, 71), packet.leadsOffIRaw) + assertEquals( + listOf(65529, 65529, 65530, 65530, 65530, 65530, 65530, 65530, 65531, 65532), + packet.leadsOffQRaw, + ) + assertEquals(listOf(0x00), packet.padding) + // The blob's ends, verbatim. No encoding claim is made for this record: it is the first half + // second of a reading and its samples are still settling. + assertEquals(listOf(0x83, 0xAE, 0x1E), packet.rawECGDataRaw.take(3)) + assertEquals(listOf(0x40, 0x02, 0xA1), packet.rawECGDataRaw.takeLast(3)) + + val accounted = Whoop5Ecg.HEADER_LENGTH + packet.rawECGDataRaw.size + packet.unusedSampleBytes + + Whoop5Ecg.LEADS_OFF_BLOCK_LENGTH + packet.padding.size + assertEquals(payload.size, accounted) // 17 + 735 + 765 + 45 + 1 + } + + /** + * A partly-filled record does NOT determine its own sample WIDTH, and the enumerator says so rather + * than picking one: 245 samples inside a 1500-byte region fits 3 bytes per sample and 4 alike. The + * full records are where the width comes from — they resolve unaided (see the test above). + */ + @Test + fun partlyFilledRecordReportsItsWidthAsAmbiguous() { + val payload = Whoop5Ecg.innerPayload(framePartial, ecgPayloadStart)!! + assertEquals(listOf(3, 4), Whoop5Ecg.rawBytesPerSampleCandidates(payload)) + assertNull(Whoop5Ecg.decodeRaw(payload)) + // 1 and 2 are ruled out by real sample bytes sitting in what they would call unused capacity. + assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 1)) + assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2)) + } + + /** + * The block's geometry, read straight off the frame with no decoder involved: the count byte at + * frame[1534], I slots at frame[1535], Q slots at frame[1557] — the SAME offsets under both counts, + * which is what identifies the block as fixed rather than packed. + */ + @Test + fun leadsOffBlockIsAtTheSameFixedOffsetsUnderBothCounts() { + assertEquals(11, Whoop5Ecg.LEADS_OFF_SLOT_COUNT) + assertEquals(11, frame11[1534].toInt() and 0xFF) + assertEquals(10, frame10[1534].toInt() and 0xFF) + + fun slots(frame: ByteArray, start: Int) = (0 until 11).map { + (frame[start + it * 2].toInt() and 0xFF) or ((frame[start + it * 2 + 1].toInt() and 0xFF) shl 8) + } + assertEquals(List(11) { 67 }, slots(frame11, 1535)) + assertEquals(List(7) { 65528 } + List(4) { 65527 }, slots(frame11, 1557)) + assertEquals(listOf(67, 67, 67, 67, 67, 66, 66, 66, 66, 66, 0), slots(frame10, 1535)) + assertEquals( + listOf(65527, 65527, 65527, 65527, 65527, 65527, 65526, 65526, 65526, 65526, 0), + slots(frame10, 1557), + ) + assertEquals(frame10.size, 1579 + 1 + 4) + } +} diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5EcgTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5EcgTest.kt index c8961c0033..2cbaa41a08 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5EcgTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5EcgTest.kt @@ -49,6 +49,18 @@ class Whoop5EcgTest { private fun u16le(values: List): List = values.flatMap { listOf(it and 0xFF, (it shr 8) and 0xFF) } + /** + * The raw record's leads-off tail: the count byte, then [Whoop5Ecg.LEADS_OFF_SLOT_COUNT] i16 I slots, + * then the same again for Q. The block is FIXED-SIZE on real hardware (see the Swift + * `Whoop5EcgRawHardwareTests`), so these fixtures zero-fill the slots the count does not reach instead + * of packing the arrays to the count. + */ + private fun leadsOffBlock(i: List, q: List): List { + require(i.size == q.size && i.size <= Whoop5Ecg.LEADS_OFF_SLOT_COUNT) + val pad = List(Whoop5Ecg.LEADS_OFF_SLOT_COUNT - i.size) { 0 } + return listOf(i.size) + u16le(i + pad) + u16le(q + pad) + } + private fun puffinFrame(type: Int, payload: List): ByteArray = Framing.puffinCommandFrame( cmd = 0x00, seq = 0x01, @@ -210,7 +222,7 @@ class Whoop5EcgTest { val rawBlob = (0 until 12).toList() // 4 samples × 3 bytes val leadsOffI = listOf(1, 2) val leadsOffQ = listOf(3, 4) - val payload = header(samples = 4) + rawBlob + listOf(2) + u16le(leadsOffI) + u16le(leadsOffQ) + val payload = header(samples = 4) + rawBlob + leadsOffBlock(leadsOffI, leadsOffQ) val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 3) assertNotNull(packet) @@ -222,14 +234,70 @@ class Whoop5EcgTest { assertEquals(3, packet?.bytesPerSample) // count ÷ numberOfECGSamples } + /** + * The SAMPLE region is fixed-size too: numberOfECGSamples says how many of its slots are valid, and + * the rest is zero-filled capacity that still has to be stepped over to reach the leads-off block. + * Only the valid bytes are carried; the unused ones are counted. Twin of the Swift + * rawPartlyFilledSampleRegionIsSteppedOverNotCarried. + */ + @Test + fun rawPartlyFilledSampleRegionIsSteppedOverNotCarried() { + val samples = listOf(1, 2, 3, 4, 5, 6) // 3 valid samples × 2 bytes + val unused = List(8) { 0 } // 4 more slots the record did not fill + val payload = header(samples = 3) + samples + unused + leadsOffBlock(listOf(5), listOf(6)) + + val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2) + assertEquals(samples, packet?.rawECGDataRaw) + assertEquals(8, packet?.unusedSampleBytes) + assertEquals(14, packet?.sampleRegionBytes) + assertEquals(2, packet?.bytesPerSample) + assertEquals(1, packet?.numberOfLeadsOffSamples) + assertEquals(listOf(5), packet?.leadsOffIRaw) + assertEquals(listOf(6), packet?.leadsOffQRaw) + assertEquals(emptyList(), packet?.padding) + } + + /** + * The zero-fill is what makes the step-over safe: a non-zero byte between the declared samples and + * the block means the block is not where this width says it is, so the decode fails closed rather + * than skipping over bytes that might be data. The stray byte LEADS the unused span so it stays + * inside that span at every end position the padding budget allows. + */ + @Test + fun rawRejectsNonZeroBytesInTheUnusedSampleRegion() { + val region = listOf(1, 2, 3, 4, 5, 6) + listOf(0x09) + List(7) { 0 } + val payload = header(samples = 3) + region + leadsOffBlock(listOf(5), listOf(6)) + assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2)) + // With the same bytes declared as samples the record is fine — it is the SKIPPING that is gated. + val full = header(samples = 7) + region + leadsOffBlock(listOf(5), listOf(6)) + assertEquals(region, Whoop5Ecg.decodeRaw(full, bytesPerSample = 2)?.rawECGDataRaw) + } + + /** A zero count still carries the full fixed block — the slots are simply all unused. */ @Test fun rawWithNoLeadsOffSamples() { - val payload = header(samples = 2) + listOf(0xAA, 0xBB, 0xCC, 0xDD) + listOf(0) + val payload = header(samples = 2) + listOf(0xAA, 0xBB, 0xCC, 0xDD) + leadsOffBlock(emptyList(), emptyList()) val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2) assertEquals(0, packet?.numberOfLeadsOffSamples) assertEquals(emptyList(), packet?.leadsOffIRaw) assertEquals(emptyList(), packet?.leadsOffQRaw) assertEquals(listOf(0xAA, 0xBB, 0xCC, 0xDD), packet?.rawECGDataRaw) + assertEquals(emptyList(), packet?.padding) + } + + /** + * The valid values come off the FRONT of each fixed array, and the unused slots are dropped rather + * than shifting Q or leaking into the padding — the defect the real capture exposed. + */ + @Test + fun rawPartiallyFilledLeadsOffBlockKeepsQAligned() { + val payload = header(samples = 2) + listOf(0, 0, 0, 0) + + leadsOffBlock(listOf(11, 12, 13), listOf(21, 22, 23)) + val packet = Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2) + assertEquals(3, packet?.numberOfLeadsOffSamples) + assertEquals(listOf(11, 12, 13), packet?.leadsOffIRaw) + assertEquals(listOf(21, 22, 23), packet?.leadsOffQRaw) + assertEquals(emptyList(), packet?.padding) } @Test @@ -238,6 +306,19 @@ class Whoop5EcgTest { assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2)) } + /** + * The block holds a fixed number of slots, so a count above it cannot describe this layout and must + * fail closed rather than read past the block. + */ + @Test + fun rawRejectsLeadsOffCountAboveTheBlockCapacity() { + // The blob is exactly full, so the block can only be where the count byte is — this pins the + // capacity check itself rather than letting the end-of-record search reject the buffer earlier. + val payload = header(samples = 2) + listOf(0, 0, 0, 0) + + listOf(Whoop5Ecg.LEADS_OFF_SLOT_COUNT + 1) + List(Whoop5Ecg.LEADS_OFF_SLOT_COUNT * 4) { 0 } + assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 2)) + } + @Test fun rawRejectsMissingLeadsOffCountByte() { val payload = header(samples = 2) + listOf(0, 0, 0, 0) @@ -259,13 +340,13 @@ class Whoop5EcgTest { @Test fun rawRejectsShortHeaderAndZeroWidth() { assertNull(Whoop5Ecg.decodeRaw(listOf(1, 2, 3), bytesPerSample = 2)) - val payload = header(samples = 2) + listOf(0, 0, 0, 0) + listOf(0) + val payload = header(samples = 2) + listOf(0, 0, 0, 0) + leadsOffBlock(emptyList(), emptyList()) assertNull(Whoop5Ecg.decodeRaw(payload, bytesPerSample = 0)) } @Test fun rawSampleWidthCandidatesAreEnumeratedNotGuessed() { - val payload = header(samples = 4) + List(8) { 0x11 } + listOf(1) + u16le(listOf(7)) + u16le(listOf(8)) + val payload = header(samples = 4) + List(8) { 0x11 } + leadsOffBlock(listOf(7), listOf(8)) val candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload) assertTrue("width 2 must be structurally admissible", candidates.contains(2)) if (candidates.size == 1) { @@ -277,10 +358,12 @@ class Whoop5EcgTest { @Test fun rawAmbiguousBufferRefusesToDecode() { - val payload = header(samples = 1) + List(6) { 0 } - val candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload, maxPadding = 8) - assertTrue("fixture is meant to be ambiguous", candidates.size > 1) - assertNull(Whoop5Ecg.decodeRaw(payload, maxPadding = 8)) + // With a single sample, an all-zero tail long enough to hold the fixed leads-off block leaves + // every width inside the padding tolerance. + val payload = header(samples = 1) + List(49) { 0 } + val candidates = Whoop5Ecg.rawBytesPerSampleCandidates(payload) + assertEquals("fixture is meant to be ambiguous", listOf(1, 2, 3, 4), candidates) + assertNull(Whoop5Ecg.decodeRaw(payload)) } // MARK: - Frame level (CRC gating) @@ -328,7 +411,7 @@ class Whoop5EcgTest { @Test fun rawFrameDecodesThroughAValidPuffinEnvelope() { - val payload = header(samples = 2) + listOf(1, 2, 3, 4) + listOf(1) + u16le(listOf(5)) + u16le(listOf(6)) + val payload = header(samples = 2) + listOf(1, 2, 3, 4) + leadsOffBlock(listOf(5), listOf(6)) val frame = puffinFrame(0x2F, payload) val packet = Whoop5Ecg.decodeRawFrame(frame, bytesPerSample = 2) assertEquals(listOf(1, 2, 3, 4), packet?.rawECGDataRaw) @@ -338,7 +421,7 @@ class Whoop5EcgTest { @Test fun rawFrameRejectsBadCrc() { - val payload = header(samples = 2) + listOf(1, 2, 3, 4) + listOf(0) + val payload = header(samples = 2) + listOf(1, 2, 3, 4) + leadsOffBlock(emptyList(), emptyList()) val frame = puffinFrame(0x2F, payload) frame[frame.size - 2] = (frame[frame.size - 2].toInt() xor 0xFF).toByte() assertNull(Whoop5Ecg.decodeRawFrame(frame, bytesPerSample = 2)) @@ -376,6 +459,100 @@ class Whoop5EcgTest { assertFalse(Whoop5Ecg.plausibleFilteredPayload(List(64) { 0 })) } + // MARK: - Structural triage: sample width + + /** + * THE DEFECT this widening fixes. The triage hardcoded `n * 2`, so a payload whose samples are 3 bytes + * wide missed the length agreement, failed the padding budget, and was discarded without a trace — + * while the probe truthfully reported that no frame passed. A populated raw flash record read off + * hardware was 1500 bytes against numberOfECGSamples = 500, i.e. 3. + */ + @Test + fun triageAcceptsAThreeBytePerSampleBufferAndNamesTheWidth() { + val payload = header(samples = 4) + List(12) { 0x11 } + assertEquals(listOf(3), Whoop5Ecg.filteredBytesPerSampleCandidates(payload)) + assertTrue(Whoop5Ecg.plausibleFilteredPayload(payload)) + } + + @Test + fun triageStillAcceptsTwoBytesPerSampleAndReportsItFirst() { + val payload = header(samples = 3) + i16le(listOf(1, 2, 3)) + assertEquals(listOf(2), Whoop5Ecg.filteredBytesPerSampleCandidates(payload)) + assertTrue(Whoop5Ecg.plausibleFilteredPayload(payload)) + assertEquals(2, Whoop5Ecg.FILTERED_WIDTH_CANDIDATES.first()) + } + + @Test + fun triageAcceptsFourBytesPerSample() { + assertEquals( + listOf(4), + Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 5) + List(20) { 0x22 }), + ) + } + + /** An ambiguous buffer REPORTS every width it admits rather than picking one. */ + @Test + fun triageReportsEveryWidthAnAmbiguousBufferAdmits() { + assertEquals( + listOf(3, 4), + Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 2) + List(8) { 0x33 }), + ) + } + + /** Only the WIDTH assumption moved: every other guard still rejects a length-perfect buffer. */ + @Test + fun wideningTheWidthDoesNotLoosenAnyOtherGuard() { + val body = List(12) { 0x11 } + assertEquals(listOf(3), Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 4) + body)) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(started = 7, samples = 4) + body).isEmpty()) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(running = 2, samples = 4) + body).isEmpty()) + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates(header(stoppedAndComplete = 9, samples = 4) + body).isEmpty() + ) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(leadsOn = 3, samples = 4) + body).isEmpty()) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(signalQuality = 9, samples = 4) + body).isEmpty()) + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates(header(arrhythmiaResult = 9, samples = 4) + body).isEmpty() + ) + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates(header(arrhythmiaStatus = 9, samples = 4) + body).isEmpty() + ) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 0)).isEmpty()) + } + + @Test + fun triageStillRejectsBuffersNoWidthExplains() { + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 7) + i16le(listOf(1, 2, 3))).isEmpty() + ) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(header(samples = 40) + i16le(listOf(1, 2))).isEmpty()) + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates( + header(samples = 1) + i16le(listOf(1)) + List(40) { 0 } + ).isEmpty() + ) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(emptyList()).isEmpty()) + assertFalse(Whoop5Ecg.plausibleFilteredPayload(header(samples = 7) + i16le(listOf(1, 2, 3)))) + } + + /** The offset math runs in Long: an Int overflow wraps NEGATIVE and would slip past the bounds test. */ + @Test + fun triageWidthMathCannotOverflow() { + val payload = header(samples = 65_535) + List(8) { 0 } + assertTrue( + Whoop5Ecg.filteredBytesPerSampleCandidates(payload, widths = listOf(Int.MAX_VALUE, 2, 3)).isEmpty() + ) + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(payload, widths = listOf(0, -1)).isEmpty()) + } + + @Test + fun triageWidthsAreReportedThroughACrcGatedFrameToo() { + val frame = puffinFrame(0x28, header(samples = 4) + List(12) { 0x11 }) + assertEquals(listOf(3), Whoop5Ecg.filteredBytesPerSampleCandidates(frame)) + frame[frame.size - 1] = (frame[frame.size - 1].toInt() xor 0xFF).toByte() + assertTrue(Whoop5Ecg.filteredBytesPerSampleCandidates(frame).isEmpty()) + } + // MARK: - Commands @Test diff --git a/android/app/src/test/resources/schema_oracle.json b/android/app/src/test/resources/schema_oracle.json index 72081fadf5..585c4e13ef 100644 --- a/android/app/src/test/resources/schema_oracle.json +++ b/android/app/src/test/resources/schema_oracle.json @@ -1,6 +1,6 @@ { - "_readme": "SHARED Room<->GRDB SCHEMA ORACLE (#775). Two byte-identical copies: Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json and android/app/src/test/resources/schema_oracle.json. SchemaOracleTests.swift compares GRDB's PRAGMA table_info/index_list against it; SchemaOracleTest.kt compares Room's exported schema JSON against it. `columns` is the iOS/GRDB shape in GRDB column order (macOS is the reference implementation); every way Android differs is spelled out in an `android` / `iosAbsent` / `androidColumnOrder` override naming a key in `divergenceReasons`. Adding a column, reordering one, or changing a type/nullability on one platform only fails both suites until it is either fixed or written down here.", - "roomVersion": 25, + "_readme": "SHARED Room<->GRDB SCHEMA ORACLE (#775). Two byte-identical copies: Packages/WhoopStore/Tests/WhoopStoreTests/Resources/schema_oracle.json and android/app/src/test/resources/schema_oracle.json. SchemaOracleTests.swift compares GRDB's PRAGMA table_info/index_list against it; SchemaOracleTest.kt compares Room's exported schema JSON against it. `columns` is the iOS/GRDB shape in GRDB column order (macOS is the reference implementation); every way Android differs is spelled out in an `android` / `iosAbsent` / `androidColumnOrder` override naming a key in `divergenceReasons`. Adding a column, reordering one, or changing a type/nullability on one platform only fails both suites until it is either fixed or written down here. `grdbMigrationLineages` is the same idea for migration IDENTIFIERS: a fork that numbered its own migrations before upstream reached those numbers declares them there instead of renumbering \u2014 GRDB keys migrations by identifier string, so renaming one wedges a database that already ran it.", + "roomVersion": 26, "grdbMigrations": [ "v1", "v2", @@ -27,15 +27,38 @@ "v23-daily-spo2-raw", "v24-rr-seq", "v25-oura-raw", + "v26-cloud-tombstone", + "v27-apple-step-hour", "v26-efficiency-heal", "v27-ppg-waveform", + "v28-phone-timezone", + "v29-daily-avg-sdnn", "v28-raw-imu", "v29-score-input-provenance", "v30-rr-ord", - "v31-deep-capture-channels" + "v31-deep-capture-channels", + "v34-spo2-pct-durable" ], + "grdbMigrationLineages": { + "fork-ios-only": { + "reason": "This fork (vishk23/noop) carries four iOS-only migrations that were numbered from v26 before upstream's own v26..v29 existed, so both lineages claim v26, v27, v28 and v29. They are NOT renumbered to sit after upstream's: GRDB keys migrations by identifier STRING, so renaming one makes it read as un-applied on a device that already ran it, and the migrator then re-runs its body against a schema that already has the table \u2014 wedging a live database. The identifiers are frozen exactly as first shipped. What still carries the vN <-> Room-version mapping is the BASELINE lineage (grdbMigrations minus every lineage declared here), which both suites still check is exactly v1..vN with no gaps and no repeats; a lineage's own vN must strictly increase; and full-identifier uniqueness is enforced across everything, because a genuinely duplicated identifier makes GRDB silently skip the second body.", + "migrations": [ + "v26-cloud-tombstone", + "v27-apple-step-hour", + "v28-phone-timezone", + "v29-daily-avg-sdnn" + ] + }, + "fork-ahead-of-upstream": { + "reason": "This fork numbers a migration AHEAD of upstream on purpose, which is the mirror image of the `fork-ios-only` case and needs its own entry rather than being folded into it. Local main carries migrations through v31; upstream ryanbr/noop already holds `v32-rr-src-channel` and `v33-workout-steps`, which this branch has not merged yet. Numbering this one v32 would therefore collide with an upstream identifier the very next time the fork merges upstream \u2014 and because GRDB keys migrations by identifier STRING, that collision is not a merge conflict a human sees: two different bodies claiming the same name means GRDB silently SKIPS the second, so one of the two tables is never created and the failure surfaces later as a missing-table error on a real device. Starting at v34 leaves upstream's v32/v33 free to arrive untouched. It is declared here rather than renumbered because the baseline lineage must stay exactly v1..vN, and because once this identifier has run on any device renaming it re-runs its body against a schema that already has the table.", + "migrations": [ + "v34-spo2-pct-durable" + ] + } + }, "divergenceReasons": { "android-orphan-synced-column": "REAL COLUMN DRIFT: `synced` exists on Android only. GRDB's v10 note is explicit that stepSample gets no `synced` column ('unused; see StreamStore'), and v12's ppgHrSample never had one either; the Room entities copied the flag from the older per-second entities. Nothing reads it on Android \u2014 the per-row upload flag belongs to an upload path that does not exist there \u2014 so it is dead width, not divergent data. Removing it needs a Room table rebuild.", + "avg-sdnn-ios-only": "FORK-ONLY COLUMN: `dailyMetric.avgSdnn` is appended by the fork's GRDB `v29-daily-avg-sdnn` \u2014 the 5-minute SDNN index, the broad-variability twin of `avgHrv` (RMSSD), window-matched to a watch's short SDNN rather than the drift-inflated whole-night SD, and computed by `HRVAnalyzer.sdnnIndex` during the nightly analysis. Additive and nullable, so it costs an existing row nothing. Room has no field for it; giving it one means a @Database version bump plus a migration, so the absence is recorded here rather than left to read as unexplained drift.", "battery-alter-append-order": "REAL COLUMN-ORDER DRIFT (CLAUDE.md: 'column order in a Room CREATE TABLE must match the entity field order'). GRDB appended `synced` (v5) then `charging` (v6) with ALTER TABLE, which can only append; the Room entity declares `charging` before `synced`. Same columns, same types, different positions \u2014 so any positional copy (INSERT INTO battery SELECT * FROM \u2026) swaps them. Fixing it means reordering the Room entity's fields, which changes Room's identity hash and therefore needs a version bump plus a no-op migration.", "grdb-boolean-affinity": "GRDB's .boolean column type is declared BOOLEAN, which SQLite gives NUMERIC affinity; Room maps Kotlin Boolean to INTEGER. For the 0/1 values both sides write, NUMERIC and INTEGER affinity store bit-identical INTEGERs, so a .noopbak round-trips. Closing it means a GRDB table rebuild.", "paired-device-alter-append-order": "REAL COLUMN-ORDER DRIFT, same shape as battery. GRDB's v16-paired-device-peripheral appended `peripheralId` at the end; the Room entity declares it fifth, between `nickname` and `sourceKind`.", @@ -115,6 +138,35 @@ ], "indices": [] }, + "appleStepHour": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v27-apple-step-hour`): hourly Apple Health step counts. The daily HealthKit collect path flattens a whole day into one `appleDaily.steps` total, so hours the phone was dead or absent read as low steps for the entire day instead of as the specific hours with no recording. Android has no HealthKit source, so no Room twin.", + "columns": [ + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "ts", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "steps", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "deviceId", + "ts" + ], + "indices": [] + }, "battery": { "platform": "both", "columns": [ @@ -178,6 +230,85 @@ "androidColumnOrderDivergence": "battery-alter-append-order", "indices": [] }, + "cloudTombstone": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v26-cloud-tombstone`): records deletions accepted from the NOOP cloud journal, keyed by (kind, editSeq) for idempotency, so a later re-import cannot resurrect a row the user deleted. Backs Strand/CloudSync/, which has no Android counterpart.", + "columns": [ + { + "name": "id", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "kind", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "startTs", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "endTs", + "affinity": "INTEGER", + "notNull": false, + "default": null + }, + { + "name": "sport", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "day", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "key", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "editSeq", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "id" + ], + "indices": [ + { + "name": "idx_cloudTombstone_kind_device", + "unique": false, + "columns": [ + "kind", + "deviceId" + ] + } + ] + }, "cursors": { "platform": "ios_only", "reason": "iOS offload/upload bookmarks (GRDB v2). Android keeps the equivalent bookmarks in SharedPreferences, not in Room.", @@ -322,6 +453,14 @@ "affinity": "INTEGER", "notNull": false, "default": null + }, + { + "name": "avgSdnn", + "affinity": "REAL", + "notNull": false, + "default": null, + "androidAbsent": true, + "divergence": "avg-sdnn-ios-only" } ], "primaryKey": [ @@ -1054,6 +1193,28 @@ "androidColumnOrderDivergence": "paired-device-alter-append-order", "indices": [] }, + "phoneTimezone": { + "platform": "ios_only", + "reason": "Fork-only iOS table (GRDB `v28-phone-timezone`): the IANA timezone the phone was in for each local calendar day, so day-bucketed metrics stay correct across travel. Android has no twin.", + "columns": [ + { + "name": "day", + "affinity": "TEXT", + "notNull": false, + "default": null + }, + { + "name": "tzId", + "affinity": "TEXT", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "day" + ], + "indices": [] + }, "ppgHrSample": { "platform": "both", "columns": [ @@ -1537,6 +1698,34 @@ ], "indices": [] }, + "spo2PctSample": { + "platform": "both", + "columns": [ + { + "name": "deviceId", + "affinity": "TEXT", + "notNull": true, + "default": null + }, + { + "name": "ts", + "affinity": "INTEGER", + "notNull": true, + "default": null + }, + { + "name": "pct", + "affinity": "INTEGER", + "notNull": true, + "default": null + } + ], + "primaryKey": [ + "deviceId", + "ts" + ], + "indices": [] + }, "spo2Sample": { "platform": "both", "columns": [ diff --git a/docs/ANALYTICS.md b/docs/ANALYTICS.md index 81e7d8a821..2f695fd348 100644 --- a/docs/ANALYTICS.md +++ b/docs/ANALYTICS.md @@ -297,6 +297,7 @@ The rule: **elevated HR alone is insufficient to call wake.** An epoch or run at - **`adaptiveOvernightHRBaseline`.** A personalised sleep band derived from recent overnight medians (self-calibrating across a supplement/fitness era), with a floor. Threaded through `detectSleep` as an optional argument that defaults to `nil` (byte-identical when unset); live cross-night wiring in `IntelligenceEngine` is a follow-up. Source: `SleepStager.swift` (`confirmSleepWithHR`, `adaptiveOvernightHRBaseline`) and `SleepStagerV2.swift` (motion-quiescent clamp), both in `Packages/StrandAnalytics`. Filed upstream as [ryanbr/noop#462](https://github.com/ryanbr/noop/issues/462). The Kotlin analytics twin (`com.noop.analytics`) is a deliberate follow-up (Swift-only contributor) — the parity contract requires the two stagers to stay byte-identical once transcribed. + ### Displayed sleep onset — the headline "Asleep at" spans the whole bridged night The Sleep screen headline ("Asleep at …") reports the onset of the **whole bridged night**, not the main session's start. A night stored as a short first-sleep fragment + a brief walk + the main session bridges into one group when the gap is under `gapBridgeMaxMin` (60 min). The display onset walk (`SleepView.nightOnsetTs` → `isPreOnsetAwakeStub`) previously mis-classified such a fragment as a spurious pre-onset lead through two stacked defects: (1) the #259 relative "minor lead" test compared the fragment's asleep minutes against 15% of the main block, so on a long main sleep a genuine short first sleep was skipped and the headline jumped forward to the main session's start; (2) the stub test read asleep minutes via the dict-only `decodeStages`, which returns nil for the segment-array `stagesJSON` an on-device **computed** night stores — so every fragment counted as 0 asleep minutes and tripped the "essentially sleepless stub" branch, bypassing defect (1)'s floor entirely. diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 7c8e747bab..0da5b3f187 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -509,6 +509,45 @@ too, not left behind. --- +## Phone timezone archive + +Every timestamp in the DB is **epoch-UTC**, and nothing else records which zone the phone was in +when a day was lived, so any wall-clock reading of a historical night (bedtime, "3am", day +boundaries) is guesswork once the wearer crosses zones (Pacific ↔ Eastern) — a hard-coded-offset +assumption already corrupted an analysis pass. This table stamps each **local calendar day** with +the phone's IANA timezone so downstream analysis can localise that specific day's samples. + +### `phoneTimezone` *(v28)* + +One row per local calendar day. Written by `PhoneTimezoneStore.upsertPhoneTimezone(day:tzId:)` +(`Packages/WhoopStore/Sources/WhoopStore/PhoneTimezoneStore.swift`), stamped from +`TimeZone.current.identifier` by `IntelligenceEngine.analyzeRecent` (the daily hook) as a +fire-and-forget write that never blocks scoring. Migration `v28-phone-timezone` +(`Packages/WhoopStore/Sources/WhoopStore/Database.swift`) — additive only, a new table, no +existing row touched, numbered past upstream's v26 and local v27. + +| Column | Type | Notes | +| --- | --- | --- | +| `day` | TEXT NOT NULL | Primary key. Local calendar day, `YYYY-MM-DD`. | +| `tzId` | TEXT NOT NULL | IANA identifier, e.g. `"America/Los_Angeles"`. | + +**Primary key:** `day`. `upsertPhoneTimezone(...)` is idempotent on it via `ON CONFLICT(day) DO +UPDATE` — re-recording the same day overwrites `tzId` in place rather than duplicating. +`phoneTimezone(day:)` reads a single day back (nil when none stored). + +**Also rides every upload as a header.** When Cloud Sync is compiled in (`CLOUD_SYNC`), +`CloudSyncClient.ingest` (`Strand/CloudSync/CloudSyncClient.swift`) sets `X-Phone-Timezone: +TimeZone.current.identifier` on `POST /ingest`, so the server records the phone's **current** zone +even before this per-day table lands in a given mirror. Default builds are offline and send no such +header (`docs/PRIVACY_SECURITY.md` §1). The noop-cloud server stores the header on its `ingestLog` +row (surfaced via `data_freshness.phoneTz`) and joins this table to attach a per-night `tzId` on +`sleep_summary`. + +**Android twin deferred** (Swift-only contributor): a Room migration adding `phoneTimezone` is +needed before an Android release, per the cross-platform migration-parity contract. + +--- + ## Index summary | Index | Table | Columns | Purpose | diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 1ff7048601..7ca12a90ae 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -810,6 +810,18 @@ Documented turn-on order: `SELECT_WRIST` → filtered on → raw-save on → dat splits `SELECT_WRIST` into its own separately-confirmed action, because it is the only one that writes strap state outliving the session **and** its value mapping is unattested. +**The `restart` argument (124 arg `2`).** `0` and `1` are the only values any build in this repo has +ever put on a wire, which leaves `2` as the one untried argument inside the family §6 records as +non-destructive. NOOP offers it as a separate, equally-gated action that sends the SAME preamble and +changes only the control byte — filtered on → raw-save on → data generation `restart`. What the strap +does with `2` is unknown: the name is read off the client enum's order, exactly like the wrist mapping, +so there is no basis for claiming it re-arms the two channel toggles itself or for claiming it needs +them already on. Changing one byte rather than also dropping the preamble is what keeps a null result +attributable. The probe report annotates every command line with the argument it was sent with +(`TOGGLE_LABRADOR_DATA_GENERATION(124) arg=2`) precisely because the two runs are otherwise +indistinguishable in a copied report. Reversal is the ordinary `stop` sequence — this leaves behind the +same strap state a normal start does. + **Packet layouts.** Both open with the same 17-byte status block (multi-byte fields little-endian): | Off | Size | Field | diff --git a/docs/bugs/2026-07-15-strap-battery-backfill-observability.md b/docs/bugs/2026-07-15-strap-battery-backfill-observability.md new file mode 100644 index 0000000000..0632f975a2 --- /dev/null +++ b/docs/bugs/2026-07-15-strap-battery-backfill-observability.md @@ -0,0 +1,679 @@ +# 2026-07-15 — strap death, battery, backfill + observability bug sweep + +Captured from a live incident: the strap drained and stopped streaming at **2026-07-14 13:27 PDT** +(last battery reading 10.9% @ 13:00, `charging=0`), a night of data went unscored, and the app was +unable to tell the wearer any of it. + +> **Triaged 2026-07-15 (second pass).** The root cause below was found AFTER the original sweep and it +> reframes most of it. Several findings were refuted — including two the first pass called "confirmed" +> — and the refuted claims have been **deleted, not annotated**, here and in the code comments that had +> started repeating them. Line numbers are re-derived against `capture-deep` at triage time; where the +> original pointer had drifted, the drift is noted. Findings not re-verified in this pass say so. + +> **Third pass, 2026-07-26 — BLE link-recovery port.** An abandoned local `main` line (diverged +> 2026-07-14, never merged) held unique work against **A1, A2, A4, A6 and B1**. It was reconciled onto +> `origin/main` as `ble-link-recovery-port`; those five sections are updated below. Two things in that +> line were **deliberately not ported**, both cases of the same trap — a change `origin/main` had already +> examined and reversed, which a naive merge would silently resurrect: +> +> 1. the **A5 cap raise** (6 → 120), whose rationale is the §0 contaminated measurement; and +> 2. a rival `StrapBatteryDisplay` that let a stale percent outlive a disconnect — i.e. **B2**, which A3's +> resolver had deliberately fixed. +> +> A third such change in that line (`Offload: stop archiving v20/v21 as "undecodable"`) is **E2**, already +> reverted on `origin/main` as harmful; it was not ported either. Nothing in this pass is strap-validated — +> both app targets compile and the pure tests pass, which per `CLAUDE.md` proves nothing about BLE +> behaviour. + +| | Meaning | +|---|---| +| **CONFIRMED** | re-derived from current code; still true | +| **FIXED** | landed; see the commit/lines named | +| **REFUTED** | was wrong — kept only as a "do not re-derive this" marker | +| **OPEN** | real and unfixed | +| **NOT RE-VERIFIED** | first-pass claim, not re-checked this pass — treat as unconfirmed | +| **NOT-IN-THIS-REPO** | about `noop-cloud`; unverifiable/unfixable from here | + +--- + +## 0. Root cause of the incident — an experimental probe left deep-buffer banking ON + +**Confirmed from the device's own prefs.** An experimental probe (`noop.probe.rawCensus`, tag +`burst-capture`, `startedAt 2026-07-14T15:36:21Z`) wrote **16 R22 SET_CONFIG flags** to the strap +(`r22Accepted` 0 → 16). Those flags turn on **deep-buffer banking**: from then on the strap banks one +**1244 B** (IMU) + one **2140 B** (optical) buffer for every **second** of history, on top of the 124 B +v18 per-second record. + +**124 B/s → 3508 B/s. Every sync had 28x more data to move, for the same wall-clock of history.** + +That is why syncs "suddenly got slow", and it is the single fact that explains the incident's headline +symptoms. The strap browned out at **22:51:34** and the flag cleared. + +Current state of the tree: +- `noopWhoop5DeepData` is **still `true`** in prefs — but that is only the app-side *permission* + (`PuffinExperiment.deepDataKey`, `Strand/BLE/PuffinExperiment.swift:21`). +- **The probe code no longer exists in the tree.** `PuffinRawProbe` / `rawCensus` have no source hits. +- `enableWhoop5DeepData()` (`Strand/BLE/BLEManager.swift:2238`) is reachable from **exactly one place**: + a Settings button (`Strand/Screens/SettingsView.swift:1381`). It is a one-shot manual send — nothing + re-applies it on connect. It additionally requires whoop5 family + the Experimental toggle + + connected + encrypted bond + **worn**. +- `2189f255` reverted a force-enable hack that had made the capture log's `isEnabled` return `true` + unconditionally. + +**So the strap cannot re-enter this state by itself.** But the R22 flags live on the *strap*, not the +phone — so the blast radius of that one button is a persistent 28x tax that survives app restarts and is +invisible from the app. **See "Needs a decision" at the bottom.** + +**What this costs the rest of this document:** any measurement taken between 2026-07-14 15:36 and the +22:51 brownout was taken at 28x. Findings that *measured a duration* (A5's "12h+") are contaminated; +findings that read *code structure* are not. + +--- + +## A. BLE / battery / connection + +### A1. Battery % on a 5/MG — **headline REFUTED, narrower defect CONFIRMED · FIXED (needs strap validation)** + +**REFUTED: "Battery % never lands on a 5.0/MG."** The doc's own A6 evidence disproves it — a +`soc=31.6` reading exists for 2026-07-15. Two mechanisms the first pass missed: + +1. **0x2A19 is SUBSCRIBED, not just read.** Discovery calls `requestNotify` on it + (`BLEManager.swift:3502`), and `enableLiveNotifications` re-subscribes it post-bond (`:2664`). +2. **`didUpdateValueFor` (`:3852-3858`) handles read responses AND notifications identically** — both + land on `state.setBattery`. So the `:3614` `enableLiveNotifications(reason: "post-bond 5/MG")` call + **does** restore the battery source. Its comment is not the lie the first pass called it. + +**The proof was unsound.** `LiveState.batteryPct` is never set to nil *by anything* — `clearBiometrics()` +(`Strand/BLE/LiveState.swift:467-478`) deliberately leaves it. So `batteryPct == nil` proves 0x2A19 never +delivered **since this app process started** — not "never". + +**CONFIRMED, and this is the real defect (narrower):** the one-shot `readValue` +(`BLEManager.swift:3497-3499`) fires in `didDiscoverCharacteristicsFor` — **pre-bond**, which an MG +rejects — and **no code path ever re-issues a read**. `enableLiveNotifications` (`:2657-2669`) only +calls `requestNotify`. Standard Battery Service pushes **on change**, so after every connect the first +reading waits for the SoC to actually move. At the measured 1.65 %/h discharge that is **up to ~36 min +of `nil` per connect** — long enough to look like "never" to a wearer, and the origin of this finding. + +**Fix 1b CONFIRMED:** `FrameRouter.swift:202-205` receives 5.0 BATTERY_LEVEL events and reads +`battery_charging` while **discarding `battery_pct`** ("Flag only — battery % keeps its family-specific +source (#77)"). The decode exists. The #77 stub-100 hazard is **4.0-only**, so gating on family would +give a 5/MG a self-healing ~8-min source. + +**FIXED** (`Strand/BLE/StrapBatteryReadPolicy.swift`, pinned by `StrandTests/StrapBatteryReadPolicyTests.swift`): +`enableLiveNotifications` now also calls `retryBatteryReadIfDue`, so every moment that already +re-subscribes notifies — post-bond 5/MG, post-bond 4.0, keep-alive, manual refresh, realtime start — is +also a moment the read can self-heal. The policy owns the two constraints that make that safe: the **#77 +family gate** (never read a 4.0's stub-100 characteristic, so the fix cannot flash a fake 100% over a real +charge) and a 30 s throttle, which is load-bearing because the call site re-enters on **every** +`.withResponse` ack, including every HISTORY_END during a multi-minute offload. + +Fix 1b (using the 5.0 BATTERY_LEVEL event's discarded `battery_pct` as a second source) was **not** taken — +the re-issued read closes the same gap without touching `FrameRouter`'s #77 invariant. + +**Still needs strap validation:** this is the BLE connect path, and per `CLAUDE.md` compile-success proves +nothing here. Both app targets build and the pure policy tests pass; the end-to-end behaviour is unverified. + +### A2. "Active · Live" overstates a raw link — **CONFIRMED · FIXED (needs strap validation)** + +`state.connected = true` at `BLEManager.swift:3108` in `didConnect`, **before** +`discoverPrimaryServices` — true the instant the raw link object exists, before any characteristic or +byte. `DevicesView.swift:101` renders "Active · Live" on `device.status == .active && live.connected`. +Combined with iOS state-restoration of a **dead** peripheral, the UI claimed "live" for a strap silent +21 hours. + +The card already carries a narrower guard for the *bond-refused* case (`DevicesView.swift:102-108`, +#221) — which shows the shape of the fix — but nothing gates on data actually flowing. `LiveState` has +the ingredient: `lastFrameAtUnix` (`:203-207`, #987). + +**FIXED:** `LiveState.linkProven` (`Strand/BLE/LiveState.swift`) — has this link actually carried a byte, +or completed its handshake? BLEManager clears it in `didConnect` and `willRestoreState`, sets it on first +data or a completed handshake, and restores it on disconnect. `DevicePillState.resolve` takes it as a new +`linkProven:` parameter and gates only the "· Live" half, so an unproven link falls back to plain +"Active" — the honest statement — rather than inventing a new pill state. + +The parameter **defaults `true`**, which is what keeps this narrow: the four non-WHOOP sources (Oura, +Huami, FTMS, standard-HR) also drive `connected`, and each already sets it only once genuinely subscribed. +They never touch the flag, so they keep an inert `true` instead of being regressed into looking offline. +Priority order is unchanged — the #221 bond-refused pill still wins. Pinned by +`StrandTests/DevicePillStateTests.swift`. + +**Still needs strap validation** (connect/restore semantics). + +### A3. Charging was structurally unrenderable without a charge % — **CONFIRMED · FIXED** + +`charging` and `batteryPct` come from **different sources and land independently**: `charging` rides the +strap's BATTERY_LEVEL event (~every 8 min, `FrameRouter.swift:202-205`), the % rides 0x2A19. The old +`LiquidBatteryButton` nested the `bolt.fill` **inside** `if let pct = live.batteryPct`, so +"charging, no % yet" could not render — it drew `bolt.slash`, which reads as *battery dead*, at a wearer +sitting on the charger. + +**This state is reachable, and reachable precisely during the incident:** BATTERY_LEVEL events are live +traffic and keep arriving mid-offload (only offload frames divert to the backfiller, +`BLEManager.swift:3864`; `FrameRouter` notes "this path never sees historical replay"). So the app held +a **fresh** charging bit and rendered a crossed-out bolt. + +> **This survives the "designed" refutation.** `guard !backfilling else { return }` +> (`BLEManager.swift:2482`) does sit above the battery poll, so *not polling mid-offload* is by design — +> and on a 5/MG that poll never runs anyway (`guard selectedModel.deviceFamily == .whoop4`, `:2503`). +> But "we deliberately don't have a %" does not license rendering a **known** charging state as its +> opposite. The absence is designed; the *claim* was wrong. + +**FIXED:** `LiquidTodayView.StrapBatteryDisplay.resolve(connected:batteryPct:charging:)` — a pure +3-case resolver (`.offline` / `.pending(charging:)` / `.charge(pct:charging:)`), pinned by +`StrandTests/LiquidBatteryDisplayTests.swift`. Charging now renders without a %; "connected but silent" +is now visually distinct from "no link"; the accessibility label no longer says a bare "Strap battery" +for three different states. + +### A4. Zombie restored peripheral spins forever — **NOT RE-VERIFIED · BOUNDED (needs strap validation)** + +First-pass claim: after iOS state-restoration returns a dead peripheral, the app marks it connected, the +handshake never completes, and it retries the **same dead handle** indefinitely — 40+ minutes of +`Backfill: deferred — connect handshake not done yet` every 1–3 min; only a force-quit recovered it. + +Re-verified only that the log line exists on both platforms (`BLEManager.swift:1562`, +`WhoopBleClient.kt:4893`). The restoration/retry behaviour itself was **not** re-traced this pass, and +it is not strap-testable right now. Treat the *claim* as unconfirmed. + +**The mechanism WAS since traced, and it is real** (Swift side): `willRestoreState` marks the link up and +re-discovers services, but the CLIENT_HELLO / bond write that follows is `.withResponse`. On a wedged +strap it is never acked, so `didWriteValueFor` never fires — and that is the **only** caller of +`startKeepAlive()`, so the 120 s liveness watchdog inside `keepAliveFire` (the machinery that would have +bounced exactly this link) is never armed. Nothing else bounds the retry. + +**BOUNDED:** `Strand/BLE/StalledHandshakePolicy.swift` arms a 90 s fuse at connect **and at restore**, then +cancels the peripheral so the existing 3 s disconnect-rescan recovers the link. It deliberately adds no new +reconnect mechanism. The fuse is picked against the fuses already in the tree — above the 60 s reboot-settle +backstop, below the 120 s liveness fuse — and every input to `shouldRecover` is a reason to stay out of the +way (`sawData`, `handshakeDone`, `intentionalDisconnect`, `autoReconnectPaused`, `bondRefused`), so it fires +only on the narrow case nothing else owns. Pinned by `StrandTests/StalledHandshakePolicyTests.swift`. + +This bounds the failure mode whether or not the original 40-minute claim is ever reproduced. **Kotlin twin +not written** — Android has no `willRestoreState` equivalent, so the trigger does not exist there. +**Still needs strap validation.** + +### A5. `maxAutoContinues = 6` throttled a healthy recovery — **REFUTED · DECIDED: the cap stays at 6** + +**This finding was the sweep's "worst practical impact". It does not survive §0.** It was briefly +"fixed" (6 → 120, `2b968d27`) and then deliberately **reverted** (`d2ec5dc3`, cap back to 6) once the +root cause was known. The reasoning is now recorded at the constant on both platforms so it is not +"fixed" a third time: + +- **The severing evidence came from a strap the probe had switched into R22 deep-data banking** — + 3508 B/s of history vs the default **124 B/s**. A default strap banks **28x less**, so one session + covers ~28x more history and an 18 h backlog drains in **~2 sessions**. The cap never fires. +- **Raising it widens the blast radius of the next guard-2 bug.** #928 and #1012 both burned the *whole + cap* in EMPTY offloads — the cap is what bounded them. +- **It applies to WHOOP 4.0 too**, where a long drain starves the realtime keep-alive (`guard + !backfilling`, #160). + +> **It was re-raised a third time, and refused (2026-07-26).** The abandoned local `main` line carried a +> commit raising the cap to 120 behind an `isDeepDrain` evidence gate. Its rationale — "each session +> recovers ~29 min of strap history, so an 18-hour backlog needs ~37 sessions" — **is the §0 contaminated +> measurement**, taken at 28x banking. When that line's unique work was ported onto `origin/main`, the cap +> change was dropped and `BackfillContinuation` left byte-identical; `StrandTests/BackfillContinuationTests` +> (21 tests) passes unchanged. Note the gate itself was the shape this section invites ("a conditional +> bypass while provably behind and advancing"), so the *mechanism* is not what was rejected — the +> **evidence** is. A raise still needs a deep-backlog case demonstrated on a **default** strap. + +The "~12h+ to recover a night" that made this finding look severe was measured **at 28x** (§0). Remove +the 28x and the arithmetic that condemned the cap evaporates. + +**FIXED (the half that was real):** the idle watchdog 60 → **15 s** on both platforms +(`BLEManager.swift:1676`, `WhoopBleClient.kt:565`, `d2ec5dc3`). Every session paid 60 s in full as dead +air on exit, and 60 s was set without measuring the inter-frame tail — a 3,018-buffer trace from a real +drain puts gaps at p50 64 ms / p90 106 ms / p99 273 ms / **max 2,520 ms**. 15 s is ~6x the measured max +and still clears the type-43 flood. (Not the ~5 s the tail alone would justify: one drain, one firmware, +and the risk is asymmetric — firing early is safe but wasteful, firing late only costs idle seconds.) + +**REFUTED, do not re-derive:** +- **There is no "~25.4 KB/s airtime ceiling"** and no "~7x real-time". That was a one-window fit; it did + not survive re-measurement across a dozen windows. The absolute link rate is not a constant of this + system. Both figures are gone from the tree. +- **The "~1.19x throughput" (and the "~3.7x → ~1.0x collapse") was never a bug.** The strap was **CAUGHT + UP**. History cannot drain faster than the rate at which it is banked — with no backlog, **~1.0x is + the floor**, not a symptom. + +Drain time is `(bytes banked per second-of-history) / (bytes the link moves per second)` and **both +terms move**. Any single headline multiplier is a property of one capture. + +**REFUTED, do not re-derive:** +- **There is no "~25.4 KB/s airtime ceiling"** and no "~7x real-time". That was a one-window fit; it did + not survive re-measurement across a dozen windows. The absolute link rate is not a constant of this + system. Both figures have been removed from the cap rationale on both platforms and from + `PuffinDeepBufferLog.swift`. +- **The "~1.19x throughput" was never a bug.** The strap was **CAUGHT UP**. You cannot drain history + faster than it is generated — with no backlog, **~1.0x is the floor**, not a symptom. + +Drain time is `(bytes banked per second-of-history) / (bytes the link moves per second)` and **both +terms move**. Any single headline multiplier is a property of one capture. + +### A6. "Charging flag never lands" — **REFUTED as stated · reframed question now CONFIRMED · fix INERT** + +The finding's own evidence contradicts its title. The row was `soc=31.6, charging=0`: + +- **`charging=0` is not "never landed" — it is "landed and read false".** A flag that never landed is + **NULL**: `Streams.swift:374-380` sets `charging` only when the frame actually reported it + (`charging: Bool? // only the BATTERY_LEVEL event reports this; nil otherwise`). A non-NULL 0 proves + the decode ran *and* the strap's own event said not-charging. +- **It was not a "live row".** `battery` table rows are inserted from `streams.battery` + (`StreamStore.swift:132`) — the **decoded offload stream**. Its `ts` is the strap's RTC stamp on a + *banked* record, not wall-clock "now". So "charging=0 **while actively charging**" compares a + historical record's timestamp against an observation made at inspection time. The doc never + established the charger was on at strap-RTC 10:58:06. + +**Was OPEN (reframed):** whether the 5/MG's BATTERY_LEVEL bit0 tracks charging **at all** is genuinely +unknown and worth settling — but it needs a controlled capture (put it on the charger, watch +`state.charging` change), not a single historical row. A3's fix is what makes that flag visible enough +to observe. + +> **CONFIRMED 2026-07-26, from the cloud mirror — the bit is not a usable charging indicator.** The +> controlled capture asked for above turned out to be unnecessary: the mirror already held one, with +> ground truth attached. A charging session on **2026-07-15, 11:07 → 13:34 MDT** carries `CHARGING_ON(7)` +> ×10, `BATTERY_PACK_CONNECTED(21)` ×3, and SoC climbing **4.7% → 100.0%** — and `charging = false` on +> every one of its ~300 battery readings. **Not `null` — `false`.** That distinction is the whole finding: +> `null` would mean the flag never reported, while `false` means the BATTERY_LEVEL path decoded the bit +> and the strap asserted *not charging* while going from nearly flat to full. The "single historical row" +> objection that (correctly) sank the first pass does not apply — this is a labelled transition, not one +> row. Caveats: one session, one strap, and the mirror was 8.9 days stale when read. + +**ATTEMPTED MITIGATION — and it is almost certainly INERT.** `Strand/BLE/StrapChargeInference.swift` stops +treating that one unverified bit as the only witness. A **rising SoC** is direct, family-independent, +decoder-independent evidence of a charge, and this codebase already trusts exactly that inference — +`BatteryEstimator.chargeStepPct` is documented as "a SoC rise larger than this marks a CHARGE", and the +runtime estimate has always restarted its discharge run on it. `LiveState.chargingEffective` resolves the +flag against that second witness; the UI reads it, so a strap on the puck reporting `charging=0` no longer +renders "not charging" **and** no longer shows a "~2 days left" discharge estimate. + +Strictly additive, and deliberately so: a confirmed `true` stays `true`, inference can only ever ADD a +charge and never deny one, and with neither witness it answers `nil` (unknown) rather than asserting +"not charging". **`LiveState.charging` remains the raw WIRE value** and is what gets persisted — so no +stored value diverges from Kotlin and no `.noopbak` contract moves. Pinned by +`StrandTests/StrapChargeInferenceTests.swift`. + +Because it is display-only, this **does not** substitute for the controlled capture — it makes the app +correct either way while the question stays open. Kotlin twin is a UI-parity item, not a data one. + +> **OPEN — the threshold is set above the real charge step, so the rescue never triggers.** `isRising` +> needs **> 1.0 pp** (`BatteryEstimator.chargeStepPct`) between CONSECUTIVE readings. The confirmed charge +> above stepped **+0.3 … +0.7 pp** per reading at a **~30 s** cadence and never once cleared it. The +> `recentRiseWindowSeconds` comment assumes readings are "~8 min apart", which is what makes 1.0 pp look +> like a low bar; at the observed cadence it is not. The live path looks no better — a 5/MG's live SoC is +> whole-percent `0x2A19`, so a charge steps by exactly `+1`, and the strict `>` that exists to survive +> that quantisation also excludes exactly 1.0. +> +> So `chargingEffective` degenerates to the raw flag and the wearer still sees "not charging" plus a +> discharge estimate on a strap that is charging — **the original harm is not actually fixed.** It is not +> a regression (the inference is additive, so behaviour equals pre-fix), but the section above overstates +> what shipped and this correction supersedes it. +> +> Do not read the green tests as coverage: `StrapChargeInferenceTests`' synthetic samples step by more +> than 1.0 pp, which is exactly why they pass. +> +> **Likely fix:** a RATE test (pp per unit time) rather than a fixed per-reading delta — ~50 pp/h charging +> vs ~1.65 pp/h discharge is a ~30x separation that holds at any cadence. **Unmeasured and needed first:** +> the `battery` table is fed by the decoded OFFLOAD stream, not the live path that populates +> `LiveState.batterySamples`, so the LIVE cadence and quantisation are still unconfirmed. The detail is +> duplicated at the type in `Strand/BLE/StrapChargeInference.swift` (GitHub issues are disabled on this +> fork, so there is no tracker item). + +### A7. `lastSeenAt` frozen at pairing → "last seen 18 days ago" — **CONFIRMED · OPEN (both platforms)** + +``` +my-whoop | added 2026-06-26 12:06:20 | lastSeen 2026-06-26 12:06:20 ← IDENTICAL +oura-api | added 2026-07-10 | lastSeen 2026-07-13 ← updates fine +``` + +`lastSeenAt` is stamped **only** on wizard-add and on promote-to-active, never refreshed from a live +link: +- Swift — `Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift:40` + (`UPDATE pairedDevice SET status = 'active', lastSeenAt = ?`) and the upsert at `:156-163`. +- Kotlin — `android/app/src/main/java/com/noop/data/DeviceRegistryDao.kt:34`, same shape + ("Promote one device to `active` and stamp its lastSeenAt"). + +**Shared stored-data semantics ⇒ a fix needs the Kotlin twin in the same commit.** Cheap in principle +(stamp on a connect/frame event); left open because *where* to stamp is a connect-path decision. +**Needs the user.** + +--- + +## B. Observability (the wearer's #1 ask) + +> "All of this stuff, like backfill progress, should be something you can see on the app. It is kind of +> misleading me right now." + +### B1. Backfill progress invisible — **PARTLY REFUTED · iOS half FIXED · remainder now FIXED** + +**REFUTED as "the app never surfaces it."** It does, on macOS and in the menu bar: +`SyncingHistoryNote` ("Syncing strap history… N chunks pulled", `Strand/Screens/ScreenScaffold.swift:199`) +is used by Sleep (`SleepView.swift:2621`), Intelligence (`IntelligenceView.swift:326`) and the menu bar +(`MenuBarContent.swift:269`); the classic `TodayView` has `SyncStatusChip` (`:4417-4430`). + +**CONFIRMED, precisely, for iOS — which is where the wearer was looking.** `RootTabView.swift:50` picks +`LiquidTodayView()` when the Liquid flag is on, and **`LiquidTodayView` had no backfill indication at +all**. Not a missing feature — a **v8 redesign regression**, the same class as #992, which dropped the +"~X days left" runtime estimate from the strap-battery row (as that row's own comment records, +`LiquidTodayView.swift:1593-1595`). + +**FIXED (iOS):** `LiquidSyncStatusRow` in the Data Sources card — "Strap history / Syncing… N chunks" +while `live.backfilling`, else "Synced N ago" from `live.lastSyncedAt`. + +**FIXED — the "how far behind" half** (2026-07-26). *"Recovering strap history / Have it through July 14, +14:31 — ~15h behind"* now renders, via `LiquidBackfillProgressRow` over the pure +`Strand/BLE/BackfillProgress.swift`. + +The plumbing objection above was right, and this is how it was resolved without a new Repository read: +BLEManager **already** reads the frontier (`collector.latestHRSampleTs()`) once per offload session, for +the auto-continue decision — exactly the cadence the readout wants. It now publishes it there via +`LiveState.setDataFrontier`, so `backfillProgress` computes over `connected` / `backfilling` / +`dataFrontierUnix` / `strapRange`, all already `@Published`. No extra query, no third piece of state. + +Two properties worth keeping: the resolver reuses `BackfillContinuation.defaultBehindGapSeconds`, so the +readout can never claim "behind" about a gap the drain has already decided is closed; and a frontier +*ahead* of the strap's reported newest (the #451 stale-epoch and #928 future-clock cases) resolves to +`.unknown` rather than a negative backlog. It also answers **between** sessions — `.pending` — which is the +state the wearer actually misread as "my data is gone". `LiquidSyncStatusRow` is kept and still answers +*that* a drain is running. Pinned by `StrandTests/BackfillProgressTests.swift`. + +### B2. Battery shown with no staleness marker — **CONFIRMED · FIXED (worst case)** + +A 21-hour-old reading rendered identically to a live one. Mechanism found: **`LiveState.batteryPct` is +never cleared** (§A1), so `LiquidBatteryButton` — which keyed off `batteryPct` alone with **no +`connected` gate** — kept rendering a dead strap's last charge as if current. The +`LiquidStrapBatteryRow` **directly below it** already required `live.connected` +(`LiquidTodayView.swift:1573`), so the two Liquid surfaces actively disagreed; the button was the wrong +one. + +**FIXED** by the same `StrapBatteryDisplay.resolve` gate (A3): no link ⇒ `.offline`, never a stale +number. Pinned by `testStaleChargeIsNotShownOnceTheLinkIsGone`. + +*Still open (softer):* a **connected** strap whose last reading is old has no age marker. "Last seen 21h +ago at 11%" needs a reading timestamp `LiveState` doesn't currently keep for battery. + +### B3. Rest carries but Charge/Effort don't — **CONFIRMED · Charge FIXED (Effort correctly excluded)** + +On a dead-strap / unscored-today day the Liquid Today (the DEFAULT screen) showed a plausible Rest — +carried from the last scored night — beside a blank Charge and a blank Effort, and the Charge state pill +still read "Calibrating" at a wearer 14 nights past the seed gate. Root cause: the Liquid heroes read +`displayDay?.recovery` / `.strain` **raw**, while the Rest hero carried (`freshRestScore`, #977) and the +vitals carried (`Repository.lastVitalsDay`) right beside them. Liquid was the ONLY surface that blanked — +classic `TodayView` carries via `lastScoredCharge` (#543), the widget/watch/Live Activity via +`Repository.widgetAnchor` (#911), and Android via `TodayScreen.lastScoredRecoveryDay`. + +**FIXED (Charge):** `LiquidTodayView.ChargeDisplay.resolve` now carries the last scored night's REAL +Charge, labelled "Last night · " (or "Latest sleep · " once #779-stale), stamped by the SAME +`TodayView.lastScoredRecoveryDay` + `carriedCaption` the classic screen uses, so the two Today screens +cannot drift. The state pill now tells the truth — only a genuinely calibrating baseline reads +"Calibrating"; a trusted baseline with no scored night reads honest "No data". The Key-Metrics Recovery +tile reads the same resolved value so the tile and hero can't disagree. Pinned by `LiquidChargeCarryTests` +(9 cases, pure resolver). Verified end-to-end in the simulator on the real on-device DB with today's row +nulled to stage the rollover: before = Charge "–" / pill "Calibrating"; after = Charge 47 carried from +Jul 14 / pill "Last night". + +**Effort deliberately does NOT carry** — Effort is *today's* accumulation, so painting yesterday's number +onto today would be a false statement, not a stale one (the same reason a blind `COALESCE` on the stored +`strain` column would be wrong). It stays blank until tonight scores. + +**Relationship to F1:** this carry SOFTENS F1's user-visible blank-out (the last good Charge shows instead +of "–"), but does NOT fix F1's root cause — the day's OWN row is still nil because the stale HRV baseline +gated it. F1's scoring-side change remains deferred to the user (see F1). + +### B4. Battery invisible over MCP — **NOT-IN-THIS-REPO** + +The `battery` table (soc/mv/charging) and the `event` table (**11,457 rows**, `BATTERY_LEVEL` carrying +`battery_pct`) were never exposed. `battery_series`, `device_events`, `imu_coverage` are built + tested +in noop-cloud (branch `tz-upload`), pending deploy. Tracked there, not here. + +### B5. Burst-capture census + firmware are not in the DB — **PARTLY STALE · OPEN (firmware half)** + +**Stale:** the census half described `PuffinRawProbe.finishSession()` writing R22 census JSON to a file + +`UserDefaults(noop.probe.rawCensus)`. **That class no longer exists in the tree** (§0) — there is no +census producer left to migrate. + +**CONFIRMED (firmware half):** firmware lives only in `UserDefaults("noop.lastFirmware")` +(`FrameRouter.swift:99`), and neither `device` nor `pairedDevice` has a firmware column. `ingest.ts` +dropping `settings.json` is a noop-cloud fact. ⇒ strap firmware is invisible to the MCP without a +producer-side migration. + +--- + +## C. Alerts — all three CONFIRMED, all OPEN, Kotlin twins match exactly + +### C1. Battery alerts fire once, then latch — no escalation — **CONFIRMED · OPEN** + +Device state: `batteryLowAlerted = true`, `batteryRuntimeAlerted = true` — **both fired, and the strap +still died.** `BatteryAlertPolicy` is nested in `Strand/System/BatteryNotifier.swift:19` (not its own +file): `lowThreshold = 15` (`:20`), `lowRearmAbove = 25` (`:21`). The latch is real (`:45-51`) — +re-arm is gated **only** on SoC recovering to 25%, so once low fires at ≤15% nothing re-fires before a +charge. The runtime alert fires at `runtimeAlertHours = 24` +(`Packages/StrandAnalytics/Sources/StrandAnalytics/BatteryEstimator.swift:255`) — at the measured +1.65 %/h that is **39.6% SoC, ~18 h early** — then latches the same way (`:271-275`). Net: **~3 hours of +silence from 15% to death.** Both toggles default ON (`Strand/Data/BehaviorStore.swift:100-101`), so +that is not the explanation. + +**Correction to the first pass:** it recorded "*Being fixed:* critical 12% alert + bedtime night-guard". +**Neither exists anywhere in the tree, on either platform.** There are exactly three battery +notification ids (`battery-low`, `battery-full`, `battery-runtime`) and three persisted gates. That work +was aspirational, not landed. + +### C2. Runtime estimate ignores the strap's cutoff — **CONFIRMED · OPEN** + +`BatteryEstimator.swift:99`: `let remaining = max(0, current) / rate` — time to **0%**. The strap appears +to stop near ~10%, so every estimate overstates runway. **No cutoff constant exists on either platform.** +The `1.5x` rated-life clamp at `:104` is a runaway ceiling, not a cutoff — it does not subtract the +unusable tail. *Caveat: the ~10% cutoff is an incident observation, not a documented constant — the +value a fix should use is not derivable from this repo.* + +### C3. Rated-path predictive alert is unreachable on a 5.0 — **CONFIRMED · OPEN** (pointer drifted) + +`ratedLifeHoursWhoop5 = 288` now lives at `BatteryEstimator.swift:23`; the first pass pointed at the +*selection* site, now `BLEManager.swift:1021-1022`. On the rated path `rate = 100/288 = 0.3472 %/h`, so +`remaining = current × 2.88` and the 24 h line falls at **8.33% SoC — below the cutoff**. The `:104` +clamp is never binding here, so it doesn't rescue it. A 5.0 falling back to the rated path gets **no +predictive alert, ever**. (A 4.0 at `ratedHours = 108` crosses at 22.2%, comfortably reachable — this is +5.0-specific.) + +**Parity note (all of C):** the Kotlin twins exist and **every constant matches** — +`com/noop/notif/BatteryAlertNotifier.kt:23`, `com/noop/analytics/BatteryEstimator.kt`, +`com/noop/ble/WhoopConnectionService.kt:262-263`. C2 is mirrored at `BatteryEstimator.kt:94`, C3 via +`ratedLifeHoursWhoop5 = 288.0`. **The bugs are portable, not absent** — any fix lands on both platforms +in the same PR. + +--- + +## D. Cloud sync (fork-local; compile-gated behind `CLOUD_SYNC`, absent from upstream and default builds) + +### D1. Upload can never complete from the background — **CONFIRMED · OPEN** ("fix built" was wrong) + +`CloudSyncClient.swift:34` still uses `URLSession.shared`. **Zero** `beginBackgroundTask`, **zero** +background `URLSessionConfiguration` in app code. A push grants ~30 s; deflate-zipping ~400 MB +(→ ~81.5 MB) blows past it and in-flight tasks die on suspension. + +**Correction: the "*(fix built)*" tag was wrong and is removed.** The working tree's new +`DeepBufferUploadPlan`/`DeepBufferUploader` add **chunking + a resume watermark** (16 MB chunks, 8 per +run, watermark persisted per chunk) to a **new `/deepbuf` lane** — that bounds how much a suspension +destroys, but the chunks still ride `URLSession.shared` and still die on suspension. **The `/ingest` +whole-DB lane D1 actually indicts is untouched** (`CloudSyncClient.swift:78-97` still streams the whole +file in one shot). + +`cloudsync.lastStatus` is written on **every** completion, success or failure — +`Self.persistLastStatus(line)` (`CloudSyncModel.swift:260`) sits *outside* the `if succeeded` block +(`:257-259`). It is only reached if `performSync` runs to completion, which is exactly why the device +showed it **frozen** at `"Uploaded 81.5 MB · 12:21 PM"` rather than showing a failure — the two halves of +this finding corroborate each other. + +### D2. Warm resume triggers no sync — **CONFIRMED · OPEN** + +`autoSyncIfDue` has exactly two call sites: `StrandiOS/App/RootTabView.swift:133` (iOS, inside `.task {}`, +which does not re-run on resume) and `Strand/App/RootView.swift:324` (macOS only — excluded from the iOS +target). `scenePhase == .active` (`StrandiOSApp.swift:205-223`) does not call CloudSync. + +*Two sharpenings:* `scenePhase` **does** reach CloudSync on the way **out** — +`CloudSyncBackgroundRefresh.schedule()` in the `.background` branch (`:233-238`) — and +`backgroundSyncIfDue` (`CloudSyncModel.swift:404`, 4 h gate) serves the BGAppRefreshTask/silent-push +lane. The gap is specifically **warm resume**, not background sync in general. + +### D3. Server volume ~1 upload from ENOSPC — **NOT-IN-THIS-REPO** + +1008 MiB total, ~461 used, ~547 free; an ingest needs ~400 MB staged alongside the ~460 MB mirror. +`MAX_INGEST_BYTES = 768MB` exceeds the entire volume. `/ingest` logs nothing on receipt, success, or the +400 path. **None of this exists in this repo** — no `ingest.ts`, no server sources. Belongs to the +`noop-cloud` checkout. + +--- + +## E. Deep data / MG + +### E1. 2140 B optical frames have no home — **diagnosis REFUTED · the real gap is CONFIRMED** + +**REFUTED — the stated action does not achieve the stated goal.** The claim was that the DB lacks +`ppgWaveformSample` because the installed build predated upstream's `v27-ppg-waveform` (#415), so +"rebuild + install current fork main to start storing them". + +The migration **is in this tree** (`Packages/WhoopStore/Sources/WhoopStore/Database.swift:591`), highest +is `v29-daily-avg-sdnn` (`:619`), and the Room twin is at parity (`WhoopDatabase.kt:550`, pinned by +`PpgWaveformMigrationTest.kt`). The "top out at v28" reading was internally consistent — the fork's local +`v28-phone-timezone` predated the merge, and GRDB keys on the identifier *string*, so the out-of-order +insert applies cleanly. + +**But it stores the wrong thing.** `ppgWaveformSample` is fed **only** from `p["ppg_waveform"]` +(`HistoricalStreams.swift:214-217`), emitted **only** by `decodeWhoop5HistoricalV26` — **hist_version 26** +records. The 2140 B buffers are **v20** (`Interpreter.swift:587`), decoded into `channel_b{n}_{half}` +fields that **nothing consumes**. The commit seals it: `ee1de9a7 … persist the WHOOP 5.0 **v26** raw PPG +waveform`. + +**CONFIRMED (the real gap):** the 2140 B v20 optical buffers — the SpO2/BP substrate — still have **no DB +table in this tree**. Rebuilding gives *v26 PPG* a home; it does nothing for v20. **Needs the user** (a +real feature, not a rebuild). + +**Their only durable path today is the reject archive** (`rejected_history.jsonl`), which +`BLEManager.swift:925-930` replays and **retro-decodes on version bump** — so the bytes survive until a +v20 decoder lands. That is exactly why exempting v20/v21 from that archive destroyed them (§E2), and why +the archive must not be "optimised" away before v20 has a table. The `puffin-deepbuffers.jsonl` research +log is **not** a substitute: it is default-OFF and gated on a different key. + +### E2. 1244 B frames logged as "rejected" — **REFUTED (nothing was wrong) · the "fix" for it was harmful and is REVERTED** + +**"Rejected" never meant "dropped".** `rejectedHistoricalRecords` (`HistoricalStreams.swift:66`) is an +independent *classification* pass, not a decode gate: decoded rows commit separately +(`Backfiller.swift:554`), and the rejected set only selects frames whose **raw bytes get archived** to +`rejected_history.jsonl` for later RE (`:580-586`) so the ack can't free the strap's only copy of an +unmapped layout. A frame there means "no consumer claimed it — **keep the bytes**", not "data lost". + +Why 1244 B appeared: the predicate (`:99-100`) logs anything with no `unix`, or with neither +`heart_rate` nor `gravity_x`. v21 (1244 B) carries `optical_ch0/1/2` and neither of those → logged. No +contradiction with working IMU decode — that runs through `Whoop5RawImu.decode` in the deep-buffer path, +a **different consumer**. Both facts were always true. **"Needs investigation" was the right instinct; +the frames were fine.** + +> **Cautionary tale — this finding caused a data-loss bug.** Acting on it, `7942f667` exempted v20/v21 +> from the reject archive ("they decode, for the same reason v26 is exempted one line above"). That is +> backwards, and `86d414ad` reverted it. **v26 is safe to exempt because it HAS a durable home** +> (`Streams.ppgWaveform` → `ppgWaveformSample`), so the archive is redundant for it. **v20/v21 have no +> home** (§E1) — `decodeWhoop5HistoricalV2021` writes `optical_ch0..2` / `channel_b*` into an in-memory +> frame with **zero production consumers on either platform**, and the frame is discarded. With the +> exemption: not archived, not persisted, **trim acked — and the strap frees the buffers permanently**. +> `PuffinDeepBufferLog` does not catch them either (default-OFF, and gated on +> `PuffinFrameRecorder.enabledKey`, a *different* key from the deep-data experiment). +> Two things that revert surfaced: `RawHistoryArchive.perVersionFloor` (`:27-32`) names "WHOOP 5 v20/v21" +> as the exact rare-layout case its eviction floor was built for (#344) — the exemption deleted precisely +> what a prior fix exists to preserve — and `BLEManager.swift:925-930` replays the archive and +> **retro-decodes it on version bump**, so for v20 (micro-decode unsolved) that archive is the ONLY path +> by which the data is ever recovered once a decoder lands. +> The exemption's throughput rationale was also the refuted "~3.7x → ~1.0x collapse" (§A5) — the strap +> was caught up. + +**Current state: reverted; the archive preserves v20/v21 again.** The real problems it gestured at are +real but live elsewhere: the false "N records couldn't be decoded" banner should stop counting known +layouts (`BLEManager.swift:1801-1803`), and the archive's at-cap read-all + hex-decode + 5 MB atomic +rewrite per chunk should be fixed **in** the archive. **Both OPEN.** + +### E3. Deep IMU capture is barely running — **mechanism CONFIRMED · evidence CORRECTED** + +**Mechanism CONFIRMED, and §0 explains the 58-minute window.** Banking needs **two** default-off gates +plus a manual tap: the Experimental toggle (`PuffinExperiment.deepDataEnabled`) **and** the capture +toggle (`PuffinFrameRecorder.enabledKey`), plus a one-shot Settings button press +(`SettingsView.swift:1381`) while connected + encrypted-bonded + **worn**. Nothing re-applies R22 on +connect. "One 58-minute window across 8 months" is precisely the shape that produces. + +**Evidence corrected:** there is **no `imuActivity` table anywhere in this tree** — not in +`Database.swift`, not in Room. `ImuActivityFeatures` is serialized only as the `"imu"` field inside +`puffin-deepbuffers.jsonl`, and **only for live frames, never offload** +(`PuffinDeepBufferLog.swift`: `let imu = isOffload ? "" : Self.decodedImuField(frame)`). So "2,945 rows" +is a cloud-side/derived artifact. **Reconcile where those rows come from before acting on E3.** + +--- + +## F. Scoring + +### F1. Charge silently vanishes after 14 stale nights — **CONFIRMED · comment FIXED · code change needs the user** + +All four first-pass pointers were **exact**. `Baselines.swift:40` documented `.stale` as "usable but no +update for > STALE_DAYS nights", while `:69` defines `usable = (status == .provisional || status == +.trusted)` — **`.stale` excluded** — with `staleDays = 14` (`:102`) and `.stale` assigned at `:223`. +`RecoveryScorer.swift:417` then does `if !hrvBaselineUsable { return nil }`. ⇒ 14 nights without a valid +HRV night ⇒ **Charge silently returns nil**, no explanation surfaced. + +**Resolved: the COMMENT was wrong, not the code.** Excluding `.stale` from `usable` is deliberate and +coherent across **every** consumer (`AnalyticsEngine`, `DailyStressEngine`, `ReadinessEngine`, +`ScoreConfidence`, `WatchRecovery`, `DaytimeBaselines`, `AppModel`, `IntelligenceEngine`): `usable` means +"trust this personal baseline", and one nobody has confirmed in >14 nights is exactly one not to trust. +The comment's word "usable" simply collided with the property name. Current behaviour is **pinned by +tests on both platforms** (`VitalBandsTests.testStaleBaselineFallsBackToPopulation`, +`VitalBandsTest.staleBaseline_fallsBackToPopulation`). + +**FIXED (comment only, both platforms):** `Baselines.swift` + `AnalyticsModels.kt:179-180` — and the +sibling `usable` doc, which claimed "at least provisionally usable (nValid ≥ MIN_NIGHTS_SEED)"; `.stale` +also clears that bar and is still excluded. **Changes no numbers.** + +**The silent-nil half is still OPEN and is a product decision.** Making `.stale` usable would flip +`VitalBands.basis` to personal (breaking pinned tests on both platforms), un-gate `RecoveryScorer`, and +move skinTempDev / Readiness / ScoreConfidence / illness signals — a **stored-value + parity change**. +The narrower fix — *surface* "no Charge: baseline stale" instead of a blank — changes no numbers. +**Needs the user.** + +**Escalation the first pass under-stated.** `baselines2.hrv` is folded to a single **end-of-history** +snapshot (`IntelligenceEngine.swift:875`) and that one state gates **every** day re-scored in pass 2. +Dead nights enter the fold as `nil` (`:843`; B3 verified an all-NULL row exists), so 15+ trailing +nil-HRV rows ⇒ `.stale` ⇒ `!usable` ⇒ `recovery` returns nil **for the whole re-scored window**, not just +the current night. It self-heals the instant one valid night lands. **Unanswered — answer this first:** +do those nils get written back over previously-stored non-nil `recovery` values? That is the difference +between "Charge is blank today" and "**Charge history got overwritten**". + +--- + +## Needs a decision + +1. **§0 — the R22 button.** Deep-buffer banking is a **persistent 28x tax written to the strap**, it is + never re-applied *or cleared* on connect, and the app cannot see it. The probe that tripped it is + gone, but the Settings button remains. Options: leave it (documented), have the app clear R22 on + connect unless deep data is deliberately on, or surface "deep banking is ON" in the UI. Also: + `noopWhoop5DeepData` is **still `true`** on the device — decide whether to clear it. +2. **A1 / A2 / A4 / A7** — all BLE/connect-path or shared-registry changes. Each needs strap validation + (`CLAUDE.md`: compile-success proves nothing here) and A7 needs a Kotlin twin. Not touched while a + live measurement is running. +3. **C1 / C2 / C3** — the alert policy is a product judgement (what SoC, what escalation, what cutoff), + and the ~10% cutoff isn't a known constant. All three need Kotlin twins. +4. **F1** — surface the stale-baseline reason (no numbers move) vs. make `.stale` usable (numbers move, + pinned tests break, needs a twin). And answer the overwrite question first. +5. **E1** — giving the 2140 B v20 optical buffers a home is a migration + decoder feature, not a rebuild. + Until then the reject archive is their only durable path — do not exempt or evict them (§E2). +6. **E2 fallout, both OPEN** — the false "N records couldn't be decoded" banner should stop counting + known layouts (`BLEManager.swift:1801-1803`), and the reject archive's at-cap read-all + hex-decode + + 5 MB atomic rewrite per chunk should be fixed **in** the archive. These are the real costs the + destructive v20/v21 exemption was reaching for. + +--- + +## What this sweep should teach the next one + +Three of the sweep's most confident findings — A5 ("worst practical impact"), A6, E2 — were artifacts, +and one of them (E2) produced a **data-destroying "fix"** before the root cause was known. The pattern is +identical each time: **a measurement taken in an unknown state was read as a property of the code.** + +- **A5 / §0** — 28x more bytes per second-of-history was read as a duty-cycle bug. +- **A6** — a historical banked record's strap-RTC timestamp was read as "now". +- **E2** — a preservation mechanism ("keep these bytes") was read as a failure ("rejected"). +- **A1** — a `nil` that means "not since process start" was read as "never". + +Find the state before optimising against the symptom. And when a finding says *"unexplained — needs +investigation; do not guess"* (E2 said exactly that), that instruction outranks a plausible-looking fix. diff --git a/docs/draft-upstream-issue-hr-wake.md b/docs/draft-upstream-issue-hr-wake.md new file mode 100644 index 0000000000..5c57ecbb29 --- /dev/null +++ b/docs/draft-upstream-issue-hr-wake.md @@ -0,0 +1,85 @@ +> **Filed as https://github.com/ryanbr/noop/issues/462 on 2026-07-14.** The fix shipped in this fork (default ON); see `docs/ANALYTICS.md` → *Motion-corroborated wake*. This file is kept as the filed issue's source. + +# Sleep scoring over-calls WAKE when resting HR is elevated but the wrist is motionless (supplement / fever / alcohol nights) + +## Summary + +Both stagers (`SleepStager` V1 and the default `SleepStagerV2`) call **wake** primarily off HR and +HR-variability, and the session-detection HR gate (`SleepStager.confirmSleepWithHR`) rejects a still +in-bed run whose median HR sits above the sleep band. Neither path consults **motion** or **posture** +before making that call. On a night whose resting HR is held elevated **without the wearer getting up** +— a supplement protocol, a fever, a hot room, alcohol — the HR-led logic reads hot-but-motionless +sleep as wake. This over-calls WAKE (tanking efficiency, Rest, and the debt ledger) and mis-places +sleep onset, and it has required manual relabeling on multiple confirmed nights. + +The motion-aware wake refinement in **#402** (merged 2026-07-13) does not prevent this, for three +compounding reasons detailed below. + +## Evidence — two confirmed nights (both manually corrected) + +**Night A (2026-07-13).** A supplement protocol held overnight resting HR at ~55–60 (vs a natural +~47–51) and skin temp +0.4…+1.2 °C. The scorer produced **194 min WAKE** across a ~6.4 h in-bed span +(efficiency ≈ 0.50). Minute-level review of the largest WAKE blocks found **zero walking cadence**: +each was a brief turn-over burst (a single-minute posture-variance spike) bracketed by minutes of +complete stillness (per-minute posture variance < 0.05, no step ticks), recurring on a ~90-min rhythm +consistent with hot-but-atonic REM being scored as wake. Correct WAKE was ~67 min. + +**Night B (2026-07-14, PDT).** Onset was scored **1:41 AM vs a sensor truth of ~1:29** (wrist still +from 1:29:00, HR at 50 by 1:31), and a **44-min solid WAKE block 1:41–2:27** was emitted where motion +shows stillness with occasional turn-overs and HR 55–65. Sleep end (7:32:38) was correct. + +The discriminating pattern, in both nights: +- **Sleeping:** per-minute posture variance < 0.01–0.05 with **zero** walk ticks; HR flat 50–65. +- **Genuinely awake / out of bed:** sustained walking cadence (walk ticks > 0 held for minutes, e.g. + 35–53 steps/min) **or** sustained high posture variance **with** an HR jump (79–100 bpm, 46–109 + steps/min). + +Elevated-but-**flat** HR with a motionless wrist is not in the "awake" column. + +## Why #402 does not prevent this + +1. **Default-OFF.** The refinement ships behind `PuffinExperiment.motionAwareWakeEnabled`, default off. + A wearer on a supplement protocol gets no protection unless they know to toggle an Experimental flag. +2. **Wrong layer.** It is a **post-pass over an already-staged hypnogram** — it only reclassifies WAKE + *stage segments inside an already-accepted session* to `light`. It never participates in + `detectSleep` / `confirmSleepWithHR`, so it cannot fix Night B's late **onset** or protect a session + the HR gate rejected outright. And on Night A the over-called WAKE is produced by the **stager's own + HR-led emission model**, upstream of any post-pass tuning. +3. **Requires dense STEP data.** Its density self-gate (`isMotionDense`) needs ≥80% of minutes to carry + `StepSample` records with an `activityClass`. Its corroboration keys on **walk-class step ticks**. + When that stream is sparse or the activity class is unattributed, the gate declines and the pass is a + no-op — exactly on the nights that need it. + +## Proposed fix — motion-corroborated wake, on by default, at both layers + +The rule: **elevated HR alone is insufficient to call wake.** An epoch or run at the night's quiescent +**motion** floor with **unchanged posture** cannot be scored WAKE on cardiac evidence alone. Corroborate +with the **gravity posture/jerk** signal both stagers already consume (always present), not step ticks. + +- **(a) Stage level (default `SleepStagerV2`).** On a **motion-quiescent** epoch (no observed movement; + peak per-second jerk at/below the night's own quiescent floor × the wake-gate multiplier), clamp the + cardiac contribution to the AWAKE emission to ≤ 0 — keep any wake-*suppressing* (low, flat HR) evidence, + discard the wake-*promoting* half. Genuine motion (the movement z-score and the night-relative jerk + gate, which by construction cannot fire on a quiescent epoch) still drives wake. A still low-HR sleep + epoch is byte-identical; only a still epoch whose *elevated* HR was about to push it awake is held. + +- **(b) Detection level (`confirmSleepWithHR`).** When a run is **deeply motion-quiescent** (≥ ~90% of its + dense-gravity minutes are posture-stable, per-minute variance < 0.05), widen the HR sleep band from + ×1.05 to ×1.30, so a supplement-elevated but motionless overnight run is not rejected. The band still + keeps a **floor**: a run whose median HR is above even the widened band is rejected even when still, so + genuine all-night in-bed wakefulness is not scored asleep. With no gravity evidence the strict band + stands (unchanged). + +- **(c) Personalised adaptive HR baseline.** Derive the sleep band from the wearer's **recent overnight + medians** (`adaptiveOvernightHRBaseline`) rather than a fixed day-median, so a sustained supplement / + fitness era self-calibrates; keep a floor so a wakeful era can't collapse the band. + +Out-of-bed detection is untouched: sustained walking cadence and sustained posture change with an HR jump +still break runs and score WAKE. + +## Notes + +- Swift lands first (`Packages/StrandAnalytics`); the Kotlin analytics twin + (`com.noop.analytics`) transcription is a deliberate follow-up to preserve the parity contract. +- All thresholds are named constants fixed a-priori from the two reference nights, not fit to labels, + consistent with the repo's rule for a derived physiological signal. diff --git a/docs/scoring-precision/daily-stress-autonomic-redesign.md b/docs/scoring-precision/daily-stress-autonomic-redesign.md new file mode 100644 index 0000000000..0b7baf6479 --- /dev/null +++ b/docs/scoring-precision/daily-stress-autonomic-redesign.md @@ -0,0 +1,217 @@ +# Daily Stress & autonomic scoring — redesign, illness detection, predictive analytics + +Status: **DailyStressEngine implemented + validated** (this file's §3). Illness/overreach flag (§4) and +predictive layer (§5) are **designed, future work**. Grounded in a four-stream literature review +(HRV sports-science; Oura/WHOOP methodology; wearable illness detection; predictive ML), all cited inline. + +--- + +## 0. Thesis + +The old daily Stress score (`StressModel`/`StressMath` in `Strand/Screens/StressView.swift`) summed two +z-scores — `zRHR = (todayRHR − meanRHR)/sdRHR` and `zHRV = (meanHRV − todayHRV)/sdHRV` — computed with a +**naive arithmetic mean + population SD on RAW HRV milliseconds** over a single 30-day window. Four things +are wrong with that, and one deeper problem sits underneath all of them. + +The four mechanical defects (each independently confirmed in the literature): +1. **Raw ms, not log.** RMSSD is right-skewed; a symmetric z on raw ms is statistically invalid and + outlier-dominated. +2. **Naive mean+SD.** No robustness to a freak night; no σ-floor, so a tight baseline saturates the logistic. +3. **Single 30-day window.** Forces the adapt-vs-hide tradeoff (below). +4. **Equal-weight RHR+HRV sum.** Throws away the *divergence* between the two channels, which is the most + informative part. + +The deeper problem — the **confound**: HRV suppression + RHR elevation is genuine autonomic load, but it is +**"sensitive but not specific"** (Altini 2021, n≈9M, [MDPI Sensors](https://www.mdpi.com/1424-8220/21/23/7932)). +A supplement, a training block, altitude, poor sleep, illness, and psychological stress all produce the same +signature. The physiology *cannot name the cause*. So a daily "stress" score that keys on it will read high +every day a user is on a sustained autonomic-shifting influence — which is exactly what NOOP's own owner's +supplement protocol produces (nightly HRV suppressed ~85 ms vs a historical ~120–155, RHR elevated ~55 vs +~50, sustained 6+ weeks). + +--- + +## 1. The adapt-vs-hide tradeoff (the crux) + +A rolling personal baseline has one knob — the window — and it forces a bad choice: + +- **Adapt (short window).** The suppressed level becomes "my recent normal" within ~2 weeks, so the daily + score self-normalizes and stops crying "high stress." **But the sustained load has been defined away** — + the score can no longer tell the user their baseline itself has moved. +- **Don't adapt (long/fixed window).** The score keeps flagging the shift forever — a permanent false + "high stress" for a benign sustained cause. + +Both Oura and WHOOP default to **adapt** (Oura recalibrates "daily … adapts if your baselines change"; +WHOOP's 14-day window absorbs a shift in ~2–3 weeks) and simply accept the blind spot. Their context tags / +journals annotate but **do not feed the score** (Oura Tags, WHOOP Journal drive a *separate* correlational +"Impacts" view). ([Oura Daytime Stress](https://ouraring.com/blog/inside-the-ring-daytime-stress/), +[WHOOP behaviors](https://www.whoop.com/us/en/thelocker/a-new-way-to-see-insights-on-which-behaviors-affect-your-recovery/)) + +**Resolution: dual baselines.** A short adaptive window drives the daily 0–3 (so it normalizes), and a long +reference powers a *separate* honest readout — "your baseline itself has shifted N% for D days." Neither +vendor publicly does this; it is the only clean escape from the tradeoff, and it is what §3 implements. + +--- + +## 2. Research foundation (cited) + +### 2.1 HRV sports-science +- **Log-domain is mandatory.** Raw RMSSD test–retest CV ≈ 30%; lnRMSSD ≈ 3–13% (e.g. 5.4% over 16 wk in + athletes). lnRMSSD is near-normal, making a symmetric z valid. (Plews 2013 [PMID 23852425](https://pubmed.ncbi.nlm.nih.gov/23852425/); + Altini; WHOOP water-polo reliability [PMC9505647](https://pmc.ncbi.nlm.nih.gov/articles/PMC9505647/).) +- **Smallest Worthwhile Change (SWC) = mean ± 0.5 SD** (Hopkins 2000; Buchheit 2014) — inside the band is + noise, don't act. +- **Short vs long window is the right structure for acute-vs-chronic** (Altini): acute = 7d dips below the + 60d band then recovers; chronic = the 60d band itself re-centers, after which the daily z auto-returns to 0. +- **Two windows do two jobs**: 7d = operational baseline; 60d = normal-range band (HRV4Training uses 60d). + +### 2.2 The coupled quadrant (the single most important design decision) +**Buchheit 2014** ([10.3389/fphys.2014.00073](https://www.frontiersin.org/articles/10.3389/fphys.2014.00073/full)) +— interpret lnRMSSD change *jointly* with RHR: + +| lnRMSSD | RHR | meaning | +|---|---|---| +| ↑ | ↓ | coping well / parasympathetic adaptation | +| ↓ | **↑** | **sympathetic overactivity = real stress/fatigue** | +| ↓ | ↓ | **parasympathetic saturation (benign)** — or overtraining only if prolonged | +| ↑ | ↑ | taper / sympathetic reversal | + +Low HRV is only "stress" in the **↓/↑** cell. Low HRV with **low** RHR is benign saturation. This is the exact +logic NOOP's recovery-saturation guard (upstream #461) already uses on Charge — it belongs in Stress too. +HRV should carry most of the weight (more sensitive); RHR is decisive precisely when it *diverges* +(Altini; WHOOP: RHR "adds signal mainly when it trends differently from HRV"). + +### 2.3 Oura / WHOOP (what they actually do) +WHOOP: lnRMSSD in deep sleep vs a 14-day baseline; HRV-primary; 0–3 stress scale. Oura: personal baseline +recalibrated daily, windows undisclosed, "cannot distinguish psychological stress from exercise, caffeine, or +other sympathetic activation" (their own words). Neither publishes the weighting, outlier handling, or a +dual-baseline. Their blogs are ~90% marketing; the rigor is in the HRV literature, not the vendor copy. + +### 2.4 Illness detection → §4. ### 2.5 Predictive ML → §5. + +--- + +## 3. Implemented: `DailyStressEngine` (StrandAnalytics) + +A pure, tested package engine replacing the inline `StressMath`. Four evidence-backed changes: + +1. **Log-domain HRV** — z-scores lnRMSSD via the shared spine's ln config (`readiness_hrv_ln`). +2. **Dual baselines** — a **14-night** adaptive baseline drives the daily 0–3; a **60-night** reference powers + a separate `ChronicShift` readout. Both fold through the RD2 **window-fold spine mode** + (`Baselines.foldHistory(..., rejectHardOutliers: false)`) so a sustained shift *adapts* (isn't rejected) + while a single freak night is Winsor-*damped*. +3. **Coupled escalation** — `saturationDamp(hrvZ:rhrZ:)` mirrors the #461 recovery guard: when HRV is + suppressed but RHR is **not** elevated, the HRV stress term is damped (benign saturation). Weight + HRV 0.6 / RHR 0.4. A `Quadrant` (balanced / sympatheticStress / parasympatheticSaturation / recovered) + carries the interpretation. +4. **SWC dead-zone** — a combined |raw| under 0.5σ reads as baseline (1.5), not stress. + +**`ChronicShift`** reports `hrvPctBelowLongTerm`, `rhrBpmAboveLongTerm`, `isSustainedLoad` (both channels +shifted together — the coupled sustained-load gate), and `daysBelowBand`. + +### Real-data validation (owner's 420-night history, replayed) +- **2026-07-10**: daily score **1.50 (neutral)** — the short baseline had adapted to the suppressed level — + while the chronic readout showed **HRV −31% below the 60-day normal**. The reframe working exactly as + designed: the daily number stops crying "stress"; the sustained load surfaces separately. +- **2026-07-13**: **2.02** — HRV genuinely dipped below the recent baseline, so a fresh elevation is correct. +- **Supplement onset (2026-06-08)**: **2.87** — the acute crash correctly reads high. +- **Coupling proven**: an identical HRV drop reads lower and classifies `parasympatheticSaturation` when RHR + is low, vs `sympatheticStress` when RHR is high. +- **Key finding**: it is the **short adaptive window**, not the log transform, that reframes the number — an + earlier single-30d-window A/B showed log-domain alone was a ~no-op on the owner's score. + +**Honest caveat for the owner specifically**: his recent RHR *is* elevated above his own baseline (≈55 vs +≈50), so he sits in the genuine **↓HRV/↑RHR sympathetic-load** cell — the redesign does not (and should not) +make his number "calm"; it *reframes* the sustained part as a baseline shift and keeps the daily number +honest about fresh dips. + +### Not yet wired (validated follow-up) +`StressModel` still calls the old `StressMath`. Wiring it to `DailyStressEngine` swaps the **primary visible +Today-card number** and should ship *with* a minimal UI surfacing of `ChronicShift` (so a lower daily number +is explained by the "baseline shifted" line), validated on real data. Sequenced deliberately after this +engine + doc land. + +--- + +## 4. Future work — signed illness / overreach flag (M7) + +The current skin-temp / respiration terms in Recovery are **symmetric penalties** (a +0.5 °C fever scores the +same as −0.5 °C benign cooling), discarding the directional illness signal. The research is clear and the fix +is well-specified. + +**Physiology (directional, concordant, ~1–3 days pre-symptom):** skin temp ↑, respiratory rate ↑, RHR ↑, +HRV ↓. Respiratory rate is the **highest-SNR anchor** (baseline varies <1 rpm; illness +2–3 rpm/~17%) and is +rarely moved by alcohol/caffeine/poor sleep — the ideal disambiguator. Temperature adds the most +discriminating power when present. (TemPredict/Oura-UCSF, *Sci Rep* 2022 [PMC8891385](https://pmc.ncbi.nlm.nih.gov/articles/PMC8891385/): +AUC 0.819, sens 82%, **spec 63%**, ~2.75 d lead; Stanford/Snyder *Nat Biomed Eng* 2020 +[PMC9020268](https://pmc.ncbi.nlm.nih.gov/articles/PMC9020268/), real-time CuSum caught 62.5%; Fitbit/Scripps +DETECT *Nat Med* 2021.) + +**Design:** +1. **Signed / one-sided** on temp & RR — only sustained **rises** accrue illness evidence; falls are neutral. +2. **Robust personal baseline** (~14–28 nights, median/MAD), excluding known-confounded nights. +3. **Physiology-anchored thresholds**: temp > +0.3–0.5 °C (above the ~0.3–0.5 °C luteal floor), RR > +1.5–2 + rpm / >10–15%, RHR > +5–7 bpm, HRV a meaningful SD below. +4. **Concordance + persistence gating**: `major` only when ≥3 of {temp↑, RR↑, RHR↑, HRV↓} agree and hold + ≥2 nights; `minor` for one strong/transient signal. Weight temp + RR highest (most infection-specific). +5. **Context attribution, not suppression**: annotate the likely driver (logged supplement, hard training, + travel/altitude, luteal phase) rather than hiding the flag; cycle-adjust temperature (the one benign cause + that mimics fever). +6. **Surface separately** from the score (a badge beside it), never folded in — Oura keeps Symptom Radar + separate from Readiness for exactly this reason. + +**Honesty bar**: even the best peer-reviewed models sit at 60–70% specificity / low PPV (4–10% in independent +validation). Ship as "a signal worth attention," never a diagnosis, and never advertise an unvalidated +lead-time/accuracy number. Much of the naive apparent signal is *behavior change after learning you're sick* +(Cleary 2022, AUC 0.75→0.63 on pre-result-only data). + +This flag **directly sharpens the daily stress score too**: a concordant temp+RR+RHR rise is the one context +where "this sustained load is likely illness/overreach, not a benign supplement" can be said honestly. + +--- + +## 5. Future work — predictive analytics (P0/P1/P2) + +The ML review's blunt conclusion: **for a per-user, on-device, privacy-first, inspectable-math app, the +honest state of the art is transparent statistical anomaly detection on a personal baseline — not deep +learning.** Every vendor "predictive readiness/recovery" score is, on inspection, a **nowcast** (a reweighted +baseline-deviation of last night's data relabeled as prediction). The only genuinely forward-looking task +with replicated evidence — illness 1–3 days out — is best solved by change-point/CuSum methods that run +trivially on-device. + +Evidence that simple beats complex *on this kind of noisy, small-N, per-user series*: M4 Competition (best +pure-ML method less accurate than the worst statistical one); Makridakis 2018; DLinear (a one-layer linear +model beat every Transformer, AAAI 2023). Per-user deep forecasting is data-starved (one user ≈ hundreds–2k +nightly points) and there is **no** evidence it beats a tuned EWMA on nightly HRV. + +**Roadmap:** +- **P0 — dual-timescale trend + change-point (largely DONE via §3).** Fast EWMA (~3–5 d) vs slow baseline + (~28–60 d) of lnRMSSD + RHR, a CV band, and a change-point flag for sustained shifts. `DailyStressEngine`'s + dual baselines + `ChronicShift` already are this; a formal online change-point (CuSum) on the short-vs-long + gap would complete it and give an explicit "your baseline shifted on ~date" event. +- **P1 — personal-baseline early-illness detector (= §4).** Multivariate deviation on RHR + RR + skin temp, + NightSignal/CuSum-style. The one task with real multi-day lead time; transparent methods *are* the SOTA. +- **P2 — optional cold-start prior** (a tiny shipped population prior to bridge the first ~14 days). The one + place a shared/pretrained component is justified; keep it small and swap to the personal baseline ASAP. + +**Explicitly do NOT build**: per-user LSTM/transformer next-day forecasters (data-starved, unfalsifiable, +contradicts inspectable-math), foundation-model "digital twins" (need cloud scale, break privacy-first), or +confident overtraining/illness *classifier* verdicts (science equivocal / PPV too low). This aligns with the +app's inspectable-math philosophy — the constraint is a feature. + +--- + +## 6. Unsettled science (flagged honestly) +- Prospective illness specificity is mediocre (~60–70%); false positives from alcohol/training/travel/luteal. +- CV-of-lnRMSSD direction with fatigue is U-shaped and context-dependent — keep it a *secondary* signal. +- The SWC 0.5 multiplier and EWMA-vs-simple-mean are conventions, not validated optima. +- Nearly all cited thresholds come from morning-supine athlete recordings, not nocturnal wrist averages. +- Saturation vs parasympathetic-overtraining (both ↓HRV/↓RHR) can't be separated without external context. + +--- + +## 7. Sources +See inline links. Anchors: Buchheit 2014 `10.3389/fphys.2014.00073`; Plews 2013 PMID 23852425; Altini 2021 +Sensors `10.3390/s21237932`; Hopkins 2000 (SWC); TemPredict PMC8891385; Snyder PMC9020268; DETECT *Nat Med* +2021; Alavi 2022 *Nat Med* PMC8799466; Quer 2020 DETECT (RHR-alone AUC 0.52); Makridakis 2018; M4 2020; +Zeng 2023 DLinear (arXiv:2205.13504). diff --git a/docs/superpowers/plans/2026-07-10-noop-cloud-1-mvp-server.md b/docs/superpowers/plans/2026-07-10-noop-cloud-1-mvp-server.md new file mode 100644 index 0000000000..cda0b4ecef --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-noop-cloud-1-mvp-server.md @@ -0,0 +1,1720 @@ +# noop-cloud Phase-1 MVP Server — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A self-hostable remote MCP server that mirrors the NOOP phone database (uploaded as a `.noopbak`) and exposes read-only biometrics tools to Claude/ChatGPT/agents over Streamable HTTP, deployed on a single Fly.io machine, with a scheduled morning report. + +**Architecture:** One Node/TypeScript process behind Fly's HTTPS proxy. `POST /ingest` (rw bearer) accepts a `.noopbak` ZIP, validates it, and atomically swaps it into `/data/mirror.sqlite`. `POST /mcp` (ro bearer) is a **stateless** Streamable HTTP MCP endpoint whose tools open `mirror.sqlite` read-only. `/data/server.sqlite` holds only an ingest log in Phase 1. A GitHub Actions cron calls the Anthropic Messages API (MCP connector, ro token) and pushes the report to ntfy. No writes, no OAuth, no sync-back in this phase. + +**Tech Stack:** TypeScript 5.6, Node 20, Express 4, `@modelcontextprotocol/sdk` **pinned 1.29.0**, `better-sqlite3` (native, pinned), `adm-zip`, `zod`, `vitest`. Fly.io (1 machine + 1 volume, WAL SQLite, autostop). Docker deploy. + +## Global Constraints + +- **MCP SDK pinned to exactly `1.29.0`** (`"@modelcontextprotocol/sdk": "1.29.0"`, no `^`). Do NOT use the 2.0.0-beta line. Rationale: the 2026-07-28 stateless spec revision is still an RC; 1.29.x is the supported stable line. +- **`better-sqlite3` pinned exact** (`"better-sqlite3": "12.11.1"`, no `^`) — native binding; a floating range risks an ABI mismatch on the Fly builder. Verified locally: 12.11.1 installs + loads on this machine's **Node 26 (ABI 147)**, and 12.x still ships a **Node-20 prebuilt** for the `node:20-slim` Docker image. `npm rebuild better-sqlite3` is the fix if a prebuilt binding ever mismatches the runtime Node. +- **Stateless MCP transport**: every `/mcp` request constructs a fresh `McpServer` + `StreamableHTTPServerTransport({ sessionIdGenerator: undefined })`. No session store — a Fly autostop between requests must not break a session. +- **The mirror is opened READ-ONLY always** (`new Database(path, { readonly: true, fileMustExist: true })`). Server code never writes to `mirror.sqlite`. Server-owned state lives only in `server.sqlite`. +- **`deviceId` is a *source* discriminator, not a physical device.** The same calendar day legitimately has rows under multiple `deviceId`s. Source families for Phase 1: WHOOP = any id that is NOT `apple-health` and NOT `oura-api` (straps use ids like `my-whoop`, plus derived `-noop`); Oura = `oura-api`; Apple = `apple-health`. Never assume one row per day. +- **Two static bearer tokens** from env: `RO_TOKEN` (read; `/mcp`) and `RW_TOKEN` (ingest; `/ingest`). Compare with `crypto.timingSafeEqual`. `rw` implies `ro`. Tokens are ≥32 bytes; never logged. +- **Ingest hard limit**: reject bodies over `MAX_INGEST_BYTES` (default 250 MB). Validate the inner SQLite by magic bytes (`SQLite format 3\0`) + `PRAGMA quick_check`. Reject a foreign DB (no `grdb_migrations` table). +- **No secrets in git.** `.env` is gitignored; `.env.example` documents keys. Fly secrets via `fly secrets set`. +- **License/voice:** new repo, MIT license, neutral third-person README. This is a standalone sibling repo at `/Users/vk/VKDEV/NOOP/noop-cloud` — NOT inside the NOOP tree, its own git. + +--- + +### Task 1: Repo scaffold, config, and `/healthz` + +**Files:** +- Create: `noop-cloud/package.json` +- Create: `noop-cloud/tsconfig.json` +- Create: `noop-cloud/vitest.config.ts` +- Create: `noop-cloud/.gitignore` +- Create: `noop-cloud/.env.example` +- Create: `noop-cloud/src/config.ts` +- Create: `noop-cloud/src/server.ts` +- Test: `noop-cloud/test/healthz.test.ts` + +**Interfaces:** +- Produces: `loadConfig(): Config` where `Config = { port: number; dataDir: string; mirrorPath: string; serverDbPath: string; roToken: string; rwToken: string; maxIngestBytes: number }`. +- Produces: `createApp(cfg: Config): express.Express` — the Express app (routes added in later tasks), with `GET /healthz` returning `{ ok: true }`. + +- [ ] **Step 1: Initialize the repo and pinned dependencies** + +```bash +mkdir -p /Users/vk/VKDEV/NOOP/noop-cloud && cd /Users/vk/VKDEV/NOOP/noop-cloud +git init -q +``` + +Create `package.json`: + +```json +{ + "name": "noop-cloud", + "version": "0.1.0", + "private": true, + "type": "module", + "license": "MIT", + "engines": { "node": ">=20" }, + "scripts": { + "build": "tsc -p tsconfig.json", + "start": "node dist/server.js", + "dev": "tsx watch src/server.ts", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "1.29.0", + "adm-zip": "0.5.16", + "better-sqlite3": "12.11.1", + "express": "^4.21.2", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/adm-zip": "^0.5.5", + "@types/better-sqlite3": "^7.6.11", + "@types/express": "^4.17.21", + "@types/node": "^20.14.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0", + "vitest": "^2.1.0" + } +} +``` + +- [ ] **Step 2: Add tsconfig, vitest config, gitignore, env template** + +`tsconfig.json`: + +```json +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": false, + "sourceMap": true + }, + "include": ["src"] +} +``` + +`vitest.config.ts`: + +```typescript +import { defineConfig } from "vitest/config"; +export default defineConfig({ test: { include: ["test/**/*.test.ts"], testTimeout: 20000 } }); +``` + +`.gitignore`: + +``` +node_modules/ +dist/ +.env +data/ +*.sqlite +*.sqlite-wal +*.sqlite-shm +test/.tmp/ +``` + +`.env.example`: + +``` +PORT=8080 +DATA_DIR=./data +RO_TOKEN=replace-with-32+-byte-random-read-token +RW_TOKEN=replace-with-32+-byte-random-write-token +MAX_INGEST_BYTES=262144000 +``` + +- [ ] **Step 3: Write the failing test for config + `/healthz`** + +`test/healthz.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import request from "node:http"; +import { loadConfig } from "../src/config.js"; +import { createApp } from "../src/server.js"; + +describe("scaffold", () => { + it("loadConfig throws when RO_TOKEN missing", () => { + const prev = process.env.RO_TOKEN; + delete process.env.RO_TOKEN; + expect(() => loadConfig()).toThrow(/RO_TOKEN/); + if (prev !== undefined) process.env.RO_TOKEN = prev; + }); + + it("GET /healthz returns ok", async () => { + process.env.RO_TOKEN = "ro-token-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + process.env.RW_TOKEN = "rw-token-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + const app = createApp(loadConfig()); + const server = app.listen(0); + const port = (server.address() as any).port; + const body = await new Promise((resolve, reject) => { + request.get(`http://127.0.0.1:${port}/healthz`, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => resolve(d)); + }).on("error", reject); + }); + server.close(); + expect(JSON.parse(body)).toEqual({ ok: true }); + }); +}); +``` + +- [ ] **Step 4: Run the test to verify it fails** + +Run: `npm install && npm test` +Expected: FAIL — `Cannot find module '../src/config.js'`. + +- [ ] **Step 5: Implement `src/config.ts`** + +```typescript +import path from "node:path"; + +export interface Config { + port: number; + dataDir: string; + mirrorPath: string; + serverDbPath: string; + roToken: string; + rwToken: string; + maxIngestBytes: number; +} + +function required(name: string): string { + const v = process.env[name]; + if (!v || v.length < 16) throw new Error(`${name} must be set (>=16 chars)`); + return v; +} + +export function loadConfig(): Config { + const dataDir = process.env.DATA_DIR ?? "./data"; + return { + port: Number(process.env.PORT ?? 8080), + dataDir, + mirrorPath: path.join(dataDir, "mirror.sqlite"), + serverDbPath: path.join(dataDir, "server.sqlite"), + roToken: required("RO_TOKEN"), + rwToken: required("RW_TOKEN"), + maxIngestBytes: Number(process.env.MAX_INGEST_BYTES ?? 262_144_000), + }; +} +``` + +- [ ] **Step 6: Implement `src/server.ts`** + +```typescript +import express from "express"; +import fs from "node:fs"; +import { Config, loadConfig } from "./config.js"; + +export function createApp(cfg: Config): express.Express { + fs.mkdirSync(cfg.dataDir, { recursive: true }); + const app = express(); + app.get("/healthz", (_req, res) => res.json({ ok: true })); + return app; +} + +// Entry point (ignored by tests, which import createApp directly). +if (process.argv[1] && process.argv[1].endsWith("server.js")) { + const cfg = loadConfig(); + createApp(cfg).listen(cfg.port, () => console.log(`noop-cloud on :${cfg.port}`)); +} +``` + +- [ ] **Step 7: Run tests + build to verify pass** + +Run: `npm test && npm run build` +Expected: PASS (2 tests); `dist/` compiles with no errors. + +- [ ] **Step 8: Commit** + +```bash +git add -A && git commit -m "chore: scaffold noop-cloud (config + /healthz + pinned deps)" +``` + +--- + +### Task 2: Bearer auth middleware (ro/rw scopes) + +**Files:** +- Create: `noop-cloud/src/auth.ts` +- Test: `noop-cloud/test/auth.test.ts` + +**Interfaces:** +- Consumes: `Config` (`roToken`, `rwToken`) from Task 1. +- Produces: `requireScope(cfg: Config, scope: "ro" | "rw"): express.RequestHandler`. `rw` token satisfies both scopes; `ro` token satisfies only `ro`. Missing/malformed/incorrect → 401 JSON `{ error: "unauthorized" }`. Uses `crypto.timingSafeEqual`. + +- [ ] **Step 1: Write the failing test** + +`test/auth.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import express from "express"; +import http from "node:http"; +import { requireScope } from "../src/auth.js"; + +const cfg = { roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y") } as any; + +function call(handlerScope: "ro" | "rw", header?: string): Promise { + const app = express(); + app.get("/p", requireScope(cfg, handlerScope), (_r, res) => res.json({ ok: true })); + const server = app.listen(0); + const port = (server.address() as any).port; + return new Promise((resolve) => { + http.get({ port, path: "/p", headers: header ? { authorization: header } : {} }, (res) => { + server.close(); resolve(res.statusCode!); + }); + }); +} + +describe("requireScope", () => { + it("401 with no header", async () => expect(await call("ro")).toBe(401)); + it("401 with wrong token", async () => expect(await call("ro", "Bearer nope")).toBe(401)); + it("ro token satisfies ro scope", async () => expect(await call("ro", `Bearer ${cfg.roToken}`)).toBe(200)); + it("rw token satisfies ro scope", async () => expect(await call("ro", `Bearer ${cfg.rwToken}`)).toBe(200)); + it("ro token does NOT satisfy rw scope", async () => expect(await call("rw", `Bearer ${cfg.roToken}`)).toBe(401)); + it("rw token satisfies rw scope", async () => expect(await call("rw", `Bearer ${cfg.rwToken}`)).toBe(200)); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- auth` +Expected: FAIL — `Cannot find module '../src/auth.js'`. + +- [ ] **Step 3: Implement `src/auth.ts`** + +```typescript +import crypto from "node:crypto"; +import type { RequestHandler } from "express"; +import type { Config } from "./config.js"; + +function safeEqual(a: string, b: string): boolean { + const ab = Buffer.from(a), bb = Buffer.from(b); + if (ab.length !== bb.length) return false; + return crypto.timingSafeEqual(ab, bb); +} + +export function requireScope(cfg: Config, scope: "ro" | "rw"): RequestHandler { + return (req, res, next) => { + const h = req.header("authorization") ?? ""; + const m = /^Bearer (.+)$/.exec(h); + if (!m) return res.status(401).json({ error: "unauthorized" }); + const token = m[1]; + const isRw = safeEqual(token, cfg.rwToken); + const isRo = safeEqual(token, cfg.roToken); + const ok = scope === "rw" ? isRw : isRo || isRw; + if (!ok) return res.status(401).json({ error: "unauthorized" }); + next(); + }; +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test -- auth` +Expected: PASS (6 tests). + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: bearer auth middleware with ro/rw scopes" +``` + +--- + +### Task 3: Test fixtures — a GRDB-shaped mirror + a `.noopbak` + +**Files:** +- Create: `noop-cloud/test/fixtures/make-fixture.ts` +- Test: `noop-cloud/test/fixtures/make-fixture.test.ts` + +**Interfaces:** +- Produces: `buildMirrorSqlite(path: string): void` — writes a SQLite file with the mirror schema and multi-source rows (WHOOP `my-whoop`, `oura-api`, `apple-health`) across days `2026-06-10..2026-06-13`, including a `grdb_migrations` table (origin marker). +- Produces: `buildNoopbak(path: string): void` — a ZIP with entry `noop-backup.sqlite` (the mirror bytes) first. Also `buildNoopbakFrom(sqlitePath, zipPath)`. +- Schema (subset sufficient for Phase-1 tools), tables + keys: `grdb_migrations(identifier TEXT)`, `hrSample(deviceId,ts,bpm)`, `dailyMetric(deviceId,day,totalSleepMin,efficiency,restingHr,avgHrv,recovery,strain,spo2Pct,skinTempDevC,respRateBpm,steps,activeKcalEst)`, `sleepSession(deviceId,startTs,endTs,efficiency,restingHr,avgHrv,stagesJSON,userEdited,startTsAdjusted)`, `workout(deviceId,startTs,sport,endTs,source,durationS,energyKcal,distanceM)`, `metricSeries(deviceId,day,key,value)`, `appleDaily(deviceId,day,steps,restingHr)`, `pairedDevice(id,brand,model,sourceKind)`. + +- [ ] **Step 1: Write the failing test** + +`test/fixtures/make-fixture.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import Database from "better-sqlite3"; +import AdmZip from "adm-zip"; +import { buildMirrorSqlite, buildNoopbak } from "./make-fixture.js"; + +const tmp = path.join(process.cwd(), "test/.tmp"); +beforeAll(() => fs.mkdirSync(tmp, { recursive: true })); + +describe("fixtures", () => { + it("mirror has multi-source dailyMetric rows and grdb_migrations", () => { + const p = path.join(tmp, "m.sqlite"); + buildMirrorSqlite(p); + const db = new Database(p, { readonly: true }); + const sources = db.prepare("SELECT DISTINCT deviceId FROM dailyMetric ORDER BY deviceId").all().map((r: any) => r.deviceId); + expect(sources).toContain("oura-api"); + expect(sources).toContain("apple-health"); + expect(sources.some((s: string) => s !== "oura-api" && s !== "apple-health")).toBe(true); + expect(db.prepare("SELECT count(*) c FROM grdb_migrations").get() as any).toHaveProperty("c"); + db.close(); + }); + + it("noopbak is a zip whose FIRST entry is noop-backup.sqlite", () => { + const p = path.join(tmp, "b.noopbak"); + buildNoopbak(p); + const entries = new AdmZip(p).getEntries(); + expect(entries[0].entryName).toBe("noop-backup.sqlite"); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- make-fixture` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `test/fixtures/make-fixture.ts`** + +```typescript +import Database from "better-sqlite3"; +import AdmZip from "adm-zip"; +import fs from "node:fs"; + +const DAYS = ["2026-06-10", "2026-06-11", "2026-06-12", "2026-06-13"]; +const tsOf = (day: string, h = 3) => Math.floor(new Date(`${day}T${String(h).padStart(2, "0")}:00:00Z`).getTime() / 1000); + +export function buildMirrorSqlite(target: string): void { + if (fs.existsSync(target)) fs.rmSync(target); + const db = new Database(target); + db.pragma("journal_mode = WAL"); + db.exec(` + CREATE TABLE grdb_migrations (identifier TEXT PRIMARY KEY); + CREATE TABLE hrSample (deviceId TEXT, ts INTEGER, bpm INTEGER, PRIMARY KEY(deviceId, ts)); + CREATE TABLE dailyMetric (deviceId TEXT, day TEXT, totalSleepMin INTEGER, efficiency REAL, + restingHr REAL, avgHrv REAL, recovery REAL, strain REAL, spo2Pct REAL, skinTempDevC REAL, + respRateBpm REAL, steps INTEGER, activeKcalEst REAL, PRIMARY KEY(deviceId, day)); + CREATE TABLE sleepSession (deviceId TEXT, startTs INTEGER, endTs INTEGER, efficiency REAL, + restingHr REAL, avgHrv REAL, stagesJSON TEXT, userEdited INTEGER DEFAULT 0, + startTsAdjusted INTEGER, PRIMARY KEY(deviceId, startTs)); + CREATE TABLE workout (deviceId TEXT, startTs INTEGER, sport TEXT, endTs INTEGER, source TEXT, + durationS REAL, energyKcal REAL, distanceM REAL, PRIMARY KEY(deviceId, startTs, sport)); + CREATE TABLE metricSeries (deviceId TEXT, day TEXT, key TEXT, value REAL, PRIMARY KEY(deviceId, day, key)); + CREATE TABLE appleDaily (deviceId TEXT, day TEXT, steps INTEGER, restingHr REAL, PRIMARY KEY(deviceId, day)); + CREATE TABLE pairedDevice (id TEXT PRIMARY KEY, brand TEXT, model TEXT, sourceKind TEXT); + `); + db.prepare("INSERT INTO grdb_migrations VALUES (?)").run("v25-oura-raw"); + const pd = db.prepare("INSERT INTO pairedDevice VALUES (?,?,?,?)"); + pd.run("my-whoop", "WHOOP", "5.0", "strap"); + pd.run("oura-api", "Oura", "Oura (cloud)", "cloudImport"); + pd.run("apple-health", "Apple", "Apple Health", "appleHealth"); + + const dm = db.prepare(`INSERT INTO dailyMetric + (deviceId,day,totalSleepMin,efficiency,restingHr,avgHrv,recovery,strain,spo2Pct,skinTempDevC,respRateBpm,steps,activeKcalEst) + VALUES (@deviceId,@day,@totalSleepMin,@efficiency,@restingHr,@avgHrv,@recovery,@strain,@spo2Pct,@skinTempDevC,@respRateBpm,@steps,@activeKcalEst)`); + const base = (deviceId: string, day: string, o: Partial) => ({ + deviceId, day, totalSleepMin: 420, efficiency: 90, restingHr: 52, avgHrv: 65, recovery: null, + strain: null, spo2Pct: 97, skinTempDevC: -0.1, respRateBpm: 14.5, steps: 8000, activeKcalEst: 500, ...o, + }); + for (const day of DAYS) { + // WHOOP is the scored source: recovery/strain present. + dm.run(base("my-whoop", day, { restingHr: 51, avgHrv: 70, recovery: 66, strain: 12.5 })); + // Oura cloud: honest data, recovery/strain null; slightly different RHR/HRV. + dm.run(base("oura-api", day, { restingHr: 53, avgHrv: 62 })); + // Apple: steps only-ish. + dm.run(base("apple-health", day, { restingHr: 54, avgHrv: null, steps: 8200 })); + } + const ss = db.prepare(`INSERT INTO sleepSession + (deviceId,startTs,endTs,efficiency,restingHr,avgHrv,stagesJSON,userEdited,startTsAdjusted) + VALUES (?,?,?,?,?,?,?,?,?)`); + for (const day of DAYS) { + ss.run("my-whoop", tsOf(day, 3), tsOf(day, 3) + 25200, 90, 51, 70, null, 0, null); + ss.run("oura-api", tsOf(day, 3) + 60, tsOf(day, 3) + 25000, 88, 53, 62, null, 0, null); + } + const wo = db.prepare(`INSERT INTO workout (deviceId,startTs,sport,endTs,source,durationS,energyKcal,distanceM) + VALUES (?,?,?,?,?,?,?,?)`); + wo.run("my-whoop", tsOf("2026-06-12", 18), "running", tsOf("2026-06-12", 18) + 1800, "whoop", 1800, 320, 5000); + wo.run("oura-api", tsOf("2026-06-11", 17), "walking", tsOf("2026-06-11", 17) + 2400, "oura", 2400, 180, 3000); + + const ms = db.prepare("INSERT INTO metricSeries VALUES (?,?,?,?)"); + for (const day of DAYS) { + ms.run("oura-api", day, "ref_sleep_score", 82); + ms.run("oura-api", day, "oura_readiness", 78); + } + const hr = db.prepare("INSERT INTO hrSample VALUES (?,?,?)"); + for (const day of DAYS) for (let i = 0; i < 5; i++) hr.run("oura-api", tsOf(day, 3) + i * 300, 55 + i); + db.close(); +} + +export function buildNoopbakFrom(sqlitePath: string, zipPath: string): void { + if (fs.existsSync(zipPath)) fs.rmSync(zipPath); + const zip = new AdmZip(); + zip.addFile("noop-backup.sqlite", fs.readFileSync(sqlitePath)); // DB entry FIRST + zip.writeZip(zipPath); +} + +export function buildNoopbak(zipPath: string): void { + const tmpSqlite = zipPath + ".src.sqlite"; + buildMirrorSqlite(tmpSqlite); + buildNoopbakFrom(tmpSqlite, zipPath); + fs.rmSync(tmpSqlite); + for (const ext of ["-wal", "-shm"]) if (fs.existsSync(tmpSqlite + ext)) fs.rmSync(tmpSqlite + ext); +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test -- make-fixture` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "test: fixture builders for mirror sqlite + .noopbak" +``` + +--- + +### Task 4: Mirror read layer + +**Files:** +- Create: `noop-cloud/src/mirror.ts` +- Test: `noop-cloud/test/mirror.test.ts` + +**Interfaces:** +- Consumes: fixtures from Task 3. +- Produces: `class Mirror` opened read-only, methods: + - `sourceFamily(deviceId: string): "whoop" | "oura" | "apple"` (module fn `sourceFamily`). + - `sources(): { deviceId: string; family: string; brand: string | null; latestDay: string | null }[]` + - `dailyMetrics(opts: { deviceId?: string; from: string; to: string }): DailyMetricRow[]` + - `sleepSummary(opts: { from: string; to: string }): SleepRow[]` + - `workoutSummary(opts: { from: string; to: string }): WorkoutRow[]` + - `metricSeries(opts: { deviceId?: string; key?: string; from: string; to: string }): MetricPointRow[]` + - `latestDataDay(): string | null` (max day across `dailyMetric`) + - `hrCoverageDays(deviceId: string): number` + - `close(): void` +- Produces types `DailyMetricRow`, `SleepRow`, `WorkoutRow`, `MetricPointRow` (plain interfaces mirroring the columns; `deviceId` + `family` included on each). + +- [ ] **Step 1: Write the failing test** + +`test/mirror.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildMirrorSqlite } from "./fixtures/make-fixture.js"; +import { Mirror, sourceFamily } from "../src/mirror.js"; + +const p = path.join(process.cwd(), "test/.tmp/mirror-read.sqlite"); +beforeAll(() => { fs.mkdirSync(path.dirname(p), { recursive: true }); buildMirrorSqlite(p); }); + +describe("Mirror", () => { + it("classifies source families", () => { + expect(sourceFamily("oura-api")).toBe("oura"); + expect(sourceFamily("apple-health")).toBe("apple"); + expect(sourceFamily("my-whoop")).toBe("whoop"); + }); + it("lists sources with latest day", () => { + const m = new Mirror(p); + const s = m.sources(); + expect(s.find((x) => x.deviceId === "oura-api")?.latestDay).toBe("2026-06-13"); + m.close(); + }); + it("filters dailyMetrics by source and range", () => { + const m = new Mirror(p); + const rows = m.dailyMetrics({ deviceId: "oura-api", from: "2026-06-11", to: "2026-06-12" }); + expect(rows.length).toBe(2); + expect(rows[0].family).toBe("oura"); + m.close(); + }); + it("reports latest data day across sources", () => { + const m = new Mirror(p); expect(m.latestDataDay()).toBe("2026-06-13"); m.close(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- mirror` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/mirror.ts`** + +```typescript +import Database from "better-sqlite3"; + +export type Family = "whoop" | "oura" | "apple"; +export function sourceFamily(deviceId: string): Family { + if (deviceId === "oura-api") return "oura"; + if (deviceId === "apple-health") return "apple"; + return "whoop"; +} + +export interface DailyMetricRow { + deviceId: string; family: Family; day: string; + totalSleepMin: number | null; efficiency: number | null; restingHr: number | null; + avgHrv: number | null; recovery: number | null; strain: number | null; spo2Pct: number | null; + skinTempDevC: number | null; respRateBpm: number | null; steps: number | null; activeKcalEst: number | null; +} +export interface SleepRow { deviceId: string; family: Family; startTs: number; endTs: number; efficiency: number | null; restingHr: number | null; avgHrv: number | null; userEdited: number; } +export interface WorkoutRow { deviceId: string; family: Family; startTs: number; endTs: number; sport: string; source: string | null; durationS: number | null; energyKcal: number | null; distanceM: number | null; } +export interface MetricPointRow { deviceId: string; family: Family; day: string; key: string; value: number; } + +const withFamily = (r: T) => ({ ...r, family: sourceFamily(r.deviceId) }); + +export class Mirror { + private db: Database.Database; + constructor(path: string) { this.db = new Database(path, { readonly: true, fileMustExist: true }); } + close(): void { this.db.close(); } + + sources() { + const rows = this.db.prepare(` + SELECT d.deviceId, MAX(d.day) AS latestDay, p.brand AS brand + FROM dailyMetric d LEFT JOIN pairedDevice p ON p.id = d.deviceId + GROUP BY d.deviceId ORDER BY d.deviceId`).all() as any[]; + return rows.map((r) => ({ deviceId: r.deviceId, family: sourceFamily(r.deviceId), brand: r.brand ?? null, latestDay: r.latestDay ?? null })); + } + dailyMetrics(opts: { deviceId?: string; from: string; to: string }): DailyMetricRow[] { + const where = ["day >= ? AND day <= ?"]; const args: any[] = [opts.from, opts.to]; + if (opts.deviceId) { where.push("deviceId = ?"); args.push(opts.deviceId); } + return (this.db.prepare(`SELECT * FROM dailyMetric WHERE ${where.join(" AND ")} ORDER BY day, deviceId`).all(...args) as any[]).map(withFamily); + } + sleepSummary(opts: { from: string; to: string }): SleepRow[] { + const lo = Math.floor(new Date(`${opts.from}T00:00:00Z`).getTime() / 1000); + const hi = Math.floor(new Date(`${opts.to}T23:59:59Z`).getTime() / 1000); + return (this.db.prepare(`SELECT deviceId,startTs,endTs,efficiency,restingHr,avgHrv,userEdited FROM sleepSession WHERE startTs >= ? AND startTs <= ? ORDER BY startTs`).all(lo, hi) as any[]).map(withFamily); + } + workoutSummary(opts: { from: string; to: string }): WorkoutRow[] { + const lo = Math.floor(new Date(`${opts.from}T00:00:00Z`).getTime() / 1000); + const hi = Math.floor(new Date(`${opts.to}T23:59:59Z`).getTime() / 1000); + return (this.db.prepare(`SELECT deviceId,startTs,endTs,sport,source,durationS,energyKcal,distanceM FROM workout WHERE startTs >= ? AND startTs <= ? ORDER BY startTs`).all(lo, hi) as any[]).map(withFamily); + } + metricSeries(opts: { deviceId?: string; key?: string; from: string; to: string }): MetricPointRow[] { + const where = ["day >= ? AND day <= ?"]; const args: any[] = [opts.from, opts.to]; + if (opts.deviceId) { where.push("deviceId = ?"); args.push(opts.deviceId); } + if (opts.key) { where.push("key = ?"); args.push(opts.key); } + return (this.db.prepare(`SELECT deviceId,day,key,value FROM metricSeries WHERE ${where.join(" AND ")} ORDER BY day, key`).all(...args) as any[]).map(withFamily); + } + latestDataDay(): string | null { return (this.db.prepare("SELECT MAX(day) AS d FROM dailyMetric").get() as any)?.d ?? null; } + hrCoverageDays(deviceId: string): number { return (this.db.prepare("SELECT COUNT(DISTINCT date(ts,'unixepoch')) AS c FROM hrSample WHERE deviceId = ?").get(deviceId) as any).c; } +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test -- mirror` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: read-only mirror query layer (multi-source aware)" +``` + +--- + +### Task 5: Ingest pipeline and `POST /ingest` + +**Files:** +- Create: `noop-cloud/src/ingest.ts` +- Modify: `noop-cloud/src/server.ts` (add `/ingest` route + server.sqlite ingest log) +- Test: `noop-cloud/test/ingest.test.ts` + +**Interfaces:** +- Consumes: `Config`, `requireScope` (rw), fixtures. +- Produces: `ingestNoopbak(buf: Buffer, cfg: Config): { ok: true; bytes: number; latestDay: string | null }` — validates and atomically swaps into `cfg.mirrorPath`. Throws `IngestError` (with `.code`) on: `too_large`, `bad_zip`, `no_sqlite_entry`, `bad_magic`, `quick_check_failed`, `foreign_db`. +- Produces: `openServerDb(cfg): Database` creating `ingestLog(id INTEGER PK, receivedAt INTEGER, bytes INTEGER, latestDay TEXT)`. +- Produces: `latestIngest(cfg): { receivedAt: number; bytes: number; latestDay: string | null } | null`. +- Route: `POST /ingest` (rw), `express.raw({ type: "*/*", limit })`, returns 200 `{ ok, bytes, latestDay }` or 4xx `{ error: code }`. + +- [ ] **Step 1: Write the failing test** + +`test/ingest.test.ts`: + +```typescript +import { describe, it, expect, beforeEach } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import Database from "better-sqlite3"; import AdmZip from "adm-zip"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak, latestIngest, IngestError } from "../src/ingest.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/ingest"); +const cfg = () => ({ dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any); +beforeEach(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); }); + +describe("ingest", () => { + it("accepts a valid .noopbak and swaps the mirror", () => { + const zip = path.join(dataDir, "in.noopbak"); buildNoopbak(zip); + const r = ingestNoopbak(fs.readFileSync(zip), cfg()); + expect(r.ok).toBe(true); expect(r.latestDay).toBe("2026-06-13"); + const db = new Database(cfg().mirrorPath, { readonly: true }); + expect((db.prepare("SELECT COUNT(*) c FROM dailyMetric").get() as any).c).toBeGreaterThan(0); db.close(); + expect(latestIngest(cfg())?.latestDay).toBe("2026-06-13"); + }); + it("rejects oversized", () => { + const c = cfg(); c.maxIngestBytes = 10; + expect(() => ingestNoopbak(Buffer.alloc(100), c)).toThrow(IngestError); + }); + it("rejects a zip with no sqlite entry", () => { + const z = new AdmZip(); z.addFile("notes.txt", Buffer.from("hi")); + try { ingestNoopbak(z.toBuffer(), cfg()); expect.fail("should throw"); } + catch (e: any) { expect(e.code).toBe("no_sqlite_entry"); } + }); + it("rejects a foreign sqlite (no grdb_migrations)", () => { + const foreign = path.join(dataDir, "f.sqlite"); const db = new Database(foreign); db.exec("CREATE TABLE x(a)"); db.close(); + const z = new AdmZip(); z.addFile("noop-backup.sqlite", fs.readFileSync(foreign)); + try { ingestNoopbak(z.toBuffer(), cfg()); expect.fail("should throw"); } + catch (e: any) { expect(e.code).toBe("foreign_db"); } + }); + it("does not corrupt an existing mirror when the new upload is invalid", () => { + const zip = path.join(dataDir, "ok.noopbak"); buildNoopbak(zip); ingestNoopbak(fs.readFileSync(zip), cfg()); + try { ingestNoopbak(Buffer.from("not a zip"), cfg()); } catch { /* expected */ } + const db = new Database(cfg().mirrorPath, { readonly: true }); + expect((db.prepare("SELECT COUNT(*) c FROM dailyMetric").get() as any).c).toBeGreaterThan(0); db.close(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- ingest` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/ingest.ts`** + +```typescript +import fs from "node:fs"; import path from "node:path"; import crypto from "node:crypto"; +import Database from "better-sqlite3"; import AdmZip from "adm-zip"; +import type { Config } from "./config.js"; + +export class IngestError extends Error { constructor(public code: string, msg?: string) { super(msg ?? code); } } +const SQLITE_MAGIC = Buffer.from("SQLite format 3", "binary"); + +export function openServerDb(cfg: Pick): Database.Database { + const db = new Database(cfg.serverDbPath); + db.pragma("journal_mode = WAL"); + db.exec("CREATE TABLE IF NOT EXISTS ingestLog (id INTEGER PRIMARY KEY AUTOINCREMENT, receivedAt INTEGER, bytes INTEGER, latestDay TEXT)"); + return db; +} +export function latestIngest(cfg: Pick) { + const db = openServerDb(cfg); + const r = db.prepare("SELECT receivedAt, bytes, latestDay FROM ingestLog ORDER BY id DESC LIMIT 1").get() as any; + db.close(); + return r ?? null; +} + +export function ingestNoopbak(buf: Buffer, cfg: Pick): { ok: true; bytes: number; latestDay: string | null } { + if (buf.length > cfg.maxIngestBytes) throw new IngestError("too_large", `body ${buf.length} > ${cfg.maxIngestBytes}`); + let zip: AdmZip; + try { zip = new AdmZip(buf); } catch { throw new IngestError("bad_zip"); } + const entry = zip.getEntries().find((e) => e.entryName.endsWith(".sqlite")); + if (!entry) throw new IngestError("no_sqlite_entry"); + const sqliteBytes = entry.getData(); + if (!sqliteBytes.subarray(0, SQLITE_MAGIC.length).equals(SQLITE_MAGIC)) throw new IngestError("bad_magic"); + + // Stage to a temp file, validate, then atomically rename into place. + fs.mkdirSync(cfg.dataDir, { recursive: true }); + const staged = path.join(cfg.dataDir, `.staged-${crypto.randomBytes(6).toString("hex")}.sqlite`); + fs.writeFileSync(staged, sqliteBytes); + try { + const sdb = new Database(staged, { readonly: true }); + try { + const qc = sdb.pragma("quick_check", { simple: true }); + if (qc !== "ok") throw new IngestError("quick_check_failed", String(qc)); + const hasGrdb = sdb.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='grdb_migrations'").get(); + if (!hasGrdb) throw new IngestError("foreign_db"); + var latestDay = (sdb.prepare("SELECT MAX(day) d FROM dailyMetric").get() as any)?.d ?? null; + } finally { sdb.close(); } + // Atomic swap: rename staged -> mirror (same filesystem). Remove stale WAL/SHM sidecars. + for (const ext of ["-wal", "-shm"]) { const s = cfg.mirrorPath + ext; if (fs.existsSync(s)) fs.rmSync(s); } + fs.renameSync(staged, cfg.mirrorPath); + } catch (e) { + if (fs.existsSync(staged)) fs.rmSync(staged); + throw e; + } + const db = openServerDb(cfg); + db.prepare("INSERT INTO ingestLog (receivedAt, bytes, latestDay) VALUES (?,?,?)").run(Math.floor(Date.now() / 1000), buf.length, latestDay); + db.close(); + return { ok: true, bytes: buf.length, latestDay }; +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test -- ingest` +Expected: PASS (5 tests). + +- [ ] **Step 5: Wire the `/ingest` route in `src/server.ts`** + +Replace `createApp` in `src/server.ts` with: + +```typescript +import express from "express"; +import fs from "node:fs"; +import { Config, loadConfig } from "./config.js"; +import { requireScope } from "./auth.js"; +import { ingestNoopbak, IngestError } from "./ingest.js"; + +export function createApp(cfg: Config): express.Express { + fs.mkdirSync(cfg.dataDir, { recursive: true }); + const app = express(); + app.get("/healthz", (_req, res) => res.json({ ok: true })); + + app.post("/ingest", requireScope(cfg, "rw"), + express.raw({ type: "*/*", limit: cfg.maxIngestBytes }), + (req, res) => { + try { + const out = ingestNoopbak(req.body as Buffer, cfg); + res.json(out); + } catch (e) { + if (e instanceof IngestError) return res.status(400).json({ error: e.code }); + console.error("ingest error", e); + res.status(500).json({ error: "ingest_failed" }); + } + }); + + return app; +} +``` + +(keep the `if (process.argv[1]...)` entry block from Task 1) + +- [ ] **Step 6: Add an HTTP-level ingest test** + +Append to `test/ingest.test.ts`: + +```typescript +import http from "node:http"; import { createApp } from "../src/server.js"; +it("POST /ingest requires rw and swaps the mirror", async () => { + process.env.RO_TOKEN = "ro".padEnd(40, "x"); process.env.RW_TOKEN = "rw".padEnd(40, "y"); + const c = cfg(); c.roToken = process.env.RO_TOKEN; c.rwToken = process.env.RW_TOKEN; c.port = 0; + const zip = path.join(dataDir, "http.noopbak"); buildNoopbak(zip); const body = fs.readFileSync(zip); + const app = createApp(c); const server = app.listen(0); const port = (server.address() as any).port; + const status = await new Promise((resolve) => { + const r = http.request({ port, path: "/ingest", method: "POST", headers: { authorization: `Bearer ${c.rwToken}`, "content-type": "application/octet-stream" } }, (res) => { res.resume(); res.on("end", () => resolve(res.statusCode!)); }); + r.end(body); + }); + server.close(); expect(status).toBe(200); +}); +``` + +- [ ] **Step 7: Run to verify pass** + +Run: `npm test -- ingest && npm run build` +Expected: PASS (6 tests); build clean. + +- [ ] **Step 8: Commit** + +```bash +git add -A && git commit -m "feat: /ingest — validate + atomic-swap .noopbak into mirror.sqlite" +``` + +--- + +### Task 6: MCP stateless transport, tool registry, and prompts + +**Files:** +- Create: `noop-cloud/src/mcp.ts` +- Create: `noop-cloud/src/tools/index.ts` +- Modify: `noop-cloud/src/server.ts` (mount `/mcp`, ro scope) +- Test: `noop-cloud/test/mcp.test.ts` + +**Interfaces:** +- Consumes: `Config`, `Mirror`, `requireScope`. +- Produces: `buildMcpServer(cfg: Config): McpServer` — a fresh server with all tools + prompts registered. Tools open a `Mirror` per call and `close()` it in a `finally`. +- Produces: `registerTools(server: McpServer, cfg: Config): void` (Task 7-10 add tool files, each exporting `register(server, cfg)`; `registerTools` calls them). +- Produces: prompts `morning_report`, `corroborate_sources`, `find_messy_data` (each returns a `messages` array seeding the agent). +- Route: `POST /mcp` (ro) constructs a fresh transport (`sessionIdGenerator: undefined`) + server per request; `DELETE`/`GET /mcp` → 405. +- **Note:** verify exact import paths against the installed `@modelcontextprotocol/sdk@1.29.0` (`node -e "console.log(require.resolve('@modelcontextprotocol/sdk/server/mcp.js'))"`). If a path differs, adjust — the API shape (`McpServer`, `registerTool`, `registerPrompt`, `StreamableHTTPServerTransport`, `server.connect`, `transport.handleRequest`) is stable in 1.29.x. + +- [ ] **Step 1: Write the failing test (MCP list-tools over HTTP)** + +`test/mcp.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/mcp"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } + +function mcpCall(port: number, token: string, body: object): Promise<{ status: number; json: any }> { + return new Promise((resolve) => { + const payload = JSON.stringify(body); + const r = http.request({ port, path: "/mcp", method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json, text/event-stream" } }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => { + // Streamable HTTP may reply as SSE (`data: {json}`) or plain JSON. + const line = d.split("\n").find((l) => l.startsWith("data:")) ?? d; + const txt = line.replace(/^data:\s*/, ""); + resolve({ status: res.statusCode!, json: txt ? JSON.parse(txt) : null }); + }); + }); + r.end(payload); + }); +} + +describe("/mcp", () => { + it("401 without ro token", async () => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + const zip = path.join(dataDir, "b.noopbak"); buildNoopbak(zip); ingestNoopbak(fs.readFileSync(zip), cfg()); + const app = createApp(cfg()); const server = app.listen(0); const port = (server.address() as any).port; + const { status } = await mcpCall(port, "wrong", { jsonrpc: "2.0", id: 1, method: "tools/list" }); + server.close(); expect(status).toBe(401); + }); + it("lists registered tools", async () => { + const app = createApp(cfg()); const server = app.listen(0); const port = (server.address() as any).port; + const { json } = await mcpCall(port, cfg().roToken, { jsonrpc: "2.0", id: 1, method: "tools/list" }); + server.close(); + const names = (json.result?.tools ?? []).map((t: any) => t.name); + expect(names).toContain("data_freshness"); + expect(names).toContain("health_snapshot"); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- mcp` +Expected: FAIL — `src/mcp.js` missing. + +- [ ] **Step 3: Implement `src/tools/index.ts` (registry stub)** + +```typescript +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { registerCoreTools } from "./core.js"; + +export function registerTools(server: McpServer, cfg: Config): void { + registerCoreTools(server, cfg); // health_snapshot + data_freshness (Task 7) + // Task 8: registerQueryTools(server, cfg) + // Task 9: registerCompareSources(server, cfg) + // Task 10: registerSearchFetch(server, cfg) +} +``` + +- [ ] **Step 4: Implement `src/mcp.ts`** + +```typescript +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "./config.js"; +import { registerTools } from "./tools/index.js"; + +export function buildMcpServer(cfg: Config): McpServer { + const server = new McpServer({ name: "noop-cloud", version: "0.1.0" }); + + server.registerPrompt("morning_report", { + title: "Morning report", + description: "Draft a plain-English morning briefing from the last few days of biometrics.", + }, () => ({ + messages: [{ role: "user", content: { type: "text", text: + "Using the noop-cloud tools, write my morning report. Call data_freshness first and state the mirror age. " + + "Then summarize the last 3 days: recovery/strain (WHOOP), sleep (duration, efficiency), resting HR and HRV trend, " + + "and any workouts. Use compare_sources to flag where WHOOP, Oura, and Apple disagree by more than ~10%. " + + "Keep it under 200 words, concrete, no medical advice." } }], + })); + + server.registerPrompt("corroborate_sources", { + title: "Corroborate sources", + description: "Cross-check WHOOP vs Oura vs Apple for a date range and surface disagreements.", + }, () => ({ + messages: [{ role: "user", content: { type: "text", text: + "Call compare_sources for the last 14 days. List days where resting HR, HRV, or sleep duration differ by more than 10% " + + "across sources, and say which source looks like the outlier. Note any per-source baseline offsets." } }], + })); + + server.registerPrompt("find_messy_data", { + title: "Find messy data", + description: "Scan recent data for likely-mislogged workouts or implausible values.", + }, () => ({ + messages: [{ role: "user", content: { type: "text", text: + "Call workout_summary and metric_series for the last 30 days. Flag workouts with implausible duration/distance/energy, " + + "duplicate workouts across sources on the same day, and daily values that break trend. Do not change anything — just list findings." } }], + })); + + registerTools(server, cfg); + return server; +} +``` + +- [ ] **Step 5: Mount `/mcp` in `src/server.ts`** + +Add to `createApp` (before `return app`): + +```typescript +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { buildMcpServer } from "./mcp.js"; + +// ... inside createApp, after /ingest: +app.post("/mcp", requireScope(cfg, "ro"), express.json({ limit: "4mb" }), async (req, res) => { + const server = buildMcpServer(cfg); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + res.on("close", () => { transport.close(); server.close(); }); + try { + await server.connect(transport); + await transport.handleRequest(req, res, req.body); + } catch (e) { + console.error("mcp error", e); + if (!res.headersSent) res.status(500).json({ jsonrpc: "2.0", error: { code: -32603, message: "internal error" }, id: null }); + } +}); +app.all("/mcp", (_req, res) => res.status(405).json({ error: "method_not_allowed" })); +``` + +- [ ] **Step 6: Run to verify pass** + +Run: `npm test -- mcp && npm run build` +Expected: PASS (2 tests) once Task 7's `core.ts` exists. If `core.js` is not yet created, temporarily stub `registerCoreTools` to a no-op that registers a `data_freshness` returning `{}`; Task 7 fills it in. (Prefer implementing Task 7 Step 3 before running this.) + +- [ ] **Step 7: Commit** + +```bash +git add -A && git commit -m "feat: stateless Streamable HTTP /mcp endpoint + prompts + tool registry" +``` + +--- + +### Task 7: Core tools — `health_snapshot` + `data_freshness` + +**Files:** +- Create: `noop-cloud/src/tools/core.ts` +- Test: `noop-cloud/test/tools-core.test.ts` + +**Interfaces:** +- Consumes: `Mirror`, `latestIngest`, `Config`. +- Produces: `registerCoreTools(server: McpServer, cfg: Config): void`. + - `data_freshness` (no args) → `{ mirrorAgeSeconds, lastIngestAt, latestDataDay, sources: [{deviceId, family, latestDay}] }`. + - `health_snapshot` (args `{ days?: number }`, default 3) → per-day, per-family roll-up of recovery/strain/sleep/restingHr/avgHrv for the most recent `days`. +- Each tool handler returns `{ content: [{ type: "text", text: }], structuredContent: }`. + +- [ ] **Step 1: Write the failing test** + +`test/tools-core.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { dataFreshness, healthSnapshot } from "../src/tools/core.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/tools-core"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("core tools", () => { + it("data_freshness reports sources + latest day", () => { + const r = dataFreshness(cfg); + expect(r.latestDataDay).toBe("2026-06-13"); + expect(r.sources.map((s) => s.deviceId)).toContain("oura-api"); + expect(r.mirrorAgeSeconds).toBeGreaterThanOrEqual(0); + }); + it("health_snapshot rolls up recent days by family", () => { + const r = healthSnapshot(cfg, { days: 2 }); + expect(r.days.length).toBe(2); + const last = r.days[r.days.length - 1]; + expect(last.day).toBe("2026-06-13"); + expect(last.whoop?.recovery).toBe(66); + expect(last.oura?.restingHr).toBe(53); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-core` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/tools/core.ts`** + +```typescript +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { Mirror, Family } from "../mirror.js"; +import { latestIngest } from "../ingest.js"; + +export function dataFreshness(cfg: Config) { + const m = new Mirror(cfg.mirrorPath); + try { + const li = latestIngest(cfg); + const now = Math.floor(Date.now() / 1000); + return { + mirrorAgeSeconds: li ? now - li.receivedAt : null, + lastIngestAt: li ? new Date(li.receivedAt * 1000).toISOString() : null, + latestDataDay: m.latestDataDay(), + sources: m.sources().map((s) => ({ deviceId: s.deviceId, family: s.family, latestDay: s.latestDay })), + }; + } finally { m.close(); } +} + +export function healthSnapshot(cfg: Config, args: { days?: number }) { + const days = Math.max(1, Math.min(args.days ?? 3, 31)); + const m = new Mirror(cfg.mirrorPath); + try { + const latest = m.latestDataDay(); + if (!latest) return { days: [] as any[] }; + const to = latest; + const from = new Date(new Date(`${to}T00:00:00Z`).getTime() - (days - 1) * 86_400_000).toISOString().slice(0, 10); + const rows = m.dailyMetrics({ from, to }); + const byDay = new Map(); + for (const r of rows) { + const d = byDay.get(r.day) ?? { day: r.day }; + const fam: Family = r.family; + d[fam] = { restingHr: r.restingHr, avgHrv: r.avgHrv, totalSleepMin: r.totalSleepMin, efficiency: r.efficiency, recovery: r.recovery, strain: r.strain, steps: r.steps }; + byDay.set(r.day, d); + } + return { from, to, days: [...byDay.values()].sort((a, b) => a.day.localeCompare(b.day)) }; + } finally { m.close(); } +} + +const asTool = (obj: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(obj, null, 2) }], structuredContent: obj as Record }); + +export function registerCoreTools(server: McpServer, cfg: Config): void { + server.registerTool("data_freshness", { + title: "Data freshness", + description: "How stale the mirror is and which sources it holds. Call this first.", + inputSchema: {}, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async () => asTool(dataFreshness(cfg))); + + server.registerTool("health_snapshot", { + title: "Health snapshot", + description: "Recent per-day roll-up (recovery, strain, sleep, resting HR, HRV) grouped by source family.", + inputSchema: { days: z.number().int().min(1).max(31).optional().describe("How many recent days (default 3).") }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (args) => asTool(healthSnapshot(cfg, args))); +} +``` + +- [ ] **Step 4: Run to verify pass (core + mcp)** + +Run: `npm test -- tools-core && npm test -- mcp && npm run build` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: core MCP tools health_snapshot + data_freshness" +``` + +--- + +### Task 8: Query tools — `metric_series`, `sleep_summary`, `workout_summary` + +**Files:** +- Create: `noop-cloud/src/tools/query.ts` +- Modify: `noop-cloud/src/tools/index.ts` (call `registerQueryTools`) +- Test: `noop-cloud/test/tools-query.test.ts` + +**Interfaces:** +- Produces: `registerQueryTools(server, cfg)` plus testable fns `metricSeries(cfg, args)`, `sleepSummary(cfg, args)`, `workoutSummary(cfg, args)`. +- `metric_series` args `{ from, to, deviceId?, key? }` → `{ points: MetricPointRow[] }` (accepts any source incl. `oura-api`; keys like `ref_sleep_score`, `oura_readiness`). +- `sleep_summary` args `{ from, to }` → sessions with computed `durationMin`, per family. +- `workout_summary` args `{ from, to }` → workouts with `startIso`, per family. +- Date args are `YYYY-MM-DD`; validate with a zod regex. + +- [ ] **Step 1: Write the failing test** + +`test/tools-query.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; import { ingestNoopbak } from "../src/ingest.js"; +import { metricSeries, sleepSummary, workoutSummary } from "../src/tools/query.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/tools-query"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("query tools", () => { + it("metric_series returns Oura reference keys", () => { + const r = metricSeries(cfg, { from: "2026-06-10", to: "2026-06-13", deviceId: "oura-api", key: "oura_readiness" }); + expect(r.points.length).toBe(4); + expect(r.points[0].value).toBe(78); + }); + it("sleep_summary computes durationMin per family", () => { + const r = sleepSummary(cfg, { from: "2026-06-10", to: "2026-06-13" }); + expect(r.sessions.length).toBeGreaterThan(0); + expect(r.sessions[0].durationMin).toBeGreaterThan(0); + }); + it("workout_summary returns a running workout", () => { + const r = workoutSummary(cfg, { from: "2026-06-10", to: "2026-06-13" }); + expect(r.workouts.some((w) => w.sport === "running")).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-query` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/tools/query.ts`** + +```typescript +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { Mirror } from "../mirror.js"; + +const DAY = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "YYYY-MM-DD"); +const range = { from: DAY, to: DAY }; +const asTool = (obj: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(obj, null, 2) }], structuredContent: obj as Record }); + +export function metricSeries(cfg: Config, args: { from: string; to: string; deviceId?: string; key?: string }) { + const m = new Mirror(cfg.mirrorPath); try { return { points: m.metricSeries(args) }; } finally { m.close(); } +} +export function sleepSummary(cfg: Config, args: { from: string; to: string }) { + const m = new Mirror(cfg.mirrorPath); + try { return { sessions: m.sleepSummary(args).map((s) => ({ ...s, startIso: new Date(s.startTs * 1000).toISOString(), durationMin: Math.round((s.endTs - s.startTs) / 60) })) }; } + finally { m.close(); } +} +export function workoutSummary(cfg: Config, args: { from: string; to: string }) { + const m = new Mirror(cfg.mirrorPath); + try { return { workouts: m.workoutSummary(args).map((w) => ({ ...w, startIso: new Date(w.startTs * 1000).toISOString(), durationMin: w.durationS ? Math.round(w.durationS / 60) : null })) }; } + finally { m.close(); } +} + +export function registerQueryTools(server: McpServer, cfg: Config): void { + server.registerTool("metric_series", { + title: "Metric series", + description: "Long-format metric points for a date range. Any source (incl. oura-api) and key (e.g. ref_sleep_score, oura_readiness).", + inputSchema: { ...range, deviceId: z.string().optional(), key: z.string().optional() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(metricSeries(cfg, a))); + + server.registerTool("sleep_summary", { + title: "Sleep summary", + description: "Sleep sessions in a date range with duration and efficiency, per source family.", + inputSchema: { ...range }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(sleepSummary(cfg, a))); + + server.registerTool("workout_summary", { + title: "Workout summary", + description: "Workouts in a date range (sport, duration, energy, distance), per source family.", + inputSchema: { ...range }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(workoutSummary(cfg, a))); +} +``` + +- [ ] **Step 4: Wire into `src/tools/index.ts`** + +Add `import { registerQueryTools } from "./query.js";` and call `registerQueryTools(server, cfg);` inside `registerTools`. + +- [ ] **Step 5: Run to verify pass** + +Run: `npm test -- tools-query && npm run build` +Expected: PASS (3 tests). + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "feat: query tools metric_series + sleep_summary + workout_summary" +``` + +--- + +### Task 9: `compare_sources` (cross-source corroboration — G4) + +**Files:** +- Create: `noop-cloud/src/tools/compare.ts` +- Modify: `noop-cloud/src/tools/index.ts` +- Test: `noop-cloud/test/tools-compare.test.ts` + +**Interfaces:** +- Produces: `compareSources(cfg, args)` + `registerCompareSources(server, cfg)`. +- Args `{ from, to, metrics?: string[] }` (default metrics `["restingHr","avgHrv","totalSleepMin"]`). +- Returns `{ from, to, days: [{ day, metrics: { [metric]: { whoop?, oura?, apple?, spreadPct } } }] }` where `spreadPct = (max-min)/mean*100` over present family values, rounded; `null` families omitted. + +- [ ] **Step 1: Write the failing test** + +`test/tools-compare.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; import { ingestNoopbak } from "../src/ingest.js"; +import { compareSources } from "../src/tools/compare.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/tools-compare"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("compare_sources", () => { + it("puts WHOOP/Oura/Apple restingHr side by side with a spread", () => { + const r = compareSources(cfg, { from: "2026-06-13", to: "2026-06-13", metrics: ["restingHr"] }); + const day = r.days[0]; + expect(day.metrics.restingHr.whoop).toBe(51); + expect(day.metrics.restingHr.oura).toBe(53); + expect(day.metrics.restingHr.apple).toBe(54); + expect(day.metrics.restingHr.spreadPct).toBeGreaterThan(0); + }); + it("omits families with a null value (Apple avgHrv)", () => { + const r = compareSources(cfg, { from: "2026-06-13", to: "2026-06-13", metrics: ["avgHrv"] }); + expect(r.days[0].metrics.avgHrv.apple).toBeUndefined(); + expect(r.days[0].metrics.avgHrv.whoop).toBe(70); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-compare` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/tools/compare.ts`** + +```typescript +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { Mirror, DailyMetricRow } from "../mirror.js"; + +const DAY = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "YYYY-MM-DD"); +const DEFAULT_METRICS = ["restingHr", "avgHrv", "totalSleepMin"] as const; +const asTool = (obj: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(obj, null, 2) }], structuredContent: obj as Record }); + +export function compareSources(cfg: Config, args: { from: string; to: string; metrics?: string[] }) { + const metrics = (args.metrics && args.metrics.length ? args.metrics : [...DEFAULT_METRICS]); + const m = new Mirror(cfg.mirrorPath); + try { + const rows = m.dailyMetrics({ from: args.from, to: args.to }); + const byDay = new Map(); + for (const r of rows) { const a = byDay.get(r.day) ?? []; a.push(r); byDay.set(r.day, a); } + const days = [...byDay.entries()].sort((a, b) => a[0].localeCompare(b[0])).map(([day, drows]) => { + const metricsOut: Record = {}; + for (const metric of metrics) { + const cell: Record = {}; + for (const r of drows) { const v = (r as any)[metric]; if (v !== null && v !== undefined) cell[r.family] = v; } + const vals = Object.values(cell); + const spreadPct = vals.length >= 2 ? Math.round(((Math.max(...vals) - Math.min(...vals)) / (vals.reduce((s, x) => s + x, 0) / vals.length)) * 1000) / 10 : 0; + metricsOut[metric] = { ...cell, spreadPct }; + } + return { day, metrics: metricsOut }; + }); + return { from: args.from, to: args.to, days }; + } finally { m.close(); } +} + +export function registerCompareSources(server: McpServer, cfg: Config): void { + server.registerTool("compare_sources", { + title: "Compare sources", + description: "Per-day WHOOP vs Oura vs Apple side by side for chosen metrics, with a spread %. The corroboration workhorse.", + inputSchema: { from: DAY, to: DAY, metrics: z.array(z.string()).optional().describe("dailyMetric columns, e.g. restingHr, avgHrv, totalSleepMin, steps.") }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(compareSources(cfg, a))); +} +``` + +- [ ] **Step 4: Wire into `src/tools/index.ts`** — import + call `registerCompareSources(server, cfg)`. + +- [ ] **Step 5: Run to verify pass** + +Run: `npm test -- tools-compare && npm run build` +Expected: PASS (2 tests). + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "feat: compare_sources — per-day WHOOP/Oura/Apple corroboration" +``` + +--- + +### Task 10: ChatGPT-shaped `search` + `fetch` + +**Files:** +- Create: `noop-cloud/src/tools/search-fetch.ts` +- Modify: `noop-cloud/src/tools/index.ts` +- Test: `noop-cloud/test/tools-search.test.ts` + +**Interfaces:** +- Produces: `search(cfg, { query })` → `{ results: [{ id, title, url }] }` and `fetch(cfg, { id })` → `{ id, title, text, url, metadata }`, plus `registerSearchFetch(server, cfg)`. +- **Contract (ChatGPT Deep Research):** these two names + shapes are fixed. `id` format = `day:YYYY-MM-DD`. `search` matches a `YYYY-MM-DD` substring in the query, else returns the most recent days. `fetch` returns a text digest of that day across sources. `url` is a stable `noop-cloud://day/` placeholder (no external fetch). + +- [ ] **Step 1: Write the failing test** + +`test/tools-search.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; import { ingestNoopbak } from "../src/ingest.js"; +import { search, fetch as fetchDay } from "../src/tools/search-fetch.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/tools-search"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("search/fetch", () => { + it("search returns day results with id/title/url", () => { + const r = search(cfg, { query: "2026-06-12" }); + expect(r.results[0].id).toBe("day:2026-06-12"); + expect(r.results[0]).toHaveProperty("title"); + expect(r.results[0]).toHaveProperty("url"); + }); + it("fetch returns a text digest for a day", () => { + const r = fetchDay(cfg, { id: "day:2026-06-13" }); + expect(r.id).toBe("day:2026-06-13"); + expect(r.text).toMatch(/resting/i); + expect(r.text.length).toBeGreaterThan(20); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-search` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement `src/tools/search-fetch.ts`** + +```typescript +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { Mirror } from "../mirror.js"; + +const urlFor = (day: string) => `noop-cloud://day/${day}`; + +export function search(cfg: Config, args: { query: string }) { + const m = new Mirror(cfg.mirrorPath); + try { + const to = m.latestDataDay(); if (!to) return { results: [] }; + const from = new Date(new Date(`${to}T00:00:00Z`).getTime() - 30 * 86_400_000).toISOString().slice(0, 10); + const daysSet = new Set(m.dailyMetrics({ from, to }).map((r) => r.day)); + let days = [...daysSet].sort().reverse(); + const m2 = /\d{4}-\d{2}-\d{2}/.exec(args.query); + if (m2) days = days.filter((d) => d === m2[0]); + return { results: days.slice(0, 20).map((d) => ({ id: `day:${d}`, title: `Biometrics for ${d}`, url: urlFor(d) })) }; + } finally { m.close(); } +} + +export function fetch(cfg: Config, args: { id: string }) { + const day = args.id.replace(/^day:/, ""); + const m = new Mirror(cfg.mirrorPath); + try { + const rows = m.dailyMetrics({ from: day, to: day }); + const lines = rows.map((r) => `${r.family}: resting HR ${r.restingHr ?? "—"}, HRV ${r.avgHrv ?? "—"}, sleep ${r.totalSleepMin ?? "—"} min, recovery ${r.recovery ?? "—"}, strain ${r.strain ?? "—"}, steps ${r.steps ?? "—"}`); + const text = rows.length ? `Biometrics for ${day}\n${lines.join("\n")}` : `No data for ${day}.`; + return { id: args.id, title: `Biometrics for ${day}`, text, url: urlFor(day), metadata: { day, sources: rows.map((r) => r.family) } }; + } finally { m.close(); } +} + +export function registerSearchFetch(server: McpServer, cfg: Config): void { + server.registerTool("search", { + title: "Search", + description: "ChatGPT Deep Research search: find day-records. Returns {id,title,url}. Use a YYYY-MM-DD in the query to target a day.", + inputSchema: { query: z.string() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => { const r = search(cfg, a); return { content: [{ type: "text", text: JSON.stringify(r) }], structuredContent: r }; }); + + server.registerTool("fetch", { + title: "Fetch", + description: "ChatGPT Deep Research fetch: full text of a record by id (e.g. day:2026-06-13).", + inputSchema: { id: z.string() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => { const r = fetch(cfg, a); return { content: [{ type: "text", text: JSON.stringify(r) }], structuredContent: r }; }); +} +``` + +- [ ] **Step 4: Wire into `src/tools/index.ts`** — import + call `registerSearchFetch(server, cfg)`. + +- [ ] **Step 5: Run all tests + build** + +Run: `npm test && npm run build` +Expected: PASS (all suites). + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "feat: ChatGPT-shaped search + fetch tools" +``` + +--- + +### Task 11: Fly.io deploy config + README + +**Files:** +- Create: `noop-cloud/Dockerfile` +- Create: `noop-cloud/.dockerignore` +- Create: `noop-cloud/fly.toml` +- Create: `noop-cloud/README.md` + +**Interfaces:** none (deploy artifacts). Deliverable: `docker build` succeeds locally; `fly.toml` declares one machine + one volume at `/data` + autostop. + +- [ ] **Step 1: Write the `Dockerfile`** + +```dockerfile +FROM node:20-slim AS build +WORKDIR /app +RUN apt-get update && apt-get install -y python3 build-essential && rm -rf /var/lib/apt/lists/* +COPY package.json package-lock.json* ./ +RUN npm install +COPY tsconfig.json ./ +COPY src ./src +RUN npm run build + +FROM node:20-slim +WORKDIR /app +ENV NODE_ENV=production +COPY package.json package-lock.json* ./ +RUN apt-get update && apt-get install -y python3 build-essential && npm install --omit=dev && apt-get purge -y python3 build-essential && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* +COPY --from=build /app/dist ./dist +ENV DATA_DIR=/data +EXPOSE 8080 +CMD ["node", "dist/server.js"] +``` + +- [ ] **Step 2: Write `.dockerignore`** + +``` +node_modules +dist +.env +data +test +.git +``` + +- [ ] **Step 3: Write `fly.toml`** (replace `` at deploy time) + +```toml +app = "noop-cloud" +primary_region = "iad" + +[build] + +[env] + DATA_DIR = "/data" + PORT = "8080" + +[[mounts]] + source = "noop_data" + destination = "/data" + +[http_service] + internal_port = 8080 + force_https = true + auto_stop_machines = "stop" + auto_start_machines = true + min_machines_running = 0 + +[[vm]] + size = "shared-cpu-1x" + memory = "512mb" + + [[http_service.checks]] + grace_period = "10s" + interval = "30s" + method = "GET" + timeout = "5s" + path = "/healthz" +``` + +- [ ] **Step 4: Write `README.md`** (self-host guide) + +````markdown +# noop-cloud + +A self-hostable remote MCP server that mirrors a NOOP health database and exposes +read-only biometrics tools to Claude, ChatGPT, and agents over Streamable HTTP. + +## Deploy (Fly.io) + +```bash +fly launch --no-deploy --name # edit fly.toml app name +fly volumes create noop_data --size 1 --region iad +fly secrets set RO_TOKEN=$(openssl rand -hex 32) RW_TOKEN=$(openssl rand -hex 32) +fly deploy +``` + +Save the two tokens. `RW_TOKEN` uploads data; `RO_TOKEN` is for read clients (Claude Code, cron). + +## Upload data (iOS Shortcut, no app changes) + +In NOOP: **Settings → Backup & Sync → Back up now** to write a `.noopbak`. Then an iOS Shortcut: +"Get File → Get Contents of URL": `POST https://.fly.dev/ingest`, header +`Authorization: Bearer `, request body = the file. + +## Connect Claude Code + +```bash +claude mcp add --transport http noop-cloud https://.fly.dev/mcp \ + --header "Authorization: Bearer " +``` + +Then ask: "call data_freshness, then health_snapshot for the last 7 days." + +## Tools + +`data_freshness`, `health_snapshot`, `metric_series`, `sleep_summary`, `workout_summary`, +`compare_sources`, plus ChatGPT Deep Research `search`/`fetch`. Prompts: `morning_report`, +`corroborate_sources`, `find_messy_data`. All read-only in this phase. +```` + +- [ ] **Step 5: Verify the image builds** + +Run: `cd /Users/vk/VKDEV/NOOP/noop-cloud && docker build -t noop-cloud .` +Expected: build succeeds (better-sqlite3 compiles under python3/build-essential). + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "chore: Fly.io deploy config (Dockerfile, fly.toml) + README" +``` + +--- + +### Task 12: Morning report — GitHub Actions cron + local dry-run + +**Files:** +- Create: `noop-cloud/scripts/report.mjs` +- Create: `noop-cloud/.github/workflows/morning-report.yml` +- Modify: `noop-cloud/README.md` (report section) + +**Interfaces:** +- Produces: `scripts/report.mjs` — reads env `ANTHROPIC_API_KEY`, `NOOP_CLOUD_URL`, `NOOP_RO_TOKEN`, `NTFY_TOPIC`; calls the Anthropic Messages API with the MCP connector (ro token, tool-allowlist), extracts the text, POSTs it to `https://ntfy.sh/`. Exits non-zero on API error. `--dry-run` prints instead of pushing. +- Cron workflow runs it daily and on manual dispatch. + +- [ ] **Step 1: Write `scripts/report.mjs`** + +```javascript +// Node 20+. Generates the morning report via the Anthropic Messages API MCP connector, pushes to ntfy. +const { ANTHROPIC_API_KEY, NOOP_CLOUD_URL, NOOP_RO_TOKEN, NTFY_TOPIC } = process.env; +const dryRun = process.argv.includes("--dry-run"); +if (!ANTHROPIC_API_KEY || !NOOP_CLOUD_URL || !NOOP_RO_TOKEN) { console.error("missing env"); process.exit(2); } + +const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "x-api-key": ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", + "anthropic-beta": "mcp-client-2025-11-20", + "content-type": "application/json", + }, + body: JSON.stringify({ + model: "claude-sonnet-5", + max_tokens: 4096, + messages: [{ role: "user", content: "Write my morning health report. Call data_freshness first and state the mirror age; if it is older than 36 hours, say so plainly. Then summarize the last 3 days (recovery/strain, sleep, resting HR + HRV trend, workouts) and flag any source disagreements via compare_sources. Under 200 words. No medical advice." }], + mcp_servers: [{ + type: "url", url: `${NOOP_CLOUD_URL.replace(/\/$/, "")}/mcp`, name: "noop-cloud", + authorization_token: NOOP_RO_TOKEN, + tool_configuration: { enabled: true, allowed_tools: ["data_freshness", "health_snapshot", "sleep_summary", "workout_summary", "compare_sources", "metric_series"] }, + }], + }), +}); +if (!res.ok) { console.error("anthropic error", res.status, await res.text()); process.exit(1); } +const data = await res.json(); +if (data.stop_reason && data.stop_reason !== "end_turn") { console.error("anthropic stop_reason", data.stop_reason); process.exit(1); } +const text = (data.content ?? []).filter((b) => b.type === "text").map((b) => b.text).join("\n").trim() || "(empty report)"; + +if (dryRun || !NTFY_TOPIC) { console.log(text); process.exit(0); } +const push = await fetch(`https://ntfy.sh/${NTFY_TOPIC}`, { method: "POST", headers: { Title: "NOOP morning report", Priority: "default" }, body: text }); +if (!push.ok) { console.error("ntfy error", push.status); process.exit(1); } +console.log("pushed report to ntfy"); +``` + +- [ ] **Step 2: Write `.github/workflows/morning-report.yml`** + +```yaml +name: morning-report +on: + schedule: + - cron: "0 13 * * *" # 13:00 UTC daily — adjust to your morning; also wakes the Fly machine + workflow_dispatch: {} +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: "20" } + - run: node scripts/report.mjs + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + NOOP_CLOUD_URL: ${{ secrets.NOOP_CLOUD_URL }} + NOOP_RO_TOKEN: ${{ secrets.NOOP_RO_TOKEN }} + NTFY_TOPIC: ${{ secrets.NTFY_TOPIC }} +``` + +- [ ] **Step 3: Document + local dry-run test** + +Add to `README.md`: + +````markdown +## Morning report + +A GitHub Actions cron (`.github/workflows/morning-report.yml`) calls the Anthropic Messages API +with this server as an MCP connector (read-only token, tool-allowlisted) and pushes the result to +[ntfy](https://ntfy.sh). Set repo secrets `ANTHROPIC_API_KEY`, `NOOP_CLOUD_URL`, `NOOP_RO_TOKEN`, +`NTFY_TOPIC` and subscribe to the topic in the ntfy app. + +Local dry-run (prints, no push): + +```bash +ANTHROPIC_API_KEY=… NOOP_CLOUD_URL=https://.fly.dev NOOP_RO_TOKEN=… node scripts/report.mjs --dry-run +``` +```` + +Run the dry-run against the deployed server (or a local `npm run dev` with an ingested fixture) and confirm a coherent report prints. + +- [ ] **Step 4: Commit** + +```bash +git add -A && git commit -m "feat: morning report via GitHub Actions cron + Anthropic MCP connector + ntfy" +``` + +--- + +## Self-Review + +**Spec coverage (§5 Phase 1):** +- Fly app `/ingest` + `/mcp` + `/healthz` → Tasks 1, 5, 6. ✅ +- Upload via iOS Shortcut (no app code) → Task 11 README documents the Shortcut against `/ingest`. ✅ +- Read tools `health_snapshot`, `metric_series`, `sleep_summary`, `workout_summary`, `data_freshness`, `compare_sources` → Tasks 7–9. ✅ +- ChatGPT `search`/`fetch` → Task 10. ✅ +- Prompts `morning_report`, `corroborate_sources`, `find_messy_data` → Task 6. ✅ +- Auth: two static bearer tokens ro/rw, server-side → Task 2; `/ingest` rw, `/mcp` ro → Tasks 5–6. ✅ +- Morning report: GitHub Actions cron → Messages API MCP connector (ro, allowlisted) → ntfy → Task 12. ✅ +- Fly single-machine + volume + WAL + autostop → Task 11 `fly.toml`. ✅ +- Two SQLite files (`mirror.sqlite` RO, `server.sqlite` for ingest log) → Tasks 4, 5. ✅ +- `/ingest` validation (size cap, magic bytes, quick_check, foreign-db reject, atomic swap, no-corruption-on-failure) → Task 5. ✅ (spec §6 security posture) +- Multi-source awareness (deviceId families incl. oura-api) → Task 4 `sourceFamily` + every tool. ✅ + +**Deferred to later phases (correctly absent):** OAuth 2.1 shim (Phase 1.5), in-app upload button (1.5), write/staging tools + `server.sqlite` staging tables (Phase 2), sync-back `/edits` (Phase 3). Token-hashing-in-DB is deferred (env tokens for MVP) — noted in Task 2 as a Phase-1.5 upgrade. + +**Placeholder scan:** No `TBD`/`handle edge cases`/prose-only steps — every code step carries runnable code. The one explicit verification note (Task 6 Step: confirm SDK import paths) is a real check, not a placeholder. + +**Type consistency:** `Config` (Task 1) is consumed unchanged everywhere. `Mirror` methods (Task 4) match tool call sites (Tasks 7–10). `sourceFamily`/`Family` used consistently. `ingestNoopbak`/`latestIngest`/`IngestError` (Task 5) match their test + `data_freshness` call sites. Tool handler return shape `{ content:[{type:"text",text}], structuredContent }` is uniform. Registry `register(server, cfg)` naming is consistent across Tasks 7–10 and `tools/index.ts`. + +**Known follow-ups to flag during execution:** (1) verify the exact `better-sqlite3` prebuilt binding matches the Fly Node-20 image (else `npm rebuild`); (2) confirm `@modelcontextprotocol/sdk@1.29.0` streamable-HTTP stateless option name (`sessionIdGenerator: undefined`) against the installed package before relying on the `/mcp` test; (3) `claude-sonnet-5` model id in `report.mjs` — confirm current id at build time. + +--- + +## Execution Handoff + +Plan complete and saved to `docs/superpowers/plans/2026-07-10-noop-cloud-1-mvp-server.md`. Two execution options: + +**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration. + +**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. + +Which approach? diff --git a/docs/superpowers/plans/2026-07-11-noop-cloud-2-gated-writes.md b/docs/superpowers/plans/2026-07-11-noop-cloud-2-gated-writes.md new file mode 100644 index 0000000000..3b35a938e3 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-noop-cloud-2-gated-writes.md @@ -0,0 +1,1138 @@ +# noop-cloud Phase 2 — Gated Writes + Edit Journal Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** AI can *propose* data corrections (with a human-readable diff and a snapshot of the original), a human *confirms* with the rw credential, confirmed edits land in an append-only journal + overlay that read tools reflect — and the pristine mirror is never touched. + +**Architecture:** All write state lives in `server.sqlite` (two new tables: `proposal`, `editJournal`). `propose_edit` validates the payload per edit-kind, captures a **before-snapshot** from the mirror, renders a human diff, and stages a proposal. `confirm_edit` (rw-only, invisible to ro callers) moves it to the append-only journal; `undo_edit` appends a reversal entry (the journal never mutates). Read tools consult a computed **overlay** derived from active journal entries. A new `GET /edits?since=` endpoint exposes the sequenced journal for Phase-3 phone pull. Scope is threaded per request: the stateless `/mcp` route detects ro-vs-rw from the bearer token and builds the MCP server with only the tools that scope may see. + +**Tech Stack:** unchanged — TypeScript 5.6, Express 4, `@modelcontextprotocol/sdk` pinned **1.29.0**, `better-sqlite3` pinned **12.11.1**, zod, vitest. Repo `/Users/vk/VKDEV/NOOP/noop-cloud` (Phase 1 deployed at `https://vk-noop-cloud.fly.dev`; 41 tests green). + +## Global Constraints + +- **The mirror is never written.** Overlay + journal live only in `server.sqlite`. Any SQL against `mirror.sqlite` stays read-only (`{ readonly: true, fileMustExist: true }`). +- **Scope model (spec §5/§6):** `propose_edit`, `list_pending`, `edit_journal` available to **ro** (routines may stage, never apply). `confirm_edit`, `reject_edit`, `undo_edit` **registered only for rw callers** — invisible in ro `tools/list`, not merely rejected. rw implies ro. +- **Journal is append-only and idempotent by edit id.** Undo appends a reversal row (`kind='undo'`) and marks the original's `undoneBySeq`; rows are never updated otherwise, never deleted. `seq` (AUTOINCREMENT) is the Phase-3 pull cursor. +- **Every proposal carries `beforeJSON`** — the affected row(s) as they exist at propose time — so "what the data originally said" is permanently auditable. +- **Edit kinds (exactly these six):** `fix_workout`, `delete_workout`, `add_workout`, `adjust_sleep_bounds`, `delete_metric_point`, `set_baseline_note`. `add_workout` rows carry `deviceId: "noop-cloud"` (the server's own source namespace — spec §5 Phase 3 rule). +- **Annotations:** propose/list/journal → `readOnlyHint` false/true as appropriate but `destructiveHint: false`; `confirm_edit` and `undo_edit` → `destructiveHint: true`. (These drive client approval UI; enforcement stays server-side.) +- **Guard pattern (house style):** tools that read the mirror keep the `!fs.existsSync(cfg.mirrorPath)` → `notIngested: true` guard. Proposing an edit against a missing mirror returns a structured error, not a throw. +- **Overlay honesty:** overlay changes what `sleep_summary` / `workout_summary` / `metric_series` return (with `edited: true` / `added: true` provenance), but `dailyMetric`-derived numbers (`health_snapshot`, `compare_sources`, `totalSleepMin`) still come from the phone's own rollups — corrections reach those only after Phase 3 applies the edit on-phone and the next upload returns. Tool descriptions must say this plainly. +- **No secrets in git**; same pins; deploy via `fly deploy --ha=false` at the end. + +--- + +### Task 1: `serverdb.ts` (shared server-DB opener + staging schema) and `staging.ts` (state machine) + +**Files:** +- Create: `noop-cloud/src/serverdb.ts` +- Create: `noop-cloud/src/staging.ts` +- Modify: `noop-cloud/src/ingest.ts` (import `openServerDb` from serverdb.js; delete its local copy) +- Test: `noop-cloud/test/staging.test.ts` + +**Interfaces:** +- Consumes: `Config` (`serverDbPath`). +- Produces (serverdb.ts): `openServerDb(cfg: Pick): Database.Database` — WAL, `CREATE TABLE IF NOT EXISTS` for `ingestLog` (unchanged shape), `proposal`, `editJournal`. +- Produces (staging.ts): + - `interface ProposalRow { id: string; kind: string; payloadJSON: string; rationale: string; beforeJSON: string | null; diffText: string; status: "pending"|"confirmed"|"rejected"; createdAt: number; resolvedAt: number | null }` + - `interface JournalRow { seq: number; editId: string; kind: string; payloadJSON: string; beforeJSON: string | null; rationale: string | null; appliedAt: number; undoneBySeq: number | null }` + - `createProposal(cfg, p: Omit): ProposalRow` + - `getProposal(cfg, id: string): ProposalRow | null` + - `listPending(cfg): ProposalRow[]` + - `resolveProposal(cfg, id: string, status: "confirmed"|"rejected"): ProposalRow | null` (only from `pending`; returns null if absent or already resolved) + - `appendJournal(cfg, e: { editId: string; kind: string; payloadJSON: string; beforeJSON: string | null; rationale: string | null }): number` — returns `seq`; UNIQUE(editId) makes double-apply throw (caller treats as idempotency signal) + - `journalSince(cfg, since: number): JournalRow[]` (seq > since, ascending) + - `activeEdits(cfg): JournalRow[]` — `undoneBySeq IS NULL AND kind != 'undo'`, ascending + - `markUndone(cfg, targetSeq: number, bySeq: number): void` + +- [ ] **Step 1: Write the failing test** + +`test/staging.test.ts`: + +```typescript +import { describe, it, expect, beforeEach } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { createProposal, getProposal, listPending, resolveProposal, appendJournal, journalSince, activeEdits, markUndone } from "../src/staging.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/staging"); +const cfg = { serverDbPath: path.join(dataDir, "server.sqlite") } as any; +beforeEach(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); }); + +const draft = (id: string) => ({ id, kind: "delete_workout", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: 100, sport: "running" }), rationale: "duplicate", beforeJSON: JSON.stringify({ sport: "running" }), diffText: "- running workout @100" }); + +describe("staging state machine", () => { + it("propose → pending → confirm; not listed after", () => { + createProposal(cfg, draft("edit_a")); + expect(listPending(cfg).map((p) => p.id)).toEqual(["edit_a"]); + const r = resolveProposal(cfg, "edit_a", "confirmed"); + expect(r?.status).toBe("confirmed"); + expect(listPending(cfg)).toEqual([]); + }); + it("resolve is single-shot (second resolve returns null)", () => { + createProposal(cfg, draft("edit_b")); + expect(resolveProposal(cfg, "edit_b", "rejected")?.status).toBe("rejected"); + expect(resolveProposal(cfg, "edit_b", "confirmed")).toBeNull(); + }); + it("journal is append-only, sequenced, idempotent by editId", () => { + const s1 = appendJournal(cfg, { editId: "edit_c", kind: "delete_workout", payloadJSON: "{}", beforeJSON: null, rationale: null }); + const s2 = appendJournal(cfg, { editId: "edit_d", kind: "set_baseline_note", payloadJSON: "{}", beforeJSON: null, rationale: null }); + expect(s2).toBe(s1 + 1); + expect(() => appendJournal(cfg, { editId: "edit_c", kind: "delete_workout", payloadJSON: "{}", beforeJSON: null, rationale: null })).toThrow(); + expect(journalSince(cfg, s1).map((j) => j.seq)).toEqual([s2]); + }); + it("undo marks original inactive; undo rows never count as active", () => { + const s1 = appendJournal(cfg, { editId: "edit_e", kind: "delete_workout", payloadJSON: "{}", beforeJSON: null, rationale: null }); + const s2 = appendJournal(cfg, { editId: "undo_of_e", kind: "undo", payloadJSON: JSON.stringify({ targetSeq: s1 }), beforeJSON: null, rationale: null }); + markUndone(cfg, s1, s2); + expect(activeEdits(cfg)).toEqual([]); + expect(journalSince(cfg, 0).length).toBe(2); // history intact + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- staging` +Expected: FAIL — `Cannot find module '../src/staging.js'`. + +- [ ] **Step 3: Implement `src/serverdb.ts`** + +```typescript +import Database from "better-sqlite3"; +import type { Config } from "./config.js"; + +/** Single opener for the server-owned DB (never the mirror). All CREATEs are idempotent. */ +export function openServerDb(cfg: Pick): Database.Database { + const db = new Database(cfg.serverDbPath); + db.pragma("journal_mode = WAL"); + db.exec(` + CREATE TABLE IF NOT EXISTS ingestLog (id INTEGER PRIMARY KEY AUTOINCREMENT, receivedAt INTEGER, bytes INTEGER, latestDay TEXT); + CREATE TABLE IF NOT EXISTS proposal ( + id TEXT PRIMARY KEY, kind TEXT NOT NULL, payloadJSON TEXT NOT NULL, rationale TEXT NOT NULL, + beforeJSON TEXT, diffText TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','confirmed','rejected')), + createdAt INTEGER NOT NULL, resolvedAt INTEGER); + CREATE TABLE IF NOT EXISTS editJournal ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, editId TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, + payloadJSON TEXT NOT NULL, beforeJSON TEXT, rationale TEXT, + appliedAt INTEGER NOT NULL, undoneBySeq INTEGER); + `); + return db; +} +``` + +- [ ] **Step 4: Implement `src/staging.ts`** + +```typescript +import type { Config } from "./config.js"; +import { openServerDb } from "./serverdb.js"; + +type C = Pick; + +export interface ProposalRow { + id: string; kind: string; payloadJSON: string; rationale: string; + beforeJSON: string | null; diffText: string; + status: "pending" | "confirmed" | "rejected"; createdAt: number; resolvedAt: number | null; +} +export interface JournalRow { + seq: number; editId: string; kind: string; payloadJSON: string; + beforeJSON: string | null; rationale: string | null; appliedAt: number; undoneBySeq: number | null; +} + +function withDb(cfg: C, fn: (db: ReturnType) => T): T { + const db = openServerDb(cfg); + try { return fn(db); } finally { db.close(); } +} + +export function createProposal(cfg: C, p: Omit): ProposalRow { + return withDb(cfg, (db) => { + const createdAt = Math.floor(Date.now() / 1000); + db.prepare(`INSERT INTO proposal (id, kind, payloadJSON, rationale, beforeJSON, diffText, status, createdAt) + VALUES (?,?,?,?,?,?, 'pending', ?)`) + .run(p.id, p.kind, p.payloadJSON, p.rationale, p.beforeJSON, p.diffText, createdAt); + return { ...p, status: "pending", createdAt, resolvedAt: null }; + }); +} +export function getProposal(cfg: C, id: string): ProposalRow | null { + return withDb(cfg, (db) => (db.prepare("SELECT * FROM proposal WHERE id = ?").get(id) as ProposalRow | undefined) ?? null); +} +export function listPending(cfg: C): ProposalRow[] { + return withDb(cfg, (db) => db.prepare("SELECT * FROM proposal WHERE status = 'pending' ORDER BY createdAt, id").all() as ProposalRow[]); +} +export function resolveProposal(cfg: C, id: string, status: "confirmed" | "rejected"): ProposalRow | null { + return withDb(cfg, (db) => { + const r = db.prepare("UPDATE proposal SET status = ?, resolvedAt = ? WHERE id = ? AND status = 'pending'") + .run(status, Math.floor(Date.now() / 1000), id); + if (r.changes === 0) return null; + return db.prepare("SELECT * FROM proposal WHERE id = ?").get(id) as ProposalRow; + }); +} +export function appendJournal(cfg: C, e: { editId: string; kind: string; payloadJSON: string; beforeJSON: string | null; rationale: string | null }): number { + return withDb(cfg, (db) => { + const r = db.prepare(`INSERT INTO editJournal (editId, kind, payloadJSON, beforeJSON, rationale, appliedAt) + VALUES (?,?,?,?,?,?)`) + .run(e.editId, e.kind, e.payloadJSON, e.beforeJSON, e.rationale, Math.floor(Date.now() / 1000)); + return Number(r.lastInsertRowid); + }); +} +export function journalSince(cfg: C, since: number): JournalRow[] { + return withDb(cfg, (db) => db.prepare("SELECT * FROM editJournal WHERE seq > ? ORDER BY seq").all(since) as JournalRow[]); +} +export function activeEdits(cfg: C): JournalRow[] { + return withDb(cfg, (db) => db.prepare("SELECT * FROM editJournal WHERE undoneBySeq IS NULL AND kind != 'undo' ORDER BY seq").all() as JournalRow[]); +} +export function markUndone(cfg: C, targetSeq: number, bySeq: number): void { + withDb(cfg, (db) => db.prepare("UPDATE editJournal SET undoneBySeq = ? WHERE seq = ? AND undoneBySeq IS NULL").run(bySeq, targetSeq)); +} +``` + +- [ ] **Step 5: Point `src/ingest.ts` at the shared opener** + +In `src/ingest.ts`: delete its local `openServerDb` function, add `import { openServerDb } from "./serverdb.js";`, keep `latestIngest` and the `ingestLog` insert working exactly as before (the table shape is unchanged). Re-export for compatibility: `export { openServerDb } from "./serverdb.js";` + +- [ ] **Step 6: Run to verify pass** + +Run: `npm test` — Expected: staging 4/4 new + all 41 existing = 45 passing. `npm run build` clean. + +- [ ] **Step 7: Commit** + +```bash +git add -A && git commit -m "feat: staging store — proposals + append-only edit journal in server.sqlite" +``` + +--- + +### Task 2: Edit kinds — payload validation, before-snapshot, human diff + +**Files:** +- Create: `noop-cloud/src/edits/kinds.ts` +- Create: `noop-cloud/src/edits/diff.ts` +- Test: `noop-cloud/test/edits-diff.test.ts` + +**Interfaces:** +- Consumes: `Mirror` (Task 4 of Phase 1), `Config`. +- Produces (kinds.ts): + - `const EDIT_KINDS = ["fix_workout","delete_workout","add_workout","adjust_sleep_bounds","delete_metric_point","set_baseline_note"] as const; type EditKind = typeof EDIT_KINDS[number];` + - `payloadSchema(kind: EditKind): z.ZodTypeAny` with shapes: + - `fix_workout`: `{ deviceId: string, startTs: number, sport: string, patch: { sport?: string, startTs?: number, endTs?: number, energyKcal?: number|null, distanceM?: number|null } }` (patch must be non-empty) + - `delete_workout`: `{ deviceId, startTs, sport }` + - `add_workout`: `{ startTs: number, endTs: number, sport: string, energyKcal?: number, distanceM?: number, notes?: string }` (deviceId is NOT accepted — forced to `"noop-cloud"`) + - `adjust_sleep_bounds`: `{ deviceId, startTs, newStartTs?: number, newEndTs?: number }` (at least one of newStartTs/newEndTs) + - `delete_metric_point`: `{ deviceId, day: string (YYYY-MM-DD regex), key: string }` + - `set_baseline_note`: `{ note: string (1..500), deviceId?: string }` +- Produces (diff.ts): + - `class EditTargetError extends Error { constructor(public code: "target_not_found"|"not_ingested", msg?: string) }` + - `captureBefore(cfg: Config, kind: EditKind, payload: any): object | null` — reads the mirror; returns the affected row for fix/delete/adjust kinds (throws `EditTargetError("target_not_found")` if absent; throws `EditTargetError("not_ingested")` if no mirror); returns `null` for `add_workout`/`set_baseline_note`. + - `renderDiff(kind: EditKind, payload: any, before: object | null): string` — short human-readable text, e.g. `sleep 2026-06-13 03:00→10:00 ⇒ end 06:00 (was 10:00)`. + +- [ ] **Step 1: Write the failing test** + +`test/edits-diff.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { payloadSchema, EDIT_KINDS } from "../src/edits/kinds.js"; +import { captureBefore, renderDiff, EditTargetError } from "../src/edits/diff.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/edits-diff"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +const RUN_TS = Math.floor(new Date("2026-06-12T18:00:00Z").getTime() / 1000); // fixture running workout + +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("edit kinds", () => { + it("validates payloads per kind", () => { + expect(payloadSchema("delete_workout").safeParse({ deviceId: "my-whoop", startTs: RUN_TS, sport: "running" }).success).toBe(true); + expect(payloadSchema("adjust_sleep_bounds").safeParse({ deviceId: "my-whoop", startTs: 1 }).success).toBe(false); // needs a new bound + expect(payloadSchema("add_workout").safeParse({ startTs: 1, endTs: 2, sport: "yoga", deviceId: "my-whoop" }).success).toBe(false); // deviceId not accepted + expect(EDIT_KINDS.length).toBe(6); + }); + it("captureBefore snapshots the real row and errors on a missing target", () => { + const before = captureBefore(cfg, "delete_workout", { deviceId: "my-whoop", startTs: RUN_TS, sport: "running" }) as any; + expect(before.sport).toBe("running"); + expect(() => captureBefore(cfg, "delete_workout", { deviceId: "my-whoop", startTs: 1, sport: "nope" })).toThrow(EditTargetError); + }); + it("renderDiff is human-readable and mentions old and new values", () => { + const before = captureBefore(cfg, "adjust_sleep_bounds", { deviceId: "my-whoop", startTs: Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000), newEndTs: Math.floor(new Date("2026-06-13T06:00:00Z").getTime() / 1000) }); + const d = renderDiff("adjust_sleep_bounds", { deviceId: "my-whoop", startTs: Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000), newEndTs: Math.floor(new Date("2026-06-13T06:00:00Z").getTime() / 1000) }, before); + expect(d).toContain("06:00"); + expect(d.toLowerCase()).toContain("end"); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- edits-diff` — Expected: FAIL, module not found. + +- [ ] **Step 3: Implement `src/edits/kinds.ts`** + +```typescript +import { z } from "zod"; + +export const EDIT_KINDS = ["fix_workout", "delete_workout", "add_workout", "adjust_sleep_bounds", "delete_metric_point", "set_baseline_note"] as const; +export type EditKind = (typeof EDIT_KINDS)[number]; + +const DAY = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "YYYY-MM-DD"); +const workoutKey = { deviceId: z.string().min(1), startTs: z.number().int(), sport: z.string().min(1) }; + +const schemas: Record = { + fix_workout: z.object({ + ...workoutKey, + patch: z.object({ + sport: z.string().min(1).optional(), startTs: z.number().int().optional(), endTs: z.number().int().optional(), + energyKcal: z.number().nullable().optional(), distanceM: z.number().nullable().optional(), + }).refine((p) => Object.keys(p).length > 0, "patch must not be empty"), + }).strict(), + delete_workout: z.object(workoutKey).strict(), + add_workout: z.object({ + startTs: z.number().int(), endTs: z.number().int(), sport: z.string().min(1), + energyKcal: z.number().optional(), distanceM: z.number().optional(), notes: z.string().max(500).optional(), + }).strict().refine((p) => p.endTs > p.startTs, "endTs must be after startTs"), + adjust_sleep_bounds: z.object({ + deviceId: z.string().min(1), startTs: z.number().int(), + newStartTs: z.number().int().optional(), newEndTs: z.number().int().optional(), + }).strict().refine((p) => p.newStartTs !== undefined || p.newEndTs !== undefined, "need newStartTs and/or newEndTs"), + delete_metric_point: z.object({ deviceId: z.string().min(1), day: DAY, key: z.string().min(1) }).strict(), + set_baseline_note: z.object({ note: z.string().min(1).max(500), deviceId: z.string().min(1).optional() }).strict(), +}; + +export function payloadSchema(kind: EditKind): z.ZodTypeAny { return schemas[kind]; } +``` + +- [ ] **Step 4: Implement `src/edits/diff.ts`** + +```typescript +import fs from "node:fs"; +import Database from "better-sqlite3"; +import type { Config } from "../config.js"; +import type { EditKind } from "./kinds.js"; + +export class EditTargetError extends Error { + constructor(public code: "target_not_found" | "not_ingested", msg?: string) { super(msg ?? code); } +} + +const iso = (ts: number) => new Date(ts * 1000).toISOString().slice(0, 16).replace("T", " ") + "Z"; + +/** Read-only peek at the mirror for the row an edit targets. null for kinds with no target. */ +export function captureBefore(cfg: Pick, kind: EditKind, payload: any): object | null { + if (kind === "add_workout" || kind === "set_baseline_note") return null; + if (!fs.existsSync(cfg.mirrorPath)) throw new EditTargetError("not_ingested"); + const db = new Database(cfg.mirrorPath, { readonly: true, fileMustExist: true }); + try { + let row: unknown; + if (kind === "fix_workout" || kind === "delete_workout") { + row = db.prepare("SELECT * FROM workout WHERE deviceId=? AND startTs=? AND sport=?").get(payload.deviceId, payload.startTs, payload.sport); + } else if (kind === "adjust_sleep_bounds") { + row = db.prepare("SELECT * FROM sleepSession WHERE deviceId=? AND startTs=?").get(payload.deviceId, payload.startTs); + } else if (kind === "delete_metric_point") { + row = db.prepare("SELECT * FROM metricSeries WHERE deviceId=? AND day=? AND key=?").get(payload.deviceId, payload.day, payload.key); + } + if (!row) throw new EditTargetError("target_not_found", `${kind}: no matching row in the mirror`); + return row as object; + } finally { db.close(); } +} + +export function renderDiff(kind: EditKind, payload: any, before: any): string { + switch (kind) { + case "delete_workout": + return `DELETE workout ${before.sport} @ ${iso(before.startTs)} (${before.durationS ? Math.round(before.durationS / 60) + " min" : "?"}, source ${before.source ?? "?"}, ${payload.deviceId})`; + case "fix_workout": { + const parts = Object.entries(payload.patch).map(([k, v]) => `${k}: ${JSON.stringify((before as any)[k])} ⇒ ${JSON.stringify(v)}`); + return `FIX workout ${before.sport} @ ${iso(before.startTs)} (${payload.deviceId}): ${parts.join(", ")}`; + } + case "add_workout": + return `ADD workout ${payload.sport} ${iso(payload.startTs)} → ${iso(payload.endTs)} (source noop-cloud${payload.energyKcal ? `, ${payload.energyKcal} kcal` : ""})`; + case "adjust_sleep_bounds": { + const bits: string[] = []; + if (payload.newStartTs !== undefined) bits.push(`start ${iso(before.startTs)} ⇒ ${iso(payload.newStartTs)}`); + if (payload.newEndTs !== undefined) bits.push(`end ${iso(before.endTs)} ⇒ ${iso(payload.newEndTs)}`); + return `ADJUST sleep @ ${iso(before.startTs)} (${payload.deviceId}): ${bits.join(", ")}`; + } + case "delete_metric_point": + return `DELETE metric ${payload.key}=${before.value} on ${payload.day} (${payload.deviceId})`; + case "set_baseline_note": + return `NOTE${payload.deviceId ? ` [${payload.deviceId}]` : ""}: ${payload.note}`; + } +} +``` + +- [ ] **Step 5: Run to verify pass** + +Run: `npm test -- edits-diff` — Expected: 3/3 PASS. Full: `npm test` → 48. Build clean. + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "feat: edit kinds — zod payloads, before-snapshot capture, human diffs" +``` + +--- + +### Task 3: Overlay computation + +**Files:** +- Create: `noop-cloud/src/edits/overlay.ts` +- Test: `noop-cloud/test/overlay.test.ts` + +**Interfaces:** +- Consumes: `activeEdits` (Task 1), payload shapes (Task 2). +- Produces: + - `interface Overlay { sleepBounds: Map; deletedWorkouts: Set; patchedWorkouts: Map; addedWorkouts: { editId: string; deviceId: "noop-cloud"; startTs: number; endTs: number; sport: string; energyKcal: number | null; distanceM: number | null; notes: string | null }[]; deletedMetricPoints: Set; baselineNotes: { note: string; deviceId: string | null; at: number }[] }` + - Keys: workouts `"${deviceId}|${startTs}|${sport}"`, sleep `"${deviceId}|${startTs}"`, metric points `"${deviceId}|${day}|${key}"`. Export `workoutKeyOf(deviceId,startTs,sport)`, `sleepKeyOf(deviceId,startTs)`, `pointKeyOf(deviceId,day,key)`. + - `computeOverlay(cfg: Pick): Overlay` — folds `activeEdits` in seq order (later edits to the same target win). + +- [ ] **Step 1: Write the failing test** + +`test/overlay.test.ts`: + +```typescript +import { describe, it, expect, beforeEach } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { appendJournal, markUndone } from "../src/staging.js"; +import { computeOverlay, workoutKeyOf, sleepKeyOf } from "../src/edits/overlay.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/overlay"); +const cfg = { serverDbPath: path.join(dataDir, "server.sqlite") } as any; +beforeEach(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); }); + +describe("overlay", () => { + it("folds active edits into lookup structures", () => { + appendJournal(cfg, { editId: "e1", kind: "delete_workout", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: 100, sport: "running" }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e2", kind: "adjust_sleep_bounds", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: 200, newEndTs: 999 }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e3", kind: "add_workout", payloadJSON: JSON.stringify({ startTs: 300, endTs: 400, sport: "yoga" }), beforeJSON: null, rationale: null }); + const o = computeOverlay(cfg); + expect(o.deletedWorkouts.has(workoutKeyOf("my-whoop", 100, "running"))).toBe(true); + expect(o.sleepBounds.get(sleepKeyOf("my-whoop", 200))?.newEndTs).toBe(999); + expect(o.addedWorkouts[0]).toMatchObject({ deviceId: "noop-cloud", sport: "yoga" }); + }); + it("undone edits leave the overlay", () => { + const s = appendJournal(cfg, { editId: "e4", kind: "delete_workout", payloadJSON: JSON.stringify({ deviceId: "d", startTs: 1, sport: "s" }), beforeJSON: null, rationale: null }); + const u = appendJournal(cfg, { editId: "u4", kind: "undo", payloadJSON: JSON.stringify({ targetSeq: s }), beforeJSON: null, rationale: null }); + markUndone(cfg, s, u); + const o = computeOverlay(cfg); + expect(o.deletedWorkouts.size).toBe(0); + }); + it("later sleep adjustments to the same session win", () => { + appendJournal(cfg, { editId: "e5", kind: "adjust_sleep_bounds", payloadJSON: JSON.stringify({ deviceId: "d", startTs: 5, newEndTs: 10 }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e6", kind: "adjust_sleep_bounds", payloadJSON: JSON.stringify({ deviceId: "d", startTs: 5, newEndTs: 20 }), beforeJSON: null, rationale: null }); + expect(computeOverlay(cfg).sleepBounds.get(sleepKeyOf("d", 5))?.newEndTs).toBe(20); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- overlay` — Expected: FAIL, module not found. + +- [ ] **Step 3: Implement `src/edits/overlay.ts`** + +```typescript +import type { Config } from "../config.js"; +import { activeEdits } from "../staging.js"; + +export const workoutKeyOf = (deviceId: string, startTs: number, sport: string) => `${deviceId}|${startTs}|${sport}`; +export const sleepKeyOf = (deviceId: string, startTs: number) => `${deviceId}|${startTs}`; +export const pointKeyOf = (deviceId: string, day: string, key: string) => `${deviceId}|${day}|${key}`; + +export interface Overlay { + sleepBounds: Map; + deletedWorkouts: Set; + patchedWorkouts: Map; + addedWorkouts: { editId: string; deviceId: "noop-cloud"; startTs: number; endTs: number; sport: string; energyKcal: number | null; distanceM: number | null; notes: string | null }[]; + deletedMetricPoints: Set; + baselineNotes: { note: string; deviceId: string | null; at: number }[]; +} + +export function computeOverlay(cfg: Pick): Overlay { + const o: Overlay = { sleepBounds: new Map(), deletedWorkouts: new Set(), patchedWorkouts: new Map(), addedWorkouts: [], deletedMetricPoints: new Set(), baselineNotes: [] }; + for (const e of activeEdits(cfg)) { + const p = JSON.parse(e.payloadJSON); + switch (e.kind) { + case "delete_workout": o.deletedWorkouts.add(workoutKeyOf(p.deviceId, p.startTs, p.sport)); break; + case "fix_workout": { + const k = workoutKeyOf(p.deviceId, p.startTs, p.sport); + const prev = o.patchedWorkouts.get(k)?.patch ?? {}; + o.patchedWorkouts.set(k, { patch: { ...prev, ...p.patch }, editId: e.editId }); + break; + } + case "add_workout": + o.addedWorkouts.push({ editId: e.editId, deviceId: "noop-cloud", startTs: p.startTs, endTs: p.endTs, sport: p.sport, energyKcal: p.energyKcal ?? null, distanceM: p.distanceM ?? null, notes: p.notes ?? null }); + break; + case "adjust_sleep_bounds": { + const k = sleepKeyOf(p.deviceId, p.startTs); + const prev = o.sleepBounds.get(k) ?? { editId: e.editId }; + o.sleepBounds.set(k, { ...prev, ...(p.newStartTs !== undefined ? { newStartTs: p.newStartTs } : {}), ...(p.newEndTs !== undefined ? { newEndTs: p.newEndTs } : {}), editId: e.editId }); + break; + } + case "delete_metric_point": o.deletedMetricPoints.add(pointKeyOf(p.deviceId, p.day, p.key)); break; + case "set_baseline_note": o.baselineNotes.push({ note: p.note, deviceId: p.deviceId ?? null, at: e.appliedAt }); break; + } + } + return o; +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test -- overlay` — 3/3 PASS. Full suite 51. Build clean. + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: overlay — fold active journal edits into read-side lookups" +``` + +--- + +### Task 4: Scope threading + `propose_edit` / `list_pending` / `edit_journal` + +**Files:** +- Create: `noop-cloud/src/tools/writes.ts` +- Modify: `noop-cloud/src/auth.ts` (export `tokenScope`) +- Modify: `noop-cloud/src/mcp.ts` (`buildMcpServer(cfg, scope)`) +- Modify: `noop-cloud/src/server.ts` (detect scope in the /mcp route; pass through) +- Modify: `noop-cloud/src/tools/index.ts` (`registerTools(server, cfg, scope)` → call `registerWriteTools`) +- Test: `noop-cloud/test/tools-writes.test.ts` + +**Interfaces:** +- Produces (auth.ts): `tokenScope(cfg: Config, authHeader: string | undefined): "rw" | "ro" | null` (timing-safe; rw checked first). +- Produces (mcp.ts): `buildMcpServer(cfg: Config, scope: "ro" | "rw"): McpServer` — existing callers in tests must pass a scope; the /mcp route passes the detected one. +- Produces (writes.ts): `registerWriteTools(server: McpServer, cfg: Config, scope: "ro" | "rw"): void` registering for ALL scopes: `propose_edit`, `list_pending`, `edit_journal`; and ONLY when `scope === "rw"`: `confirm_edit`, `reject_edit`, `undo_edit` (Task 5 implements those bodies; this task registers propose/list/journal only and leaves a `// Task 5:` marker). + - `propose_edit(kind, payload, rationale)`: validate kind ∈ EDIT_KINDS and payload via `payloadSchema`; `captureBefore` (structured errors `{ error: "not_ingested" | "target_not_found" }` as tool results, not throws); `renderDiff`; `createProposal` with id `"edit_" + crypto.randomBytes(5).toString("hex")`; returns `{ id, kind, diff, rationale, status: "pending", hint: "confirm_edit requires the read-write credential" }`. + - `list_pending()`: `{ pending: [{ id, kind, diff, rationale, createdAt }] }`. + - `edit_journal({ since?: number })`: `{ edits: JournalRow[], latestSeq: number }` (since default 0). + - Annotations: propose `{ readOnlyHint: false, destructiveHint: false }`; list/journal `{ readOnlyHint: true }`. + +- [ ] **Step 1: Write the failing test** + +`test/tools-writes.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/writes"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } +const RUN_TS = Math.floor(new Date("2026-06-12T18:00:00Z").getTime() / 1000); + +function mcp(port: number, token: string, body: object): Promise { + return new Promise((resolve) => { + const payload = JSON.stringify(body); + const r = http.request({ port, path: "/mcp", method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json, text/event-stream" } }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => { + const line = d.split("\n").find((l) => l.startsWith("data:")) ?? d; + resolve(JSON.parse(line.replace(/^data:\s*/, ""))); + }); + }); + r.end(payload); + }); +} + +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg()); }); + +describe("write tools + scope", () => { + it("ro caller sees propose/list/journal but NOT confirm/reject/undo; rw sees all", async () => { + const app = createApp(cfg()); const server = app.listen(0); const port = (server.address() as any).port; + const ro = await mcp(port, cfg().roToken, { jsonrpc: "2.0", id: 1, method: "tools/list" }); + const rw = await mcp(port, cfg().rwToken, { jsonrpc: "2.0", id: 2, method: "tools/list" }); + server.close(); + const roNames = ro.result.tools.map((t: any) => t.name); + const rwNames = rw.result.tools.map((t: any) => t.name); + expect(roNames).toContain("propose_edit"); + expect(roNames).not.toContain("confirm_edit"); + expect(rwNames).toContain("confirm_edit"); + expect(rwNames).toContain("undo_edit"); + }); + it("propose_edit validates, snapshots, and stages; list_pending shows it", async () => { + const app = createApp(cfg()); const server = app.listen(0); const port = (server.address() as any).port; + const p = await mcp(port, cfg().roToken, { jsonrpc: "2.0", id: 3, method: "tools/call", params: { name: "propose_edit", arguments: { kind: "delete_workout", payload: { deviceId: "my-whoop", startTs: RUN_TS, sport: "running" }, rationale: "test dup" } } }); + const out = p.result.structuredContent; + expect(out.status).toBe("pending"); + expect(out.diff).toContain("running"); + const l = await mcp(port, cfg().roToken, { jsonrpc: "2.0", id: 4, method: "tools/call", params: { name: "list_pending", arguments: {} } }); + server.close(); + expect(l.result.structuredContent.pending.map((x: any) => x.id)).toContain(out.id); + }); + it("propose_edit on a missing target returns a structured error", async () => { + const app = createApp(cfg()); const server = app.listen(0); const port = (server.address() as any).port; + const p = await mcp(port, cfg().roToken, { jsonrpc: "2.0", id: 5, method: "tools/call", params: { name: "propose_edit", arguments: { kind: "delete_workout", payload: { deviceId: "my-whoop", startTs: 1, sport: "ghost" }, rationale: "x" } } }); + server.close(); + expect(p.result.structuredContent.error).toBe("target_not_found"); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-writes` — Expected: FAIL. + +- [ ] **Step 3: `src/auth.ts` — add tokenScope (keep everything else)** + +```typescript +export function tokenScope(cfg: Config, authHeader: string | undefined): "rw" | "ro" | null { + const m = /^Bearer (.+)$/.exec(authHeader ?? ""); + if (!m) return null; + if (safeEqual(m[1], cfg.rwToken)) return "rw"; + if (safeEqual(m[1], cfg.roToken)) return "ro"; + return null; +} +``` + +- [ ] **Step 4: `src/mcp.ts` — thread scope** + +Change signature to `export function buildMcpServer(cfg: Config, scope: "ro" | "rw"): McpServer` and the registry call to `registerTools(server, cfg, scope);`. Prompts unchanged. + +- [ ] **Step 5: `src/server.ts` — detect scope in the /mcp route** + +```typescript +// inside createApp, replace the /mcp POST handler's first line: +app.post("/mcp", requireScope(cfg, "ro"), express.json({ limit: "4mb" }), async (req, res) => { + const scope = tokenScope(cfg, req.header("authorization")) ?? "ro"; // requireScope already vetted it + const server = buildMcpServer(cfg, scope); + // ... rest unchanged +``` +(add `tokenScope` to the auth import) + +- [ ] **Step 6: `src/tools/index.ts` — signature + write tools** + +```typescript +import { registerWriteTools } from "./writes.js"; +export function registerTools(server: McpServer, cfg: Config, scope: "ro" | "rw"): void { + registerCoreTools(server, cfg); + registerQueryTools(server, cfg); + registerCompareSources(server, cfg); + registerSearchFetch(server, cfg); + registerWriteTools(server, cfg, scope); +} +``` + +- [ ] **Step 7: Implement `src/tools/writes.ts` (propose/list/journal; Task 5 adds the rest)** + +```typescript +import crypto from "node:crypto"; +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { EDIT_KINDS, EditKind, payloadSchema } from "../edits/kinds.js"; +import { captureBefore, renderDiff, EditTargetError } from "../edits/diff.js"; +import { createProposal, listPending, journalSince } from "../staging.js"; + +const asTool = (obj: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(obj, null, 2) }], structuredContent: obj as Record }); + +export function registerWriteTools(server: McpServer, cfg: Config, scope: "ro" | "rw"): void { + server.registerTool("propose_edit", { + title: "Propose a data edit", + description: "Stage a correction (nothing is applied until a human confirms with the read-write credential). Kinds: " + EDIT_KINDS.join(", ") + ". Returns a human-readable diff and the proposal id.", + inputSchema: { + kind: z.enum(EDIT_KINDS), + payload: z.record(z.unknown()).describe("Kind-specific payload; see the kind's schema."), + rationale: z.string().min(3).max(500).describe("Why this edit is correct — recorded in the audit journal."), + }, + annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + }, async (a) => { + const parsed = payloadSchema(a.kind as EditKind).safeParse(a.payload); + if (!parsed.success) return asTool({ error: "invalid_payload", detail: parsed.error.issues.map((i) => i.message).join("; ") }); + let before: object | null; + try { before = captureBefore(cfg, a.kind as EditKind, parsed.data); } + catch (e) { return asTool({ error: e instanceof EditTargetError ? e.code : "capture_failed" }); } + const id = "edit_" + crypto.randomBytes(5).toString("hex"); + const diff = renderDiff(a.kind as EditKind, parsed.data, before); + createProposal(cfg, { id, kind: a.kind, payloadJSON: JSON.stringify(parsed.data), rationale: a.rationale, beforeJSON: before ? JSON.stringify(before) : null, diffText: diff }); + return asTool({ id, kind: a.kind, diff, rationale: a.rationale, status: "pending", hint: "confirm_edit requires the read-write credential" }); + }); + + server.registerTool("list_pending", { + title: "List pending proposals", + description: "Staged edits awaiting confirm/reject.", + inputSchema: {}, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async () => asTool({ pending: listPending(cfg).map((p) => ({ id: p.id, kind: p.kind, diff: p.diffText, rationale: p.rationale, createdAt: p.createdAt })) })); + + server.registerTool("edit_journal", { + title: "Edit journal", + description: "Append-only audit log of every confirmed edit (and undos). since = last seq you have.", + inputSchema: { since: z.number().int().min(0).optional() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => { + const edits = journalSince(cfg, a.since ?? 0); + return asTool({ edits, latestSeq: edits.length ? edits[edits.length - 1].seq : (a.since ?? 0) }); + }); + + if (scope === "rw") { + registerResolutionTools(server, cfg); // Task 5 + } +} + +// Task 5 implements: confirm_edit, reject_edit, undo_edit +export function registerResolutionTools(_server: McpServer, _cfg: Config): void {} +``` + +- [ ] **Step 8: Fix existing test callers of buildMcpServer** — `test/mcp.test.ts` calls go through `createApp` (no direct buildMcpServer callers), so no change expected; if any test imports `buildMcpServer` directly, pass `"ro"`. + +- [ ] **Step 9: Run to verify pass** + +Run: `npm test` — Expected: 54 passing (51 + 3 new). Build clean. + +- [ ] **Step 10: Commit** + +```bash +git add -A && git commit -m "feat: scope-threaded MCP + propose_edit/list_pending/edit_journal" +``` + +--- + +### Task 5: `confirm_edit` / `reject_edit` / `undo_edit` (rw-only) + +**Files:** +- Modify: `noop-cloud/src/tools/writes.ts` (implement `registerResolutionTools`) +- Test: `noop-cloud/test/tools-resolve.test.ts` + +**Interfaces:** +- Consumes: `resolveProposal`, `getProposal`, `appendJournal`, `journalSince`, `markUndone` (Task 1). +- Produces tools (rw-registered only): + - `confirm_edit({ id })`: proposal must be pending → `resolveProposal(confirmed)` → `appendJournal({ editId: id, kind, payloadJSON, beforeJSON, rationale })` → `{ id, seq, applied: true, diff }`. Absent/already-resolved → `{ error: "not_pending" }`. If `appendJournal` throws UNIQUE (editId already journaled) → `{ id, applied: true, note: "already applied" }` (idempotent). + - `reject_edit({ id })`: pending → rejected → `{ id, rejected: true }`; else `{ error: "not_pending" }`. + - `undo_edit({ seq })`: journal row must exist, `kind != 'undo'`, `undoneBySeq IS NULL` → append `{ editId: "undo_" + , kind: "undo", payloadJSON: {targetSeq} }` → `markUndone` → `{ undoneSeq: seq, bySeq }`. Else `{ error: "not_undoable" }`. + - Annotations: confirm + undo `{ readOnlyHint: false, destructiveHint: true }`; reject `{ readOnlyHint: false, destructiveHint: false }`. + +- [ ] **Step 1: Write the failing test** + +`test/tools-resolve.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/resolve"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } +const RUN_TS = Math.floor(new Date("2026-06-12T18:00:00Z").getTime() / 1000); + +function mcp(port: number, token: string, id: number, name: string, args: object): Promise { + return new Promise((resolve) => { + const payload = JSON.stringify({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } }); + const r = http.request({ port, path: "/mcp", method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json, text/event-stream" } }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => { + const line = d.split("\n").find((l) => l.startsWith("data:")) ?? d; + resolve(JSON.parse(line.replace(/^data:\s*/, "")).result.structuredContent); + }); + }); + r.end(payload); + }); +} + +let port = 0; let server: any; +beforeAll(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg()); + const app = createApp(cfg()); server = app.listen(0); port = (server.address() as any).port; +}); + +describe("confirm/reject/undo", () => { + it("full lifecycle: propose → confirm → journal → undo → journal grows, overlay empties", async () => { + const p = await mcp(port, cfg().roToken, 1, "propose_edit", { kind: "delete_workout", payload: { deviceId: "my-whoop", startTs: RUN_TS, sport: "running" }, rationale: "dup" }); + const c = await mcp(port, cfg().rwToken, 2, "confirm_edit", { id: p.id }); + expect(c.applied).toBe(true); expect(c.seq).toBeGreaterThan(0); + const c2 = await mcp(port, cfg().rwToken, 3, "confirm_edit", { id: p.id }); + expect(c2.error).toBe("not_pending"); // single-shot resolution + const j1 = await mcp(port, cfg().rwToken, 4, "edit_journal", {}); + expect(j1.edits.length).toBe(1); + const u = await mcp(port, cfg().rwToken, 5, "undo_edit", { seq: c.seq }); + expect(u.undoneSeq).toBe(c.seq); + const j2 = await mcp(port, cfg().rwToken, 6, "edit_journal", {}); + expect(j2.edits.length).toBe(2); // append-only: undo is a new entry + const u2 = await mcp(port, cfg().rwToken, 7, "undo_edit", { seq: c.seq }); + expect(u2.error).toBe("not_undoable"); + }); + it("reject leaves no journal entry", async () => { + const p = await mcp(port, cfg().roToken, 8, "propose_edit", { kind: "set_baseline_note", payload: { note: "oura reads low" }, rationale: "baseline" }); + const r = await mcp(port, cfg().rwToken, 9, "reject_edit", { id: p.id }); + expect(r.rejected).toBe(true); + const j = await mcp(port, cfg().rwToken, 10, "edit_journal", {}); + expect(j.edits.filter((e: any) => e.editId === p.id).length).toBe(0); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- tools-resolve` — Expected: FAIL (confirm_edit not implemented/registered). + +- [ ] **Step 3: Implement `registerResolutionTools` in `src/tools/writes.ts`** + +```typescript +import { getProposal, resolveProposal, appendJournal, markUndone, journalSince } from "../staging.js"; // extend the existing import + +export function registerResolutionTools(server: McpServer, cfg: Config): void { + server.registerTool("confirm_edit", { + title: "Confirm a proposed edit", + description: "Apply a pending proposal to the edit journal + overlay. Requires the read-write credential. The mirror itself is never modified; corrections reach the phone in Phase 3.", + inputSchema: { id: z.string().min(1) }, + annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, + }, async (a) => { + const resolved = resolveProposal(cfg, a.id, "confirmed"); + if (!resolved) { + const existing = getProposal(cfg, a.id); + if (existing?.status === "confirmed") return asTool({ id: a.id, applied: true, note: "already applied" }); + return asTool({ error: "not_pending" }); + } + try { + const seq = appendJournal(cfg, { editId: resolved.id, kind: resolved.kind, payloadJSON: resolved.payloadJSON, beforeJSON: resolved.beforeJSON, rationale: resolved.rationale }); + return asTool({ id: resolved.id, seq, applied: true, diff: resolved.diffText }); + } catch { + return asTool({ id: resolved.id, applied: true, note: "already applied" }); // UNIQUE(editId) — idempotent + } + }); + + server.registerTool("reject_edit", { + title: "Reject a proposed edit", + description: "Discard a pending proposal (no journal entry, no data change).", + inputSchema: { id: z.string().min(1) }, + annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + }, async (a) => { + const r = resolveProposal(cfg, a.id, "rejected"); + return asTool(r ? { id: a.id, rejected: true } : { error: "not_pending" }); + }); + + server.registerTool("undo_edit", { + title: "Undo a confirmed edit", + description: "Reverse a journal entry by appending an undo record (history is never deleted).", + inputSchema: { seq: z.number().int().min(1) }, + annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, + }, async (a) => { + const target = journalSince(cfg, a.seq - 1).find((e) => e.seq === a.seq); + if (!target || target.kind === "undo" || target.undoneBySeq !== null) return asTool({ error: "not_undoable" }); + const bySeq = appendJournal(cfg, { editId: "undo_" + crypto.randomBytes(5).toString("hex"), kind: "undo", payloadJSON: JSON.stringify({ targetSeq: a.seq }), beforeJSON: null, rationale: null }); + markUndone(cfg, a.seq, bySeq); + return asTool({ undoneSeq: a.seq, bySeq }); + }); +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test` — Expected: 56 passing. Build clean. + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: confirm/reject/undo — rw-only resolution over the append-only journal" +``` + +--- + +### Task 6: Overlay-aware read tools + +**Files:** +- Modify: `noop-cloud/src/tools/query.ts` (sleep_summary bounds; workout_summary delete/patch/add; metric_series point deletes) +- Modify: `noop-cloud/src/tools/core.ts` (data_freshness gains `pendingEdits`, `journalSeq`, `baselineNotes`) +- Test: `noop-cloud/test/tools-overlay-reads.test.ts` + +**Interfaces:** +- Consumes: `computeOverlay`, key helpers (Task 3), `listPending`, `journalSince` (Task 1). +- Produces (behavioral contract): + - `sleep_summary`: a session with an overlay bound adjustment returns adjusted `startTs`/`endTs`, recomputed `durationMin`, plus `edited: true, editId`. Unedited sessions unchanged (no `edited` key). + - `workout_summary`: deleted workouts absent; patched workouts carry patch fields + `edited: true, editId`; added workouts appear with `deviceId: "noop-cloud"`, `family: "cloud"`... **No** — keep `sourceFamily` untouched (Phase-1 contract): added rows get `family: "whoop"`? Wrong either way — added rows carry `deviceId: "noop-cloud"` and `family` computed by `sourceFamily("noop-cloud")` which yields "whoop" today. To avoid misclassification, `workout_summary` sets `family: "cloud" as any` for added rows explicitly and marks `added: true, editId`. (Phase 3 will register the namespace properly; the tool-level label is honest today.) + - `metric_series`: deleted points absent. + - `data_freshness`: adds `pendingEdits: number`, `journalSeq: number` (latest seq, 0 if none), `baselineNotes: [{note, deviceId, at}]`. + - Tool descriptions for sleep_summary/workout_summary/metric_series append: "Reflects confirmed server-side edits; dailyMetric-derived numbers (health_snapshot/compare_sources) update only after the phone applies edits (Phase 3) and re-uploads." + +- [ ] **Step 1: Write the failing test** + +`test/tools-overlay-reads.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { appendJournal } from "../src/staging.js"; +import { sleepSummary, workoutSummary, metricSeries } from "../src/tools/query.js"; +import { dataFreshness } from "../src/tools/core.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/overlay-reads"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +const SLEEP_TS = Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000); +const RUN_TS = Math.floor(new Date("2026-06-12T18:00:00Z").getTime() / 1000); +const NEW_END = Math.floor(new Date("2026-06-13T06:00:00Z").getTime() / 1000); + +beforeAll(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); + appendJournal(cfg, { editId: "e_sleep", kind: "adjust_sleep_bounds", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: SLEEP_TS, newEndTs: NEW_END }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e_del", kind: "delete_workout", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: RUN_TS, sport: "running" }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e_add", kind: "add_workout", payloadJSON: JSON.stringify({ startTs: RUN_TS + 7200, endTs: RUN_TS + 9000, sport: "yoga" }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e_pt", kind: "delete_metric_point", payloadJSON: JSON.stringify({ deviceId: "oura-api", day: "2026-06-13", key: "oura_readiness" }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "e_note", kind: "set_baseline_note", payloadJSON: JSON.stringify({ note: "oura RHR reads ~2bpm high", deviceId: "oura-api" }), beforeJSON: null, rationale: null }); +}); + +describe("overlay-aware reads", () => { + it("sleep_summary reflects adjusted end + recomputed duration + provenance", () => { + const s = sleepSummary(cfg, { from: "2026-06-13", to: "2026-06-13" }).sessions.find((x: any) => x.deviceId === "my-whoop"); + expect(s.endTs).toBe(NEW_END); + expect(s.durationMin).toBe(Math.round((NEW_END - SLEEP_TS) / 60)); + expect(s.edited).toBe(true); + }); + it("workout_summary hides deleted, shows added with provenance", () => { + const w = workoutSummary(cfg, { from: "2026-06-11", to: "2026-06-13" }); + expect(w.workouts.find((x: any) => x.sport === "running")).toBeUndefined(); + const added = w.workouts.find((x: any) => x.sport === "yoga"); + expect(added.deviceId).toBe("noop-cloud"); + expect(added.added).toBe(true); + }); + it("metric_series drops deleted points", () => { + const r = metricSeries(cfg, { deviceId: "oura-api", key: "oura_readiness", from: "2026-06-10", to: "2026-06-13" }); + expect(r.points.length).toBe(3); // fixture had 4 days + }); + it("data_freshness reports journal state + notes", () => { + const f = dataFreshness(cfg) as any; + expect(f.journalSeq).toBeGreaterThanOrEqual(5); + expect(f.baselineNotes[0].note).toContain("RHR"); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- overlay-reads` — Expected: FAIL (endTs unadjusted, running present, 4 points, no journalSeq). + +- [ ] **Step 3: Apply overlay in `src/tools/query.ts`** + +```typescript +import { computeOverlay, workoutKeyOf, sleepKeyOf, pointKeyOf } from "../edits/overlay.js"; + +export function sleepSummary(cfg: Config, args: { from: string; to: string }) { + if (!fs.existsSync(cfg.mirrorPath)) return { sessions: [], notIngested: true }; + const overlay = computeOverlay(cfg); + const m = new Mirror(cfg.mirrorPath); + try { + const sessions = m.sleepSummary(args).map((s) => { + const adj = overlay.sleepBounds.get(sleepKeyOf(s.deviceId, s.startTs)); + const startTs = adj?.newStartTs ?? s.startTs; + const endTs = adj?.newEndTs ?? s.endTs; + return { ...s, startTs, endTs, startIso: new Date(startTs * 1000).toISOString(), durationMin: Math.round((endTs - startTs) / 60), ...(adj ? { edited: true, editId: adj.editId } : {}) }; + }); + return { sessions }; + } finally { m.close(); } +} + +export function workoutSummary(cfg: Config, args: { from: string; to: string }) { + if (!fs.existsSync(cfg.mirrorPath)) return { workouts: [], notIngested: true }; + const overlay = computeOverlay(cfg); + const lo = Math.floor(new Date(`${args.from}T00:00:00Z`).getTime() / 1000); + const hi = Math.floor(new Date(`${args.to}T23:59:59Z`).getTime() / 1000); + const m = new Mirror(cfg.mirrorPath); + try { + const base = m.workoutSummary(args) + .filter((w) => !overlay.deletedWorkouts.has(workoutKeyOf(w.deviceId, w.startTs, w.sport))) + .map((w) => { + const patch = overlay.patchedWorkouts.get(workoutKeyOf(w.deviceId, w.startTs, w.sport)); + const merged = patch ? { ...w, ...patch.patch, edited: true, editId: patch.editId } : w; + const durationS = (merged as any).endTs && merged.startTs ? (merged as any).endTs - merged.startTs : merged.durationS; + // durationS AFTER the spread so the recomputed value overrides the mirror's stale field. + return { ...merged, durationS, startIso: new Date(merged.startTs * 1000).toISOString(), durationMin: durationS != null ? Math.round(durationS / 60) : null }; + }); + const added = overlay.addedWorkouts + .filter((w) => w.startTs >= lo && w.startTs <= hi) + .map((w) => ({ deviceId: w.deviceId, family: "cloud" as any, startTs: w.startTs, endTs: w.endTs, sport: w.sport, source: "noop-cloud", durationS: w.endTs - w.startTs, energyKcal: w.energyKcal, distanceM: w.distanceM, startIso: new Date(w.startTs * 1000).toISOString(), durationMin: Math.round((w.endTs - w.startTs) / 60), added: true, editId: w.editId })); + return { workouts: [...base, ...added].sort((a, b) => a.startTs - b.startTs) }; + } finally { m.close(); } +} + +export function metricSeries(cfg: Config, args: { from: string; to: string; deviceId?: string; key?: string }) { + if (!fs.existsSync(cfg.mirrorPath)) return { points: [], notIngested: true }; + const overlay = computeOverlay(cfg); + const m = new Mirror(cfg.mirrorPath); + try { return { points: m.metricSeries(args).filter((p) => !overlay.deletedMetricPoints.has(pointKeyOf(p.deviceId, p.day, p.key))) }; } + finally { m.close(); } +} +``` +Append to the three registered tool descriptions: `" Reflects confirmed server-side edits; dailyMetric-derived numbers update only after Phase-3 phone sync."` + +- [ ] **Step 4: `src/tools/core.ts` — data_freshness additions** + +```typescript +import { listPending, journalSince } from "../staging.js"; +import { computeOverlay } from "../edits/overlay.js"; +// inside dataFreshness's returned object (both the notIngested early-return and the normal path): +// pendingEdits: listPending(cfg).length, +// journalSeq: (() => { const j = journalSince(cfg, 0); return j.length ? j[j.length - 1].seq : 0; })(), +// baselineNotes: computeOverlay(cfg).baselineNotes, +``` +(server.sqlite exists independently of the mirror, so these fields are correct in both paths.) + +- [ ] **Step 5: Run to verify pass** + +Run: `npm test` — Expected: 60 passing. Build clean. + +- [ ] **Step 6: Commit** + +```bash +git add -A && git commit -m "feat: overlay-aware reads — edits reflected with provenance markers" +``` + +--- + +### Task 7: `GET /edits?since=` (Phase-3 pull endpoint) + +**Files:** +- Modify: `noop-cloud/src/server.ts` (new route, ro scope, before the error middleware) +- Test: `noop-cloud/test/edits-endpoint.test.ts` + +**Interfaces:** +- Produces: `GET /edits?since=` (ro): `{ edits: JournalRow[], latestSeq: number }` — the FULL journal including `undo` rows (the phone replays reversals too). `since` optional/default 0; non-numeric → 400 `{ error: "bad_since" }`. + +- [ ] **Step 1: Write the failing test** + +`test/edits-endpoint.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { appendJournal } from "../src/staging.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/edits-ep"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } + +function get(port: number, pathq: string, token?: string): Promise<{ status: number; json: any }> { + return new Promise((resolve) => { + http.get({ port, path: pathq, headers: token ? { authorization: `Bearer ${token}` } : {} }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => resolve({ status: res.statusCode!, json: d ? JSON.parse(d) : null })); + }); + }); +} + +beforeAll(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + appendJournal(cfg(), { editId: "a", kind: "set_baseline_note", payloadJSON: "{}", beforeJSON: null, rationale: null }); + appendJournal(cfg(), { editId: "b", kind: "undo", payloadJSON: JSON.stringify({ targetSeq: 1 }), beforeJSON: null, rationale: null }); +}); + +describe("GET /edits", () => { + it("requires auth", async () => { + const app = createApp(cfg()); const s = app.listen(0); const port = (s.address() as any).port; + expect((await get(port, "/edits")).status).toBe(401); s.close(); + }); + it("returns the sequenced journal including undo rows, filtered by since", async () => { + const app = createApp(cfg()); const s = app.listen(0); const port = (s.address() as any).port; + const all = await get(port, "/edits", cfg().roToken); + expect(all.json.edits.length).toBe(2); + expect(all.json.latestSeq).toBe(2); + const tail = await get(port, "/edits?since=1", cfg().roToken); + expect(tail.json.edits.map((e: any) => e.editId)).toEqual(["b"]); + const bad = await get(port, "/edits?since=zzz", cfg().roToken); + expect(bad.status).toBe(400); s.close(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npm test -- edits-endpoint` — Expected: 404s / FAIL. + +- [ ] **Step 3: Add the route in `src/server.ts`** (after `/mcp`, before the error middleware) + +```typescript +import { journalSince } from "./staging.js"; + +app.get("/edits", requireScope(cfg, "ro"), (req, res) => { + const raw = req.query.since; + const since = raw === undefined ? 0 : Number(raw); + if (!Number.isInteger(since) || since < 0) return res.status(400).json({ error: "bad_since" }); + const edits = journalSince(cfg, since); + res.json({ edits, latestSeq: edits.length ? edits[edits.length - 1].seq : since }); +}); +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `npm test` — Expected: 62 passing. Build clean. + +- [ ] **Step 5: Commit** + +```bash +git add -A && git commit -m "feat: GET /edits — sequenced journal pull for Phase-3 phone sync" +``` + +--- + +### Task 8: README, deploy, live smoke + +**Files:** +- Modify: `noop-cloud/README.md` (add "Editing your data (Phase 2)" section) +- Deploy: `fly deploy --ha=false` + +- [ ] **Step 1: README section** (append after Tools) + +````markdown +## Editing your data (Phase 2) + +The AI can *propose* corrections; nothing changes until you confirm with the read-write credential. +The mirror (your uploaded data) is never modified — confirmed edits live in an append-only journal +and an overlay that read tools reflect (marked `edited: true` / `added: true`). Every proposal +records the original row (`before`) and your rationale; `undo_edit` reverses by appending, so the +audit trail is complete forever. + +- Read-only callers (routines, cron, shared agents) can `propose_edit`, `list_pending`, `edit_journal`. +- `confirm_edit` / `reject_edit` / `undo_edit` exist **only** for read-write callers — invisible otherwise. +- `GET /edits?since=` (read token) streams the journal for downstream sync. +- Honesty note: `health_snapshot` / `compare_sources` aggregate the phone's own daily rollups; those + numbers update after the phone applies your edits (Phase 3) and re-uploads. +```` + +- [ ] **Step 2: Full suite + build one last time** + +Run: `npm test && npm run build` — Expected: 62 passing, clean. + +- [ ] **Step 3: Commit + deploy** + +```bash +git add -A && git commit -m "docs: Phase-2 gated-writes README section" +fly deploy --ha=false --yes +``` +Expected: image builds on Fly's remote builder; machine updates; `https://vk-noop-cloud.fly.dev/healthz` → `{ok:true}`. + +- [ ] **Step 4: Live smoke (curl, using .env tokens)** + +```bash +source .env; BASE=https://vk-noop-cloud.fly.dev +# ro sees propose but not confirm +curl -s -X POST "$BASE/mcp" -H "Authorization: Bearer $RO_TOKEN" -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | grep -c confirm_edit # expect 0 +curl -s -X POST "$BASE/mcp" -H "Authorization: Bearer $RW_TOKEN" -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' -d '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | grep -c confirm_edit # expect >=1 +# journal endpoint +curl -s "$BASE/edits" -H "Authorization: Bearer $RO_TOKEN" +``` + +- [ ] **Step 5: Ledger the deploy** in `/Users/vk/VKDEV/NOOP/noop/.superpowers/sdd/progress.md`. + +--- + +## Self-Review + +**Spec coverage (spec §5 Phase 2 + §6):** staging tables ✅ (T1); propose→diff→confirm/reject ✅ (T2/T4/T5); list_pending/edit_journal/undo ✅ (T4/T5); six edit kinds ✅ (T2); ro-can-stage/rw-applies with tool *invisibility* ✅ (T4/T5 via scope threading); destructive/readOnly annotations ✅; overlay consulted by read tools, mirror pristine ✅ (T6); journal append-only + idempotent + sequenced for Phase 3 ✅ (T1/T5/T7); `/edits?since=` ✅ (T7); deploy ✅ (T8). Honesty constraint (dailyMetric-derived numbers) documented in tool descriptions + README ✅ (T6/T8). + +**Placeholder scan:** none — every code step carries complete code. Task 4's `registerResolutionTools` no-op is an explicit, named Task-5 seam, implemented in Task 5 with full code. + +**Type consistency:** `ProposalRow`/`JournalRow` (T1) used by T4/T5/T7; overlay keys + `computeOverlay` (T3) consumed in T6 with the same helpers; `buildMcpServer(cfg, scope)` signature change is propagated (mcp.ts, server.ts, tools/index.ts) in one task (T4); `asTool` shape matches Phase 1's envelope. + +**Known deltas from Phase 1 the implementers must respect:** suite count grows 41→45→48→51→54→56→60→62; `openServerDb` moves to serverdb.ts with a compatibility re-export from ingest.ts. + +--- + +## Execution Handoff + +Plan complete and saved to `docs/superpowers/plans/2026-07-11-noop-cloud-2-gated-writes.md`. Two execution options: + +**1. Subagent-Driven (recommended)** — fresh subagent per task, review between tasks. + +**2. Inline Execution** — executing-plans in this session with checkpoints. diff --git a/docs/superpowers/plans/2026-07-11-noop-cloud-2b-granular.md b/docs/superpowers/plans/2026-07-11-noop-cloud-2b-granular.md new file mode 100644 index 0000000000..27dc983b00 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-noop-cloud-2b-granular.md @@ -0,0 +1,358 @@ +# noop-cloud Phase 2b — Granular Reads + Granular Edit Kinds + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Extends the Phase-2 plan (2026-07-11-noop-cloud-2-gated-writes.md); execute AFTER its Task 7. Phase 2's Task 8 (README/deploy) is SUPERSEDED by this plan's Task 4 — one deploy covers both. + +**Goal:** The AI can read the raw evidence (per-second heart rate, stage-by-stage hypnogram, movement) and edit at the same granularity (rewrite a night's stages, delete artifact HR ranges) — so "you were actually asleep until 6am" is provable from data and fixable end-to-end. + +**Architecture:** Two new read tools over existing mirror tables (`hrSample`, `sleepSession.stagesJSON`), downsample-windowed to keep responses LLM-sized. Two new edit kinds flowing through the existing propose→confirm→journal→overlay rail; `hr_series` and `sleep_detail` are overlay-aware from day one. + +**Tech Stack:** unchanged (SDK 1.29.0, better-sqlite3 12.11.1). Repo `/Users/vk/VKDEV/NOOP/noop-cloud`. + +## Global Constraints + +- Mirror read-only always; all edit state in server.sqlite; journal append-only (unchanged from Phase 2). +- `hr_series`: max span **7 days** per call (reject wider with `{ error: "span_too_wide" }`); optional `bucketSeconds` (60..3600) returns per-bucket `{ts, avg, min, max, n}`; raw mode caps at **5000 samples** (`truncated: true` + hint to bucket when hit). Multi-source: optional `deviceId`, else all sources, each sample carries `deviceId`/`family`. +- `sleep_detail`: one session by `(deviceId, startTs)`; returns bounds (overlay-adjusted, `edited` marker), `stages` parsed from stagesJSON (`[{start,end,stage}]`, overlay-replaced when an `edit_sleep_stages` edit is active), plus `hrDuringSleep` (bucketed 5-min from hrSample within bounds, same deviceId's family fallback: exact deviceId first, else any source overlapping the window — labeled per sample). +- New edit kinds (total becomes **eight**): `edit_sleep_stages` payload `{ deviceId, startTs, stages: [{start,end,stage}] (1..96 segments, contiguous non-overlapping ascending, stage ∈ awake|light|deep|rem) }`; `delete_hr_range` payload `{ deviceId, fromTs, toTs (toTs>fromTs, span ≤ 6h) }`. Both `.strict()`, both captureBefore (sleep row must exist / count of HR samples in range must be > 0 → else `target_not_found`), human diffs show old→new (stage minutes summary; sample count dropped). +- Overlay: `stageEdits: Map`; `deletedHrRanges: {deviceId, fromTs, toTs, editId}[]`. `hr_series` excludes samples inside deleted ranges (matching deviceId); `sleep_detail` reflects both stage edits and HR deletions; `metric_series`/others unchanged. +- Tool annotations `readOnlyHint: true` for both reads; guard pattern (`notIngested`) on both. +- Fixture additions must keep every existing test green (fixtures gain stagesJSON on one session + denser HR; adjust existing counts ONLY by appending new data that no existing assertion pins — existing hrSample rows are 5/day under oura-api and `hrCoverageDays` asserts 4 days: keep additions inside the same 4 days). + +--- + +### Task 1: Fixture enrichment + `hr_series` / `sleep_detail` read tools + +**Files:** +- Modify: `noop-cloud/test/fixtures/make-fixture.ts` (one WHOOP sleep session gets a real stagesJSON; add my-whoop HR samples across the 2026-06-13 night, 5-min cadence 03:00–10:00) +- Create: `noop-cloud/src/tools/granular.ts` +- Modify: `noop-cloud/src/tools/index.ts` (register) +- Test: `noop-cloud/test/tools-granular.test.ts` + +**Interfaces:** +- Produces: `hrSeries(cfg, args: { from: string(ISO datetime or YYYY-MM-DD), to: string, deviceId?: string, bucketSeconds?: number })` → `{ samples: [{ts, bpm, deviceId, family}] , truncated?: true }` or `{ buckets: [{ts, avg, min, max, n, deviceId, family}] }`; `{ error: "span_too_wide" }` past 7d; notIngested guard. +- Produces: `sleepDetail(cfg, args: { deviceId: string, startTs: number })` → `{ session: {deviceId, family, startTs, endTs, durationMin, efficiency, restingHr, avgHrv, edited?}, stages: [{start,end,stage}] | null, stagesEdited?: true, hrDuringSleep: [{ts,bpm,deviceId}], notFound?: true }`. +- Produces: `registerGranularTools(server, cfg)` (both `readOnlyHint: true`). +- Mirror gains two methods (Modify `src/mirror.ts`): `hrSamplesRange(opts: {fromTs: number, toTs: number, deviceId?: string, limit: number}): {deviceId: string, ts: number, bpm: number}[]` and `sleepSessionAt(deviceId: string, startTs: number): (SleepRow & { stagesJSON: string | null }) | null`. + +- [ ] **Step 1: Failing test** — `test/tools-granular.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { hrSeries, sleepDetail } from "../src/tools/granular.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/granular"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +const SLEEP_TS = Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000); + +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("granular reads", () => { + it("hr_series raw returns my-whoop night samples with family", () => { + const r = hrSeries(cfg, { from: "2026-06-13T03:00:00Z", to: "2026-06-13T10:00:00Z", deviceId: "my-whoop" }) as any; + expect(r.samples.length).toBeGreaterThanOrEqual(80); // 5-min cadence over 7h + expect(r.samples[0].family).toBe("whoop"); + }); + it("hr_series buckets aggregate", () => { + const r = hrSeries(cfg, { from: "2026-06-13T03:00:00Z", to: "2026-06-13T10:00:00Z", deviceId: "my-whoop", bucketSeconds: 3600 }) as any; + expect(r.buckets.length).toBeGreaterThanOrEqual(6); + expect(r.buckets[0]).toHaveProperty("avg"); + }); + it("hr_series rejects >7d span", () => { + expect((hrSeries(cfg, { from: "2026-06-01", to: "2026-06-13" }) as any).error).toBe("span_too_wide"); + }); + it("sleep_detail returns stages + in-sleep HR", () => { + const r = sleepDetail(cfg, { deviceId: "my-whoop", startTs: SLEEP_TS }) as any; + expect(r.session.durationMin).toBeGreaterThan(0); + expect(r.stages.length).toBeGreaterThanOrEqual(3); + expect(r.hrDuringSleep.length).toBeGreaterThanOrEqual(80); + }); + it("sleep_detail notFound for a ghost session", () => { + expect((sleepDetail(cfg, { deviceId: "my-whoop", startTs: 1 }) as any).notFound).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run to verify FAIL** (`npm test -- tools-granular`). + +- [ ] **Step 3: Fixture additions** — in `buildMirrorSqlite` after the existing hr inserts: + +```typescript +// Granular night data (2026-06-13, my-whoop): stages on the sleep row + 5-min HR across the night. +const night = tsOf("2026-06-13", 3); +const stages = JSON.stringify([ + { start: night, end: night + 3600, stage: "light" }, + { start: night + 3600, end: night + 10800, stage: "deep" }, + { start: night + 10800, end: night + 18000, stage: "rem" }, + { start: night + 18000, end: night + 25200, stage: "light" }, +]); +db.prepare("UPDATE sleepSession SET stagesJSON = ? WHERE deviceId = 'my-whoop' AND startTs = ?").run(stages, night); +for (let t = night; t <= night + 25200; t += 300) hr.run("my-whoop", t, 46 + Math.round(8 * Math.abs(Math.sin(t / 3000)))); +``` + +- [ ] **Step 4: Mirror methods** — append to `src/mirror.ts` class: + +```typescript +hrSamplesRange(opts: { fromTs: number; toTs: number; deviceId?: string; limit: number }): { deviceId: string; ts: number; bpm: number }[] { + const where = ["ts >= ? AND ts <= ?"]; const args: any[] = [opts.fromTs, opts.toTs]; + if (opts.deviceId) { where.push("deviceId = ?"); args.push(opts.deviceId); } + args.push(opts.limit); + return this.db.prepare(`SELECT deviceId, ts, bpm FROM hrSample WHERE ${where.join(" AND ")} ORDER BY ts LIMIT ?`).all(...args) as any[]; +} +sleepSessionAt(deviceId: string, startTs: number) { + return (this.db.prepare("SELECT deviceId, startTs, endTs, efficiency, restingHr, avgHrv, userEdited, stagesJSON FROM sleepSession WHERE deviceId = ? AND startTs = ?").get(deviceId, startTs) as any) ?? null; +} +``` + +- [ ] **Step 5: Implement `src/tools/granular.ts`** + +```typescript +import fs from "node:fs"; +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { Config } from "../config.js"; +import { Mirror, sourceFamily } from "../mirror.js"; +import { computeOverlay, sleepKeyOf } from "../edits/overlay.js"; + +const asTool = (obj: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(obj, null, 2) }], structuredContent: obj as Record }); +const toTs = (s: string, endOfDay = false) => /^\d{4}-\d{2}-\d{2}$/.test(s) + ? Math.floor(new Date(`${s}T${endOfDay ? "23:59:59" : "00:00:00"}Z`).getTime() / 1000) + : Math.floor(new Date(s).getTime() / 1000); +const MAX_SPAN_S = 7 * 86_400, RAW_CAP = 5000; + +function dropDeleted(samples: { deviceId: string; ts: number; bpm: number }[], ranges: { deviceId: string; fromTs: number; toTs: number }[]) { + if (!ranges.length) return samples; + return samples.filter((s) => !ranges.some((r) => r.deviceId === s.deviceId && s.ts >= r.fromTs && s.ts <= r.toTs)); +} + +export function hrSeries(cfg: Config, args: { from: string; to: string; deviceId?: string; bucketSeconds?: number }) { + if (!fs.existsSync(cfg.mirrorPath)) return { samples: [], notIngested: true }; + const fromTs = toTs(args.from), toT = toTs(args.to, true); + if (!Number.isFinite(fromTs) || !Number.isFinite(toT)) return { error: "bad_range" }; + if (toT - fromTs > MAX_SPAN_S) return { error: "span_too_wide", maxDays: 7 }; + const ranges = computeOverlay(cfg).deletedHrRanges; + const m = new Mirror(cfg.mirrorPath); + try { + const raw = dropDeleted(m.hrSamplesRange({ fromTs, toTs: toT, deviceId: args.deviceId, limit: 200_000 }), ranges); + if (args.bucketSeconds) { + const b = Math.min(3600, Math.max(60, args.bucketSeconds)); + const byBucket = new Map(); + for (const s of raw) { + const key = `${s.deviceId}|${Math.floor(s.ts / b) * b}`; + const cur = byBucket.get(key) ?? { ts: Math.floor(s.ts / b) * b, deviceId: s.deviceId, sum: 0, min: s.bpm, max: s.bpm, n: 0 }; + cur.sum += s.bpm; cur.min = Math.min(cur.min, s.bpm); cur.max = Math.max(cur.max, s.bpm); cur.n += 1; + byBucket.set(key, cur); + } + const buckets = [...byBucket.values()].sort((a, b2) => a.ts - b2.ts) + .map((x) => ({ ts: x.ts, avg: Math.round((x.sum / x.n) * 10) / 10, min: x.min, max: x.max, n: x.n, deviceId: x.deviceId, family: sourceFamily(x.deviceId) })); + return { buckets }; + } + const capped = raw.slice(0, RAW_CAP); + return { samples: capped.map((s) => ({ ...s, family: sourceFamily(s.deviceId) })), ...(raw.length > RAW_CAP ? { truncated: true, hint: "pass bucketSeconds to aggregate" } : {}) }; + } finally { m.close(); } +} + +export function sleepDetail(cfg: Config, args: { deviceId: string; startTs: number }) { + if (!fs.existsSync(cfg.mirrorPath)) return { notIngested: true }; + const overlay = computeOverlay(cfg); + const m = new Mirror(cfg.mirrorPath); + try { + const row = m.sleepSessionAt(args.deviceId, args.startTs); + if (!row) return { notFound: true }; + const adj = overlay.sleepBounds.get(sleepKeyOf(args.deviceId, args.startTs)); + const startTs = adj?.newStartTs ?? row.startTs, endTs = adj?.newEndTs ?? row.endTs; + const stageEdit = overlay.stageEdits.get(sleepKeyOf(args.deviceId, args.startTs)); + const stages = stageEdit ? stageEdit.stages : (row.stagesJSON ? JSON.parse(row.stagesJSON) : null); + const hr = dropDeleted(m.hrSamplesRange({ fromTs: startTs, toTs: endTs, deviceId: args.deviceId, limit: RAW_CAP }), overlay.deletedHrRanges); + const hrAny = hr.length ? hr : dropDeleted(m.hrSamplesRange({ fromTs: startTs, toTs: endTs, limit: RAW_CAP }), overlay.deletedHrRanges); + return { + session: { deviceId: row.deviceId, family: sourceFamily(row.deviceId), startTs, endTs, durationMin: Math.round((endTs - startTs) / 60), efficiency: row.efficiency, restingHr: row.restingHr, avgHrv: row.avgHrv, ...(adj ? { edited: true, editId: adj.editId } : {}) }, + stages, ...(stageEdit ? { stagesEdited: true, editId: stageEdit.editId } : {}), + hrDuringSleep: hrAny.map((s) => ({ ts: s.ts, bpm: s.bpm, deviceId: s.deviceId })), + }; + } finally { m.close(); } +} + +export function registerGranularTools(server: McpServer, cfg: Config): void { + server.registerTool("hr_series", { + title: "Heart-rate series", + description: "Raw or bucketed heart-rate samples for a time range (max 7 days). Use to inspect what actually happened (e.g. verify sleep/wake from HR). Reflects confirmed HR deletions.", + inputSchema: { from: z.string(), to: z.string(), deviceId: z.string().optional(), bucketSeconds: z.number().int().min(60).max(3600).optional() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(hrSeries(cfg, a))); + + server.registerTool("sleep_detail", { + title: "Sleep night detail", + description: "One night's full evidence: bounds, stage-by-stage hypnogram, and in-sleep heart-rate. Reflects confirmed stage edits, bound adjustments, and HR deletions.", + inputSchema: { deviceId: z.string(), startTs: z.number().int() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (a) => asTool(sleepDetail(cfg, a))); +} +``` +NOTE: this imports `overlay.deletedHrRanges` / `overlay.stageEdits`, which exist only after Task 2. For THIS task, add both fields to the `Overlay` interface + `computeOverlay` as empty structures (`stageEdits: new Map(), deletedHrRanges: []`) with a `// filled by edit kinds in 2b Task 2` comment — keeps this task self-contained and green. + +- [ ] **Step 6: register in `src/tools/index.ts`** (`registerGranularTools(server, cfg);`). + +- [ ] **Step 7: Full suite** — existing tests must stay green (fixture additions are within pinned bounds: my-whoop hrSamples are new rows; `hrCoverageDays("oura-api")` untouched). Expected: prior count + 5. + +- [ ] **Step 8: Commit** — `feat: granular reads — hr_series + sleep_detail (overlay-aware)` + +--- + +### Task 2: Edit kinds `edit_sleep_stages` + `delete_hr_range` + +**Files:** +- Modify: `noop-cloud/src/edits/kinds.ts` (two new kinds; EDIT_KINDS becomes 8) +- Modify: `noop-cloud/src/edits/diff.ts` (captureBefore + renderDiff for both) +- Modify: `noop-cloud/src/edits/overlay.ts` (fill `stageEdits` + `deletedHrRanges`) +- Test: `noop-cloud/test/edits-granular.test.ts` + +**Interfaces:** +- `edit_sleep_stages` schema: `{ deviceId: min1, startTs: int, stages: array(1..96 of {start:int,end:int,stage:enum(awake|light|deep|rem)}) }` `.strict()` + refine: ascending, contiguous (`stages[i].end === stages[i+1].start`), every `end > start`. +- `delete_hr_range` schema: `{ deviceId: min1, fromTs: int, toTs: int }` `.strict()` + refine `toTs > fromTs && toTs - fromTs <= 21_600`. +- captureBefore: `edit_sleep_stages` → the sleepSession row (target_not_found if absent); `delete_hr_range` → `{ count }` from `SELECT COUNT(*) FROM hrSample WHERE deviceId=? AND ts BETWEEN ? AND ?` (target_not_found if 0). +- renderDiff: stages → `RESTAGE sleep @ : light 60m/deep 120m/rem 120m/light 120m ⇒ ` (compute per-stage minutes from old stagesJSON if present, else "no stages"); hr → `DELETE ${count} HR samples ${iso(fromTs)} → ${iso(toTs)} (${deviceId})`. +- overlay: `edit_sleep_stages` → `stageEdits.set(sleepKeyOf(deviceId,startTs), {stages, editId})` (later wins); `delete_hr_range` → push `{deviceId, fromTs, toTs, editId}`. + +- [ ] **Step 1: Failing test** — `test/edits-granular.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { appendJournal } from "../src/staging.js"; +import { payloadSchema, EDIT_KINDS } from "../src/edits/kinds.js"; +import { captureBefore, renderDiff } from "../src/edits/diff.js"; +import { computeOverlay, sleepKeyOf } from "../src/edits/overlay.js"; +import { hrSeries, sleepDetail } from "../src/tools/granular.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/edits-granular"); +const cfg = { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000 } as any; +const NIGHT = Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000); +const NEW_STAGES = [{ start: NIGHT, end: NIGHT + 10800, stage: "deep" }, { start: NIGHT + 10800, end: NIGHT + 25200, stage: "light" }]; + +beforeAll(() => { fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg); }); + +describe("granular edit kinds", () => { + it("EDIT_KINDS is 8 and schemas validate", () => { + expect(EDIT_KINDS.length).toBe(8); + expect(payloadSchema("edit_sleep_stages").safeParse({ deviceId: "my-whoop", startTs: NIGHT, stages: NEW_STAGES }).success).toBe(true); + expect(payloadSchema("edit_sleep_stages").safeParse({ deviceId: "d", startTs: 1, stages: [{ start: 5, end: 4, stage: "deep" }] }).success).toBe(false); + expect(payloadSchema("delete_hr_range").safeParse({ deviceId: "my-whoop", fromTs: NIGHT, toTs: NIGHT + 30000 }).success).toBe(false); // >6h + }); + it("captureBefore + renderDiff for both kinds", () => { + const b1 = captureBefore(cfg, "edit_sleep_stages", { deviceId: "my-whoop", startTs: NIGHT, stages: NEW_STAGES }) as any; + expect(b1.stagesJSON).toBeTruthy(); + expect(renderDiff("edit_sleep_stages", { deviceId: "my-whoop", startTs: NIGHT, stages: NEW_STAGES }, b1)).toContain("deep"); + const b2 = captureBefore(cfg, "delete_hr_range", { deviceId: "my-whoop", fromTs: NIGHT, toTs: NIGHT + 3600 }) as any; + expect(b2.count).toBeGreaterThan(0); + expect(renderDiff("delete_hr_range", { deviceId: "my-whoop", fromTs: NIGHT, toTs: NIGHT + 3600 }, b2)).toMatch(/DELETE \d+ HR/); + }); + it("journal entries flow into overlay and granular reads", () => { + appendJournal(cfg, { editId: "g1", kind: "edit_sleep_stages", payloadJSON: JSON.stringify({ deviceId: "my-whoop", startTs: NIGHT, stages: NEW_STAGES }), beforeJSON: null, rationale: null }); + appendJournal(cfg, { editId: "g2", kind: "delete_hr_range", payloadJSON: JSON.stringify({ deviceId: "my-whoop", fromTs: NIGHT, toTs: NIGHT + 3599 }), beforeJSON: null, rationale: null }); + const o = computeOverlay(cfg); + expect(o.stageEdits.get(sleepKeyOf("my-whoop", NIGHT))?.stages.length).toBe(2); + expect(o.deletedHrRanges.length).toBe(1); + const d = sleepDetail(cfg, { deviceId: "my-whoop", startTs: NIGHT }) as any; + expect(d.stagesEdited).toBe(true); + expect(d.stages.length).toBe(2); + const hr = hrSeries(cfg, { from: "2026-06-13T03:00:00Z", to: "2026-06-13T10:00:00Z", deviceId: "my-whoop" }) as any; + const inRange = hr.samples.filter((s: any) => s.ts >= NIGHT && s.ts <= NIGHT + 3599); + expect(inRange.length).toBe(0); // deleted range excluded + }); +}); +``` + +- [ ] **Step 2: FAIL run.** +- [ ] **Step 3: kinds.ts** — extend `EDIT_KINDS`, add both schemas with the refines above (stage enum via `z.enum(["awake","light","deep","rem"])`; contiguity refine loops pairs). +- [ ] **Step 4: diff.ts** — `captureBefore` cases (`edit_sleep_stages` → sleepSession row; `delete_hr_range` → `{count}` via COUNT query, throw target_not_found on 0); `renderDiff` cases with stage-minute summaries: helper `stageSummary(stages)` → `"light 60m/deep 120m/…"` (merge repeated stages by summing). +- [ ] **Step 5: overlay.ts** — fill the two structures in the switch (replacing the Task-1 placeholders' comment). +- [ ] **Step 6: Full suite green (prior + 3); build clean.** +- [ ] **Step 7: Commit** — `feat: granular edit kinds — edit_sleep_stages + delete_hr_range` + +--- + +### Task 3: End-to-end lifecycle test through the live MCP surface + +**Files:** +- Test: `noop-cloud/test/tools-granular-e2e.test.ts` + +**Interfaces:** none new — this pins the full loop the feature exists for. + +- [ ] **Step 1: Write the test** (propose edit_sleep_stages via ro → confirm via rw → sleep_detail reflects → undo via rw → sleep_detail reverts; same HTTP harness as test/tools-resolve.test.ts — copy its `mcp()` helper verbatim): + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { buildNoopbak } from "./fixtures/make-fixture.js"; +import { ingestNoopbak } from "../src/ingest.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/granular-e2e"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } +const NIGHT = Math.floor(new Date("2026-06-13T03:00:00Z").getTime() / 1000); + +function mcp(port: number, token: string, id: number, name: string, args: object): Promise { + return new Promise((resolve) => { + const payload = JSON.stringify({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } }); + const r = http.request({ port, path: "/mcp", method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json, text/event-stream" } }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => { + const line = d.split("\n").find((l) => l.startsWith("data:")) ?? d; + resolve(JSON.parse(line.replace(/^data:\s*/, "")).result.structuredContent); + }); + }); + r.end(payload); + }); +} + +let port = 0; let server: any; +beforeAll(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + const z = path.join(dataDir, "b.noopbak"); buildNoopbak(z); ingestNoopbak(fs.readFileSync(z), cfg()); + const app = createApp(cfg()); server = app.listen(0); port = (server.address() as any).port; +}); + +describe("granular edit full loop over MCP", () => { + it("propose(ro) → confirm(rw) → sleep_detail reflects → undo(rw) → reverts", async () => { + const stages = [{ start: NIGHT, end: NIGHT + 12600, stage: "deep" }, { start: NIGHT + 12600, end: NIGHT + 25200, stage: "light" }]; + const p = await mcp(port, cfg().roToken, 1, "propose_edit", { kind: "edit_sleep_stages", payload: { deviceId: "my-whoop", startTs: NIGHT, stages }, rationale: "HR shows deep sleep until 6:30, band misread movement" }); + expect(p.status).toBe("pending"); + expect(p.diff).toContain("deep"); + const c = await mcp(port, cfg().rwToken, 2, "confirm_edit", { id: p.id }); + expect(c.applied).toBe(true); + const d1 = await mcp(port, cfg().roToken, 3, "sleep_detail", { deviceId: "my-whoop", startTs: NIGHT }); + expect(d1.stagesEdited).toBe(true); + expect(d1.stages.length).toBe(2); + const u = await mcp(port, cfg().rwToken, 4, "undo_edit", { seq: c.seq }); + expect(u.undoneSeq).toBe(c.seq); + const d2 = await mcp(port, cfg().roToken, 5, "sleep_detail", { deviceId: "my-whoop", startTs: NIGHT }); + expect(d2.stagesEdited).toBeUndefined(); + expect(d2.stages.length).toBe(4); // fixture's original hypnogram + }); +}); +``` + +- [ ] **Step 2: Run (expect PASS if Tasks 1-2 are correct — this is an integration pin, not TDD-new-code). Full suite green.** +- [ ] **Step 3: Commit** — `test: granular edit full loop (propose→confirm→read→undo) over live MCP` + +--- + +### Task 4: README + deploy + live smoke (supersedes Phase-2 Task 8) + +- [ ] **Step 1:** README: add Phase-2 "Editing your data" section from the Phase-2 plan's Task 8 text, PLUS a "Granular evidence & edits" paragraph naming hr_series / sleep_detail / edit_sleep_stages / delete_hr_range and the 3am-vs-6am example. +- [ ] **Step 2:** `npm test && npm run build` — all green. +- [ ] **Step 3:** Commit `docs: Phase-2 + granular editing README`; then `fly deploy --ha=false --yes`. +- [ ] **Step 4: Live smoke** (`.env` tokens): ro tools/list contains hr_series+sleep_detail+propose_edit, NOT confirm_edit; rw contains confirm_edit; `GET /edits` 200; propose+confirm+undo one edit_sleep_stages round-trip against the fixture data currently on the server; `data_freshness` shows journalSeq ≥ 2. +- [ ] **Step 5:** Ledger the deploy in `/Users/vk/VKDEV/NOOP/noop/.superpowers/sdd/progress.md`. + +--- + +## Self-Review + +**Coverage vs the user's ask:** read any day at sensor granularity (hr_series 7d-windowed, sleep_detail full night) ✅; reason from evidence (both tools' descriptions steer that use) ✅; edit duration/stages/HR — adjust_sleep_bounds (P2) + edit_sleep_stages + delete_hr_range ✅; full audit loop (same propose/confirm/journal/undo rail) ✅; write-back seam (journal entries carry kind+payload+before for Phase-3 apply; stages align with the app's userEdited protection) ✅. +**Placeholder scan:** Task 1 Step 5's overlay placeholder is explicit and filled by Task 2 (named seam, same pattern as P2 T4→T5). No TBDs. +**Type consistency:** `Overlay.stageEdits`/`deletedHrRanges` names consistent across Tasks 1/2; `hrSamplesRange`/`sleepSessionAt` signatures match call sites; sleepKeyOf reused; EDIT_KINDS length assertions updated (6→8) — NOTE for implementers: `test/edits-diff.test.ts` asserts `EDIT_KINDS.length).toBe(6)` from Phase 2 Task 2 — Task 2 of THIS plan must update that assertion to 8 in the same commit. diff --git a/docs/superpowers/plans/2026-07-12-noop-cloud-3-phone-sync.md b/docs/superpowers/plans/2026-07-12-noop-cloud-3-phone-sync.md new file mode 100644 index 0000000000..99fad928b8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-noop-cloud-3-phone-sync.md @@ -0,0 +1,296 @@ +# noop-cloud Phase 3 — Phone Sync-Back + Apple Health Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Tasks 1 is in `/Users/vk/VKDEV/NOOP/noop-cloud` (TypeScript); Tasks 2–6 are in `/Users/vk/VKDEV/NOOP/noop` (Swift, personal fork lane). Steps use checkbox syntax. + +**Goal:** Confirmed cloud edits (sleep bounds/stages, workout fixes/deletes/adds, HR-range deletions, metric-point deletions) land on VK's iPhone — applied through recompute-safe store APIs — and corrected sleep/workouts/HR re-export to Apple Health via the merged write-back; every applied edit is acked back so the server journal tracks phone state. + +**Architecture:** The phone PULLS `GET /edits?since=` (never pushed to). A gated `Strand/CloudSync/` module (compiled only under `CLOUD_SYNC`, same untracked-xcconfig mechanism as `OURA_CLOUD_IMPORT`) fetches active journal rows, applies them via a `CloudEditApplier` that uses `applySleepEdit`/`updateSleepStages` (userEdited=1 → survives recompute), physical deletes + a new `cloudTombstone` table (migration v26) so recompute/re-import can't resurrect deletions, and the `"noop-cloud"` source namespace for adds/corrected copies. After a batch that touched sleep/workouts/HR, the module triggers the existing HealthKit write-back for the affected window. Cursor lives in the store's `cursors` table; acks POST back with rw. + +**Tech Stack:** Swift (app target `Strand/CloudSync/`, package `Packages/WhoopStore`), XCTest (StrandTests + WhoopStore package tests), TypeScript/vitest for the server ack endpoint. Device deploy via the established personal-signing pipeline. + +## Global Constraints + +- **Fork-only, flag-gated.** Every new app-target file starts with `#if CLOUD_SYNC` and the header comment pattern used by `Strand/Oura/*` ("Compiled ONLY when the CLOUD_SYNC compilation condition is set…"). The flag is added to `SWIFT_ACTIVE_COMPILATION_CONDITIONS` in the untracked `Strand/Oura/OuraSecrets.xcconfig` (rename-scope stays; the example file documents it). A default build contains none of this code. +- **Never mutate another source's rows in place.** Sleep edits go through `applySleepEdit` / `updateSleepStages` ONLY (they set/require `userEdited=1`). `fix_workout` = tombstone original + corrected copy under `deviceId "noop-cloud"`. `add_workout` under `"noop-cloud"`. Deletions = physical delete + `cloudTombstone` row; resurrection guards live in the store's upsert/insert paths. +- **Journal replay semantics:** apply rows in `seq` order where `kind != "undo" AND undoneBySeq == null` and seq > cursor. A row that arrives ALREADY undone is skipped (never applied). An edit undone AFTER the phone applied it is **out of MVP scope**: the applier detects it (acked seq now shows `undoneBySeq != null`) and surfaces "N server undos need manual attention" in the sync status — documented, not silently ignored. +- **`set_baseline_note` is server-only** — the applier records it as acked-no-op. +- **Idempotency:** re-applying an already-applied edit must be harmless (applySleepEdit is last-write-wins on the same values; deletes of absent rows are 0-row no-ops; adds under noop-cloud upsert on the same key). The cursor only advances after a successful batch + ack. +- **Migrations:** new migration is **`v26-cloud-tombstone`**, registered AFTER `v25-oura-raw`, additive only, with a migration test. Never edit existing migrations. +- **HealthKit:** iOS-only (`#if os(iOS)`); reuse the merged write-back (`HealthKitBridge.writeBack` path) rather than new HK write code — Task 5 first READS `StrandiOS/Health/HealthKitBridge.swift` to determine the public trigger + dedup semantics and adapts (expose a minimal public entry if needed; do not duplicate its sample-building logic). +- **Both app targets must compile** (`Strand` macOS + `NOOPiOS`) — CLOUD_SYNC files are shared; HealthKit calls guarded `#if os(iOS) && canImport(HealthKit)`. CI won't catch app-target breaks: build both locally. +- **Server change (Task 1) is additive**: `POST /edits/ack {seqs:[...]}` (rw) sets `ackedAt` on journal rows (new nullable column via `ALTER TABLE` guarded `IF NOT EXISTS`-style check); `GET /edits` gains `ackedAt` passthrough. SDK/pins unchanged; deploy at the end of Task 1. +- **Commits:** conventional, no Claude attribution (repo rule). + +--- + +### Task 1 (noop-cloud repo): `POST /edits/ack` + `ackedAt` + +**Files:** +- Modify: `noop-cloud/src/serverdb.ts` (add `ackedAt INTEGER` to editJournal CREATE; plus a startup `ALTER TABLE` catch for existing DBs) +- Modify: `noop-cloud/src/staging.ts` (`ackEdits(cfg, seqs: number[]): number`; include `ackedAt` in `JournalRow`) +- Modify: `noop-cloud/src/server.ts` (route) +- Test: `noop-cloud/test/edits-ack.test.ts` + +**Interfaces:** +- Produces: `POST /edits/ack` (rw scope), JSON body `{ seqs: number[] }` (1..500 ints) → `{ acked: number }` (count of rows updated where `ackedAt IS NULL`); invalid body → 400 `{ error: "bad_seqs" }`. +- `JournalRow` gains `ackedAt: number | null`; `GET /edits` responses now carry it (additive field). + +- [ ] **Step 1: Failing test** `test/edits-ack.test.ts`: + +```typescript +import { describe, it, expect, beforeAll } from "vitest"; +import fs from "node:fs"; import path from "node:path"; import http from "node:http"; +import { appendJournal, journalSince } from "../src/staging.js"; +import { createApp } from "../src/server.js"; + +const dataDir = path.join(process.cwd(), "test/.tmp/edits-ack"); +function cfg() { return { dataDir, mirrorPath: path.join(dataDir, "mirror.sqlite"), serverDbPath: path.join(dataDir, "server.sqlite"), maxIngestBytes: 262_144_000, roToken: "ro".padEnd(40, "x"), rwToken: "rw".padEnd(40, "y"), port: 0 } as any; } +function post(port: number, token: string, body: object): Promise<{ status: number; json: any }> { + return new Promise((resolve) => { + const payload = JSON.stringify(body); + const r = http.request({ port, path: "/edits/ack", method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" } }, (res) => { + let d = ""; res.on("data", (c) => (d += c)); res.on("end", () => resolve({ status: res.statusCode!, json: d ? JSON.parse(d) : null })); + }); + r.end(payload); + }); +} +beforeAll(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); fs.mkdirSync(dataDir, { recursive: true }); + appendJournal(cfg(), { editId: "a1", kind: "set_baseline_note", payloadJSON: "{}", beforeJSON: null, rationale: null }); + appendJournal(cfg(), { editId: "a2", kind: "set_baseline_note", payloadJSON: "{}", beforeJSON: null, rationale: null }); +}); +describe("POST /edits/ack", () => { + it("requires rw (ro → 401)", async () => { + const app = createApp(cfg()); const s = app.listen(0); const port = (s.address() as any).port; + expect((await post(port, cfg().roToken, { seqs: [1] })).status).toBe(401); s.close(); + }); + it("acks rows once and exposes ackedAt via GET /edits", async () => { + const app = createApp(cfg()); const s = app.listen(0); const port = (s.address() as any).port; + const r1 = await post(port, cfg().rwToken, { seqs: [1, 2] }); + expect(r1.json.acked).toBe(2); + const r2 = await post(port, cfg().rwToken, { seqs: [1, 2] }); + expect(r2.json.acked).toBe(0); // already acked + s.close(); + const rows = journalSince(cfg(), 0); + expect(rows.every((e: any) => e.ackedAt !== null)).toBe(true); + }); + it("400 on garbage", async () => { + const app = createApp(cfg()); const s = app.listen(0); const port = (s.address() as any).port; + expect((await post(port, cfg().rwToken, { seqs: "x" })).status).toBe(400); s.close(); + }); +}); +``` + +- [ ] **Step 2: FAIL run** (`npm test -- edits-ack`). +- [ ] **Step 3: serverdb.ts** — add `ackedAt INTEGER` to the editJournal CREATE, and after the `db.exec`, a guarded migration for pre-existing DBs: + +```typescript +const cols = db.prepare("PRAGMA table_info(editJournal)").all() as any[]; +if (!cols.some((c) => c.name === "ackedAt")) db.exec("ALTER TABLE editJournal ADD COLUMN ackedAt INTEGER"); +``` + +- [ ] **Step 4: staging.ts** — `ackedAt: number | null` on `JournalRow`; and: + +```typescript +export function ackEdits(cfg: C, seqs: number[]): number { + return withDb(cfg, (db) => { + const stmt = db.prepare("UPDATE editJournal SET ackedAt = ? WHERE seq = ? AND ackedAt IS NULL"); + const now = Math.floor(Date.now() / 1000); + let n = 0; for (const s of seqs) n += stmt.run(now, s).changes; + return n; + }); +} +``` + +- [ ] **Step 5: server.ts route** (after GET /edits, before error middleware): + +```typescript +app.post("/edits/ack", requireScope(cfg, "rw"), express.json({ limit: "64kb" }), (req, res) => { + const seqs = (req.body as any)?.seqs; + if (!Array.isArray(seqs) || seqs.length === 0 || seqs.length > 500 || !seqs.every((s) => Number.isInteger(s) && s > 0)) { + return res.status(400).json({ error: "bad_seqs" }); + } + res.json({ acked: ackEdits(cfg, seqs) }); +}); +``` +(import `ackEdits` alongside `journalSince`) + +- [ ] **Step 6: PASS + full suite (expect 77) + build.** +- [ ] **Step 7: Commit `feat: POST /edits/ack — phone-applied tracking` and deploy: `fly deploy --ha=false --yes`; smoke: ack with rw → `{acked:N}`, ro → 401.** + +--- + +### Task 2 (NOOP repo): migration v26 + `CloudTombstoneStore` + resurrection guards + +**Files:** +- Modify: `Packages/WhoopStore/Sources/WhoopStore/Database.swift` (migration `v26-cloud-tombstone`) +- Create: `Packages/WhoopStore/Sources/WhoopStore/CloudTombstoneStore.swift` +- Modify: `Packages/WhoopStore/Sources/WhoopStore/JournalWorkoutAppleCache.swift` (`upsertWorkouts` skips tombstoned) +- Modify: `Packages/WhoopStore/Sources/WhoopStore/MetricSeriesStore.swift` (`upsertMetricSeries` skips tombstoned) +- Modify: `Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift` (`insert` filters HR samples inside tombstoned ranges) +- Create: `Packages/WhoopStore/Sources/WhoopStore/CloudEditDeletes.swift` (`deleteHrRange`, `deleteMetricPoint` physical deletes) +- Test: `Packages/WhoopStore/Tests/WhoopStoreTests/CloudTombstoneTests.swift` + +**Interfaces:** +- Migration `v26-cloud-tombstone`: + +```swift +migrator.registerMigration("v26-cloud-tombstone") { db in + try db.create(table: "cloudTombstone") { t in + t.autoIncrementedPrimaryKey("id") + t.column("kind", .text).notNull() // "workout" | "hrRange" | "metricPoint" + t.column("deviceId", .text).notNull() + t.column("startTs", .integer) // workout: startTs; hrRange: fromTs + t.column("endTs", .integer) // hrRange: toTs + t.column("sport", .text) // workout + t.column("day", .text) // metricPoint + t.column("key", .text) // metricPoint + t.column("editSeq", .integer).notNull() // server journal seq (idempotency) + t.column("createdAt", .integer).notNull() + t.uniqueKey(["kind", "editSeq"]) + } + try db.create(index: "idx_cloudTombstone_kind_device", on: "cloudTombstone", columns: ["kind", "deviceId"]) +} +``` +- `CloudTombstoneStore` (actor-friendly `WhoopStore` extension, same style as `OuraRawStore`): + - `addTombstone(kind: String, deviceId: String, startTs: Int?, endTs: Int?, sport: String?, day: String?, key: String?, editSeq: Int) async throws` (INSERT OR IGNORE on the unique key) + - `workoutTombstones(deviceId: String) async throws -> [(startTs: Int, sport: String)]` + - `hrTombstoneRanges(deviceId: String) async throws -> [(fromTs: Int, toTs: Int)]` + - `metricPointTombstones(deviceId: String) async throws -> [(day: String, key: String)]` +- Guards (all inside the store's existing write paths, same DB connection): + - `upsertWorkouts`: before insert, drop input rows matching a `cloudTombstone` row `kind='workout' AND deviceId=? AND startTs=? AND sport=?` (single pre-query into a Set). + - `upsertMetricSeries`: same for `kind='metricPoint'` on `(day,key)`. + - `StreamStore.insert`: load `hrRange` tombstones for the deviceId once per call; filter `streams.hr` samples with `fromTs <= ts <= toTs`. +- `CloudEditDeletes`: `deleteHrRange(deviceId: String, fromTs: Int, toTs: Int) async throws -> Int`; `deleteMetricPoint(deviceId: String, day: String, key: String) async throws -> Int` (plain DELETEs returning changes). + +- [ ] **Step 1: Failing test** `CloudTombstoneTests.swift`: + +```swift +import XCTest +@testable import WhoopStore +import WhoopProtocol + +final class CloudTombstoneTests: XCTestCase { + func testWorkoutTombstoneBlocksResurrection() async throws { + let store = try await WhoopStore.inMemory() + _ = try await store.upsertWorkouts([WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", durationS: 100, energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, zonesJSON: nil, notes: nil)], deviceId: "d1") + try await store.addTombstone(kind: "workout", deviceId: "d1", startTs: 100, endTs: nil, sport: "running", day: nil, key: nil, editSeq: 7) + _ = try await store.deleteWorkouts(deviceId: "d1", sport: "running", from: 100, to: 100) + // Re-import attempts to resurrect — the guard must drop it. + let n = try await store.upsertWorkouts([WorkoutRow(startTs: 100, endTs: 200, sport: "running", source: "whoop", durationS: 100, energyKcal: nil, avgHr: nil, maxHr: nil, strain: nil, distanceM: nil, zonesJSON: nil, notes: nil)], deviceId: "d1") + XCTAssertEqual(n, 0) + // Idempotent tombstone insert (same kind+editSeq). + try await store.addTombstone(kind: "workout", deviceId: "d1", startTs: 100, endTs: nil, sport: "running", day: nil, key: nil, editSeq: 7) + } + func testHrRangeTombstoneFiltersInserts() async throws { + let store = try await WhoopStore.inMemory() + try await store.addTombstone(kind: "hrRange", deviceId: "d1", startTs: 1000, endTs: 2000, sport: nil, day: nil, key: nil, editSeq: 8) + try await store.insert(Streams(hr: [HRSample(ts: 1500, bpm: 60), HRSample(ts: 2500, bpm: 61)]), deviceId: "d1") + let survivors = try await store.hrSamples(deviceId: "d1", from: 0, to: 10_000, limit: 10) + XCTAssertEqual(survivors.map(\.ts), [2500]) + } + func testDeleteHelpersAndMetricGuard() async throws { + let store = try await WhoopStore.inMemory() + try await store.insert(Streams(hr: [HRSample(ts: 1200, bpm: 60)]), deviceId: "d1") + XCTAssertEqual(try await store.deleteHrRange(deviceId: "d1", fromTs: 1000, toTs: 2000), 1) + _ = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "ref_sleep_score", value: 80)], deviceId: "d1") + try await store.addTombstone(kind: "metricPoint", deviceId: "d1", startTs: nil, endTs: nil, sport: nil, day: "2026-07-01", key: "ref_sleep_score", editSeq: 9) + XCTAssertEqual(try await store.deleteMetricPoint(deviceId: "d1", day: "2026-07-01", key: "ref_sleep_score"), 1) + let n = try await store.upsertMetricSeries([MetricPoint(day: "2026-07-01", key: "ref_sleep_score", value: 81)], deviceId: "d1") + XCTAssertEqual(n, 0) // guard drops the tombstoned point + } +} +``` +(Adjust `WorkoutRow`/`MetricPoint`/`Streams` initializers to the real signatures — read the source files first; the test intent is contractual, exact initializers are whatever the package defines.) + +- [ ] **Step 2: FAIL** — `swift test --package-path Packages/WhoopStore --filter CloudTombstoneTests`. +- [ ] **Step 3: implement migration + store + guards + deletes** per Interfaces (read each modified file first; guards are small pre-filters, mirror existing code style; keep guard queries indexed). +- [ ] **Step 4: PASS filter + FULL package suite** (`swift test --package-path Packages/WhoopStore`) — zero regressions. +- [ ] **Step 5: Commit** `feat(store): cloudTombstone (v26) + resurrection guards + cloud delete helpers` + +--- + +### Task 3 (NOOP repo): `CloudSyncClient` + settings (gated app-target module) + +**Files:** +- Create: `Strand/CloudSync/CloudSyncSettings.swift` (Keychain URL+token, same pattern as `OuraTokenStore`) +- Create: `Strand/CloudSync/CloudEdit.swift` (Decodable journal row + `/edits` response) +- Create: `Strand/CloudSync/CloudSyncClient.swift` (fetch since-cursor, ack) +- Test: `StrandTests/CloudSyncClientTests.swift` (URLProtocol stub, same pattern as `OuraURLProtocolStub`) + +**Interfaces:** +- All files `#if CLOUD_SYNC` … `#endif`, header comment per house pattern. +- `CloudSyncSettings`: `static var serverURL: String?` / `static var token: String?` (Keychain service `"com.noop.cloudsync"`, account `"server-url"` / `"rw-token"`); `static var isConfigured: Bool`. +- `CloudEdit: Decodable, Equatable`: `seq: Int, editId: String, kind: String, payloadJSON: String, beforeJSON: String?, rationale: String?, appliedAt: Int, undoneBySeq: Int?, ackedAt: Int?`. +- `CloudSyncClient` (init `(baseURL: URL, token: String, session: URLSession = .shared)`): + - `fetchEdits(since: Int) async throws -> (edits: [CloudEdit], latestSeq: Int)` — GET `/edits?since=`, Bearer auth, decodes `{edits, latestSeq}`. + - `ack(seqs: [Int]) async throws -> Int` — POST `/edits/ack`, returns `acked`. + - Non-2xx → `CloudSyncError.badResponse(status, bodyPrefix)`. +- Tests: stubbed 200 happy paths (fixture JSON), 401 → error, malformed JSON → error. + +- [ ] **Step 1: failing tests → Step 2: implement → Step 3: PASS via** `xcodebuild -project Strand.xcodeproj -scheme Strand -destination 'platform=macOS' test -only-testing:StrandTests/CloudSyncClientTests CODE_SIGNING_ALLOWED=NO` (after adding `CLOUD_SYNC` to the untracked xcconfig — see Task 6 Step 1; for tests, StrandTests shares the app xcconfig so the flag applies). +- [ ] **Step 4: Commit** `feat(cloudsync): gated client + keychain settings (fork lane)` + +--- + +### Task 4 (NOOP repo): `CloudEditApplier` — journal → store, recompute-safe + +**Files:** +- Create: `Strand/CloudSync/CloudEditApplier.swift` +- Test: `StrandTests/CloudEditApplierTests.swift` (against `WhoopStore.inMemory()`) + +**Interfaces:** +- `struct CloudApplySummary: Equatable { var applied: Int; var skipped: Int; var needsAttention: Int; var touchedSleep: Bool; var touchedWorkouts: Bool; var touchedHr: Bool; var appliedSeqs: [Int] }` +- `enum CloudEditApplier { static func apply(_ edits: [CloudEdit], store: WhoopStore) async -> CloudApplySummary }` + - Processes in seq order. Skips (`skipped += 1`): `kind == "undo"`, `undoneBySeq != nil`, `set_baseline_note`, unknown kinds (forward-compat), malformed payloads (decode failure — log, skip, DO NOT crash). + - `adjust_sleep_bounds` → decode `{deviceId, startTs, newStartTs?, newEndTs?}`; fetch the session's current bounds via the store read; call `applySleepEdit(deviceId:detectedStartTs:newStartTs:newEndTs:)` (missing new value = keep current). 0-row result (session absent) → `needsAttention`. + - `edit_sleep_stages` → `{deviceId, startTs, stages[]}` re-encoded to the app's stagesJSON shape `[{start,end,stage}]` → `updateSleepStages(deviceId:detectedStartTs:stagesJSON:)`. NOTE: `updateSleepStages` only touches `userEdited=1` rows — when the target row has `userEdited=0`, first promote it via `applySleepEdit` with its CURRENT bounds (sets the flag), then update stages. 0 rows → `needsAttention`. + - `delete_workout` → `{deviceId, startTs, sport}` → `deleteWorkouts(deviceId:sport:from:startTs to:startTs)` + `addTombstone(kind:"workout"…, editSeq: seq)`. + - `fix_workout` → tombstone + delete original (as above), then upsert the corrected copy (original fields overlaid with `patch`) under `deviceId "noop-cloud"`; requires `beforeJSON` (the server snapshot) for the original's full fields — if `beforeJSON` nil → `needsAttention`. + - `add_workout` → upsert under `"noop-cloud"`. + - `delete_hr_range` → `deleteHrRange` + `addTombstone(kind:"hrRange"…)`; `delete_metric_point` → `deleteMetricPoint` + tombstone. + - First successful apply that used `"noop-cloud"` registers the pairedDevice row once (id `"noop-cloud"`, brand `"noop-cloud"`, model `"Cloud edits"`, `.cloudImport`, `.paired`) — same registration style as `OuraSyncWriter.persist` step 0. + - Sets `touchedSleep/touchedWorkouts/touchedHr` per kind; `appliedSeqs` collects for ack. +- Tests (in-memory store): one test per kind family — sleep bounds survive a re-`upsertSleepSessions` (recompute simulation asserts userEdited protection engaged); stage edit on a userEdited=0 row (promotion path); fix_workout produces tombstone + noop-cloud copy; malformed payload skips without throwing; undone row skipped; summary flags correct. + +- [ ] **Steps: failing tests → implement → PASS (`-only-testing:StrandTests/CloudEditApplierTests`) → commit** `feat(cloudsync): CloudEditApplier — recompute-safe journal application` + +--- + +### Task 5 (NOOP repo): sync coordinator + Apple Health re-export + UI card + +**Files:** +- Create: `Strand/CloudSync/CloudSyncCoordinator.swift` (fetch→apply→ack→cursor; HK trigger) +- Create: `Strand/CloudSync/CloudSyncModel.swift` (@MainActor ObservableObject; mirrors `OuraConnectModel` shape: busy/statusText; actions `pullNow(repo:)`, `saveSettings(url:token:)`) +- Modify: `Strand/Screens/DataSourcesView.swift` (add `cloudSyncCard` under `#if CLOUD_SYNC`, staggered below the Oura card; fields: server URL, token (SecureField), "Pull edits now" button, status line) +- Modify: `StrandiOS/Health/HealthKitBridge.swift` (ONLY IF NEEDED: expose a public `func rerunWriteBack(whoopStore:days:)` wrapper around the existing private write-back — read the file first; if a public trigger already exists, use it) +- Test: `StrandTests/CloudSyncCoordinatorTests.swift` + +**Interfaces:** +- `CloudSyncCoordinator.pull(store: WhoopStore, client: CloudSyncClient) async throws -> CloudApplySummary`: + 1. `since = store.cursor("cloud_edits") ?? 0` (the cursors table — `setCursor`/`cursor` exist on the store). + 2. `fetchEdits(since:)` → `CloudEditApplier.apply` → if `appliedSeqs` non-empty: `ack(seqs:)` then `setCursor("cloud_edits", latestSeq)`. Cursor advances to `latestSeq` even when everything was skipped-but-acked… **No**: cursor advances to `latestSeq` only after ack succeeds; if ack throws, cursor stays (re-apply is idempotent by design). + 3. Returns summary (UI renders "Applied 3 · skipped 1 · 0 need attention"). +- HK trigger (in `CloudSyncModel.pullNow`, iOS-only guard): when `summary.touchedSleep || touchedWorkouts || touchedHr`, call the write-back entry (days: 30) inside a `do/catch` that appends "· Apple Health re-export failed: " to status rather than failing the pull. +- Coordinator tests use a stub `CloudSyncClient` seam (protocol `CloudEditFetching` with fetch/ack, client conforms — mirror `OuraPageFetching` pattern): cursor advances only after ack; ack failure keeps cursor; needsAttention surfaces in summary. + +- [ ] **Steps: read HealthKitBridge.swift first (record the chosen trigger + its dedup semantics in the report) → failing tests → implement → PASS coordinator tests + BOTH app builds (`Strand` macOS CODE_SIGNING_ALLOWED=NO; `NOOPiOS` device destination signed) → commit** `feat(cloudsync): coordinator + Data Sources card + Apple Health re-export hook` + +--- + +### Task 6 (controller, not subagent): flag, device deploy, live E2E + +- [ ] **Step 1:** Add `CLOUD_SYNC` to the untracked xcconfig (`Strand/Oura/OuraSecrets.xcconfig`): append ` CLOUD_SYNC` to its `SWIFT_ACTIVE_COMPILATION_CONDITIONS` line (create the line `SWIFT_ACTIVE_COMPILATION_CONDITIONS = $(inherited) OURA_CLOUD_IMPORT CLOUD_SYNC` if absent). Mirror documentation line in `OuraSecrets.example.xcconfig` (committed). +- [ ] **Step 2:** `xcodegen generate`; build + install NOOPiOS on the iPhone (established pipeline); launch. +- [ ] **Step 3:** On-device: Data Sources → Cloud Sync card → paste `https://vk-noop-cloud.fly.dev` + rw token (from noop-cloud/.env) → **Pull edits now** (server journal currently holds the smoke-test rows: one edit_sleep_stages + its undo → expect "applied 0 · skipped 2" — both rows are undone/undo). +- [ ] **Step 4:** Real E2E: via Claude Code (noop-cloud MCP): propose+confirm a small `set_baseline_note` AND an `adjust_sleep_bounds` against a REAL night from VK's data (after his real .noopbak upload) — or against fixture data if the mirror still holds fixtures (then bounds target the fixture night, which won't exist on the phone → exercises needsAttention honestly). Pull on phone; verify NOOP UI reflects the sleep change (if real data) and Apple Health shows the corrected night (Health app). Ack verified: `GET /edits` shows `ackedAt` set. +- [ ] **Step 5:** Ledger + memory update. + +--- + +## Self-Review + +**Spec coverage (design §5 Phase 3):** pull transport `GET /edits?since=` ✅ (T1 ack + existing endpoint; T3 client); apply rules — sleep via `userEdited=1` paths ✅ (T4, incl. the updateSleepStages promotion subtlety), corrections under `"noop-cloud"` namespace + registration ✅ (T4), DB-backed tombstones replacing the UserDefaults gap ✅ (T2), ack-back ✅ (T1/T5). Apple Health via merged write-back ✅ (T5). Fork-only gating ✅ (Global Constraints + T6). +**Placeholder scan:** T2 test notes initializer signatures must be read from source — that is an explicit instruction to verify, not a TBD; all other steps carry code or exact commands. T5 deliberately defers HealthKitBridge internals to a read-first step because the file is #249's merged code (unknown to this plan) — the task's Interface pins the REQUIRED behavior (public trigger, days window, error containment). +**Type consistency:** `CloudEdit` fields = server `JournalRow` + `ackedAt` (T1); `CloudApplySummary` consumed by T5; store APIs referenced (applySleepEdit/updateSleepStages/deleteWorkouts/upsertWorkouts/upsertMetricSeries/insert/cursor/setCursor) verified to exist by controller scout (grep) — signatures re-checked in-task. +**Known risk:** `updateSleepStages` stage-string vocabulary (app uses its stagesJSON `stage` strings — verify in-task that "awake|light|deep|rem" match the app's legend; if the app uses different tokens (e.g. "wake"), map in the applier and record it). diff --git a/docs/superpowers/plans/2026-07-12-sleep-night-stitch.md b/docs/superpowers/plans/2026-07-12-sleep-night-stitch.md new file mode 100644 index 0000000000..8fbd9aa3a2 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-sleep-night-stitch.md @@ -0,0 +1,575 @@ +# Sleep night-stitch (#364) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A night split by a short mid-night wake presents as ONE night — one `.inBed` span per bridged night in the Apple Health / Health Connect exports with the gap as a `wake` stage, and the gap rendered as a wake segment (and counted as awake minutes) in both sleep screens. + +**Architecture:** One new pure primitive in the analytics layer (`bridgedNightGroups` — the existing #561 bridge loop extracted so it returns EVERY bridged group plus each group's inter-fragment gaps), a pure merged-plan builder per write path (`HealthWriteback.mergedSleepPlan` in StrandImport; `HealthExportPlan.sleepSessions` grows grouping in Kotlin), and four thin consumers. Detector and stored sessions untouched; grouping is read-time. + +**Tech Stack:** Swift 5.9 packages (StrandAnalytics, StrandImport; `swift test` on macOS), app-target Swift (HealthKit, SwiftUI — compile via `xcodebuild`, no default CI), Kotlin/JVM (JUnit4 via `testFullDebugUnitTest` — no local toolchain, verified on fork Android CI), Health Connect SDK. + +## Global Constraints + +- Spec: `docs/superpowers/specs/2026-07-12-sleep-night-stitch-design.md`; maintainer direction on issue #364. +- Reuse `gapBridgeMaxMin` (60) + `nightTailBridgeMaxMin` (90) + `isOvernightOnset` — NO new threshold. +- The `mainNightGroupIndices` refactor must be byte-identical (the #561/#861 golden tests pin it). +- Cross-platform parity: `bridgedNightGroups` Swift↔Kotlin identical semantics; same fixtures + expected values in both test suites. +- HealthKit dedup: merged samples keyed by the group's EARLIEST fragment's immutable `startTs`; the delete predicate must carry EVERY fragment's key. Health Connect: same rule via `clientRecordId`, plus explicit `deleteRecords` for absorbed fragment ids (HC does not auto-remove). +- Spans use `effectiveStartTs` (edited onset); keys use immutable `startTs`. Kotlin `SleepInput` must gain that split (it currently conflates them). +- `Packages/` may not import AppKit/UIKit/HealthKit; StrandImport does NOT depend on StrandAnalytics (grouping happens in the caller; the plan builders take pre-grouped fragments / compute their own seams). +- No Claude attribution anywhere. Commit style: `feat(sleep): …` / `fix(sleep): …` lowercase. +- Work happens on branch `feat/sleep-night-stitch-364` off `upstream/main` in a worktree (VK's `main` is ~72 ahead of upstream; the PR must be clean). + +--- + +### Task 1: Swift analytics primitive — `bridgedNightGroups` (+ refactor `mainNightGroupIndices`) + +**Files:** +- Modify: `Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStageTotals.swift` (insert after `bridgeAdjacent`, ~line 306; refactor `mainNightGroupIndices` ~line 325) +- Test: `Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BridgedNightGroupsTests.swift` (create) + +**Interfaces:** +- Consumes: existing `NightBlock`, `gapBridgeMaxMin`, `nightTailBridgeMaxMin`, `isOvernightOnset`. +- Produces (later tasks rely on these exact names): + +```swift +public struct BridgedNightGroup: Equatable { + /// Indices into the ORIGINAL input array, ascending by effective onset. + public let indices: [Int] + /// Inter-fragment wake seams inside this group, ascending: (prev bridged end, next fragment start). + public let gaps: [GapSpan] + public struct GapSpan: Equatable { + public let start: Int, end: Int + public init(start: Int, end: Int) { self.start = start; self.end = end } + } + public init(indices: [Int], gaps: [GapSpan]) { self.indices = indices; self.gaps = gaps } +} + +/// EVERY bridged night group over `blocks` (not just the main night): the same two-tier bridge +/// `mainNightGroupIndices` applies (#561/#861), minus the winner pick. Groups ordered by start. +public static func bridgedNightGroups(_ blocks: [NightBlock], offsetSec: Int) -> [BridgedNightGroup] +``` + +- [ ] **Step 1: Write the failing tests** (`BridgedNightGroupsTests.swift`) + +```swift +import XCTest +@testable import StrandAnalytics + +/// Pins the all-groups bridge (#364): the SAME two-tier gap bridge the main-night selector applies, +/// returned for EVERY group so the Health write-back and the sleep screens can present each bridged +/// night as one night with its wake seams explicit. +final class BridgedNightGroupsTests: XCTestCase { + private typealias B = SleepStageTotals.NightBlock + // 2026-01-02 00:00 UTC for readable fixtures. + private let t0 = 1_767_312_000 + + func testShortWakeBridgesTwoFragmentsWithOneGap() { + // 23:00→02:00, 16-min wake, 02:16→06:00 — the #364 example shape. + let a = B(start: t0 - 3_600, end: t0 + 2 * 3_600) + let b = B(start: t0 + 2 * 3_600 + 16 * 60, end: t0 + 6 * 3_600) + let groups = SleepStageTotals.bridgedNightGroups([a, b], offsetSec: 0) + XCTAssertEqual(groups.count, 1) + XCTAssertEqual(groups[0].indices, [0, 1]) + XCTAssertEqual(groups[0].gaps, [.init(start: a.end, end: b.start)]) + } + + func testDaytimeNapStaysItsOwnGroup() { + let night = B(start: t0 - 3_600, end: t0 + 6 * 3_600) // 23:00→06:00 + let nap = B(start: t0 + 14 * 3_600, end: t0 + 15 * 3_600) // 14:00→15:00 + let groups = SleepStageTotals.bridgedNightGroups([night, nap], offsetSec: 0) + XCTAssertEqual(groups.map(\.indices), [[0], [1]]) + XCTAssertTrue(groups.allSatisfy { $0.gaps.isEmpty }) + } + + func testNightTailBridgesOvernightGapOverSixtyMinutes() { + // 75-min gap, second fragment onset 04:15 (overnight band) → bridged by the #861 night-tail rule. + let a = B(start: t0 - 3_600, end: t0 + 3 * 3_600) + let b = B(start: t0 + 3 * 3_600 + 75 * 60, end: t0 + 7 * 3_600) + let groups = SleepStageTotals.bridgedNightGroups([a, b], offsetSec: 0) + XCTAssertEqual(groups.map(\.indices), [[0, 1]]) + XCTAssertEqual(groups[0].gaps, [.init(start: a.end, end: b.start)]) + } + + func testThreeFragmentNightYieldsTwoGaps() { + let a = B(start: t0, end: t0 + 3_600) + let b = B(start: t0 + 3_600 + 600, end: t0 + 2 * 3_600) + let c = B(start: t0 + 2 * 3_600 + 900, end: t0 + 3 * 3_600) + let groups = SleepStageTotals.bridgedNightGroups([a, b, c], offsetSec: 0) + XCTAssertEqual(groups.map(\.indices), [[0, 1, 2]]) + XCTAssertEqual(groups[0].gaps, [.init(start: a.end, end: b.start), + .init(start: b.end, end: c.start)]) + } + + func testContainedFragmentProducesNoGap() { + // b sits inside a (gap < 0): bridged, no seam emitted. + let a = B(start: t0, end: t0 + 4 * 3_600) + let b = B(start: t0 + 3_600, end: t0 + 2 * 3_600) + let groups = SleepStageTotals.bridgedNightGroups([a, b], offsetSec: 0) + XCTAssertEqual(groups.map(\.indices), [[0, 1]]) + XCTAssertTrue(groups[0].gaps.isEmpty) + } + + func testUnsortedInputIndicesReferOriginalPositions() { + let late = B(start: t0 + 2 * 3_600 + 16 * 60, end: t0 + 6 * 3_600) + let early = B(start: t0 - 3_600, end: t0 + 2 * 3_600) + let groups = SleepStageTotals.bridgedNightGroups([late, early], offsetSec: 0) + XCTAssertEqual(groups.map(\.indices), [[0, 1]]) // ascending original indices [1(early)…] sorted → [0,1]? NO: + // indices are into the ORIGINAL array: early is index 1, late is index 0 → group indices sorted ascending [0, 1]. + XCTAssertEqual(groups[0].gaps, [.init(start: early.end, end: late.start)]) + } + + func testEmptyInputYieldsNoGroups() { + XCTAssertTrue(SleepStageTotals.bridgedNightGroups([], offsetSec: 0).isEmpty) + } + + /// The refactor guard: the main-night pick over the new all-groups pass must stay byte-identical + /// to the existing golden behaviour (the #561 suite pins the rest). + func testMainNightGroupIndicesUnchangedForBiphasicPlusNap() { + let a = B(start: t0 - 3_600, end: t0 + 2 * 3_600) + let b = B(start: t0 + 2 * 3_600 + 16 * 60, end: t0 + 6 * 3_600) + let nap = B(start: t0 + 14 * 3_600, end: t0 + 15 * 3_600) + XCTAssertEqual(SleepStageTotals.mainNightGroupIndices([a, b, nap], offsetSec: 0), [0, 1]) + } +} +``` + +- [ ] **Step 2: Run to verify they fail** — `swift test --package-path Packages/StrandAnalytics --filter BridgedNightGroupsTests` → FAIL: `bridgedNightGroups` unresolved. + +- [ ] **Step 3: Implement** — insert `BridgedNightGroup` + `bridgedNightGroups` after `bridgeAdjacent` (~line 306). Extract the existing loop from `mainNightGroupIndices` verbatim, adding gap capture: + +```swift + /// One bridged night group over the whole input: the fragments (as ORIGINAL indices) plus the + /// inter-fragment wake seams. Produced by `bridgedNightGroups` for the write-back / display + /// consumers that must present a briefly-interrupted night as ONE night (#364). + public struct BridgedNightGroup: Equatable { + public let indices: [Int] + public let gaps: [GapSpan] + public struct GapSpan: Equatable { + public let start: Int, end: Int + public init(start: Int, end: Int) { self.start = start; self.end = end } + } + public init(indices: [Int], gaps: [GapSpan]) { self.indices = indices; self.gaps = gaps } + } + + /// EVERY bridged group over `blocks` — the same two-tier bridge `mainNightGroupIndices` applies + /// (#561 short-wake + #861 overnight night-tail), WITHOUT the winner pick, so the Health + /// write-back and the sleep screens group ALL nights exactly as the day totals do. Each group's + /// `gaps` are the seams between consecutive bridged fragments (empty when fragments touch or + /// overlap) — folded in as `wake` by the consumers. Groups ordered by start; indices are into + /// the ORIGINAL array, ascending. Pure + deterministic; Kotlin twin `bridgedNightGroups`. (#364) + public static func bridgedNightGroups(_ blocks: [NightBlock], offsetSec: Int) -> [BridgedNightGroup] { + guard !blocks.isEmpty else { return [] } + let order = blocks.indices.sorted { blocks[$0].start < blocks[$1].start } + let bridgeS = gapBridgeMaxMin * 60 + let nightTailBridgeS = nightTailBridgeMaxMin * 60 + var bridged: [NightBlock] = [] + var groups: [[Int]] = [] + var gaps: [[BridgedNightGroup.GapSpan]] = [] + for idx in order { + let b = blocks[idx] + if let last = bridged.last { + let gap = b.start - last.end + let bridges = gap >= 0 + && (gap < bridgeS + || (gap < nightTailBridgeS && isOvernightOnset(b.start, offsetSec: offsetSec))) + if bridges { + if gap > 0 { + gaps[gaps.count - 1].append(.init(start: last.end, end: b.start)) + } + bridged[bridged.count - 1] = NightBlock(start: last.start, end: max(last.end, b.end)) + groups[groups.count - 1].append(idx) + continue + } + // A CONTAINED fragment (negative gap) still belongs to the night it sits inside. + if gap < 0 { + bridged[bridged.count - 1] = NightBlock(start: last.start, end: max(last.end, b.end)) + groups[groups.count - 1].append(idx) + continue + } + } + bridged.append(b) + groups.append([idx]) + gaps.append([]) + } + return zip(groups, gaps).map { BridgedNightGroup(indices: $0.sorted(), gaps: $1) } + } +``` + + **CAREFUL — byte-identity check before refactoring `mainNightGroupIndices`:** the existing loop treats `gap < 0` via `gap >= 0 &&` → a contained fragment does NOT bridge there and starts its own bridged span. Verify against the #561 golden tests what the current behaviour is for overlapping fragments; if the existing suite pins "negative gap → separate group", DROP the `gap < 0` containment branch above and emit a `GapSpan` only when `gap > 0` (matching `testContainedFragmentProducesNoGap` would then need `[[0],[1]]` — adjust that one test to pin the ACTUAL legacy semantics). The refactored `mainNightGroupIndices` becomes: + +```swift + public static func mainNightGroupIndices(_ blocks: [NightBlock], offsetSec: Int, + habitualMidsleepSec: Int? = nil) -> [Int]? { + guard !blocks.isEmpty else { return nil } + let all = bridgedNightGroups(blocks, offsetSec: offsetSec) + let bridgedSpans = all.map { g -> NightBlock in + let s = g.indices.map { blocks[$0].start }.min() ?? 0 + let e = g.indices.map { blocks[$0].end }.max() ?? 0 + return NightBlock(start: s, end: e) + } + guard let winner = mainNightIndex(bridgedSpans, offsetSec: offsetSec, + habitualMidsleepSec: habitualMidsleepSec) else { return nil } + return all[winner].indices + } +``` + + Only perform this refactor if the FULL StrandAnalytics suite stays green; otherwise keep the original body and have `bridgedNightGroups` be a sibling (duplicated loop, one comment cross-referencing) — byte-identity of the pick outranks DRY here. + +- [ ] **Step 4: Run** — `swift test --package-path Packages/StrandAnalytics` → ALL green (new + #561/#861 goldens). +- [ ] **Step 5: Commit** — `feat(sleep): bridgedNightGroups — the #561 bridge over EVERY night, with wake seams (#364)` + +--- + +### Task 2: Kotlin analytics twin — `bridgedNightGroups` + +**Files:** +- Modify: `android/app/src/main/java/com/noop/analytics/SleepStageTotals.kt` (after `bridgeAdjacent`, ~line 246) +- Test: `android/app/src/test/java/com/noop/analytics/SleepStageTotalsTest.kt` (append; file exists) + +**Interfaces (produces):** + +```kotlin +data class BridgedNightGroup(val indices: List, val gaps: List>) +fun bridgedNightGroups(blocks: List, offsetSec: Long): List +``` + +- [ ] **Step 1: Transcribe the implementation** — same loop as Swift (Long timestamps), same doc comment marking it the Swift twin, same containment decision as resolved in Task 1 Step 3. Refactor Kotlin `mainNightGroupIndices` the same way ONLY if the Kotlin golden tests are also confirmed green on CI at the end; otherwise sibling. +- [ ] **Step 2: Transcribe the tests** — same seven cases, same fixture timestamps (`1_767_312_000L`), same expected indices/gaps, JUnit4 `assertEquals`. Local red/green is impossible (no JVM toolchain) — parity with the Swift twin (which DID fail-then-pass) is the evidence; fork Android CI at Task 8 is the executable check. Note this deviation in the PR. +- [ ] **Step 3: Commit** — `feat(sleep): kotlin bridgedNightGroups twin (#364)` + +--- + +### Task 3: StrandImport — `HealthWriteback.mergedSleepPlan` + +**Files:** +- Modify: `Packages/StrandImport/Sources/StrandImport/HealthWriteback.swift` +- Test: `Packages/StrandImport/Tests/StrandImportTests/HealthWritebackTests.swift` (exists as the stageIntervals suite — append; if the file is named differently, `grep -rln stageIntervals Packages/StrandImport/Tests`) + +**Interfaces:** +- Consumes: existing `stageIntervals`, `StageInterval`, `StageKind`. +- Produces: + +```swift +public enum HealthWriteback { + // StageKind gains: case unspecified (bridge maps it to .asleepUnspecified) + + public struct SleepFragment: Equatable { + public let startTs: Int // immutable detected key + public let effectiveStartTs: Int // edited onset when present + public let endTs: Int + public let stagesJSON: String? + public init(startTs: Int, effectiveStartTs: Int, endTs: Int, stagesJSON: String?) + } + public struct MergedSleepEntry: Equatable { + public let keyStartTs: Int // group representative = earliest fragment's startTs + public let spanStart: Int // earliest effectiveStartTs + public let spanEnd: Int // latest endTs + public let intervals: [StageInterval] // fragments' stages (unspecified when undecodable) + wake seams + public let allKeyStartTs: [Int] // every fragment's startTs — the delete-key set + } + /// `groups`: fragments already grouped by `SleepStageTotals.bridgedNightGroups` (each inner + /// array ascending by effectiveStartTs). + public static func mergedSleepPlan(groups: [[SleepFragment]]) -> [MergedSleepEntry] +} +``` + +- [ ] **Step 1: Write the failing tests** (append to the writeback suite) + +```swift + // MARK: - mergedSleepPlan (#364 night-stitch) + + private func frag(_ start: Int, _ end: Int, stages: String? = nil, eff: Int? = nil) -> HealthWriteback.SleepFragment { + .init(startTs: start, effectiveStartTs: eff ?? start, endTs: end, stagesJSON: stages) + } + private func stagesJSON(_ segs: [(Int, Int, String)]) -> String { + "[" + segs.map { "{\"start\":\($0.0),\"end\":\($0.1),\"stage\":\"\($0.2)\"}" }.joined(separator: ",") + "]" + } + + func testMergedPlanFoldsTwoFragmentsWithWakeSeam() { + let t = 1_767_312_000 + let a = frag(t, t + 7_200, stages: stagesJSON([(t, t + 7_200, "light")])) + let b = frag(t + 7_200 + 960, t + 14_400, stages: stagesJSON([(t + 7_200 + 960, t + 14_400, "deep")])) + let plan = HealthWriteback.mergedSleepPlan(groups: [[a, b]]) + XCTAssertEqual(plan.count, 1) + let e = plan[0] + XCTAssertEqual(e.keyStartTs, t) + XCTAssertEqual(e.spanStart, t) + XCTAssertEqual(e.spanEnd, t + 14_400) + XCTAssertEqual(e.allKeyStartTs, [t, t + 7_200 + 960]) + // The seam sits EXACTLY on [prev.end, next.effectiveStart] as .awake. + XCTAssertTrue(e.intervals.contains(.init(start: t + 7_200, end: t + 7_200 + 960, kind: .awake))) + XCTAssertEqual(e.intervals.first?.kind, .light) + XCTAssertEqual(e.intervals.last?.kind, .deep) + } + + func testMergedPlanSingleFragmentMatchesLegacyShape() { + let t = 1_767_312_000 + let a = frag(t, t + 7_200, stages: stagesJSON([(t, t + 3_600, "light"), (t + 3_600, t + 7_200, "rem")])) + let e = HealthWriteback.mergedSleepPlan(groups: [[a]])[0] + XCTAssertEqual(e.keyStartTs, t) + XCTAssertEqual(e.allKeyStartTs, [t]) + XCTAssertEqual(e.intervals, HealthWriteback.stageIntervals(stagesJSON: a.stagesJSON, + sessionStart: t, sessionEnd: t + 7_200)) + } + + func testMergedPlanUnstagedFragmentsDegradeToUnspecifiedPlusSeam() { + let t = 1_767_312_000 + let a = frag(t, t + 7_200) // no stagesJSON + let b = frag(t + 7_200 + 960, t + 14_400) // no stagesJSON + let e = HealthWriteback.mergedSleepPlan(groups: [[a, b]])[0] + XCTAssertEqual(e.intervals, [ + .init(start: t, end: t + 7_200, kind: .unspecified), + .init(start: t + 7_200, end: t + 7_200 + 960, kind: .awake), + .init(start: t + 7_200 + 960, end: t + 14_400, kind: .unspecified), + ]) + } + + func testMergedPlanEditedOnsetMovesSpanButNotKey() { + let t = 1_767_312_000 + let a = frag(t, t + 7_200, eff: t + 600) // user moved bedtime 10 min later + let e = HealthWriteback.mergedSleepPlan(groups: [[a]])[0] + XCTAssertEqual(e.keyStartTs, t) // immutable key + XCTAssertEqual(e.spanStart, t + 600) // edited onset drives the span + } +``` + +- [ ] **Step 2: Run to verify failure** — `swift test --package-path Packages/StrandImport --filter HealthWriteback` → FAIL (`SleepFragment` unresolved). +- [ ] **Step 3: Implement** — add `case unspecified` to `StageKind`; add the two structs; implement: + +```swift + /// Fold one bridged night group (#364) into a single write-back entry: the span, the merged + /// stage timeline (each fragment's decoded stages, or one honest `.unspecified` block for a + /// fragment with no timing, with every inter-fragment seam as `.awake`), the representative + /// immutable key, and the COMPLETE per-fragment key set so a night previously exported as two + /// entries deletes both before the merged write. + public static func mergedSleepPlan(groups: [[SleepFragment]]) -> [MergedSleepEntry] { + var out: [MergedSleepEntry] = [] + for group in groups where !group.isEmpty { + let frags = group.sorted { $0.effectiveStartTs < $1.effectiveStartTs } + var intervals: [StageInterval] = [] + var prevEnd: Int? = nil + for f in frags { + guard f.endTs > f.effectiveStartTs else { continue } + if let p = prevEnd, f.effectiveStartTs > p { + intervals.append(StageInterval(start: p, end: f.effectiveStartTs, kind: .awake)) + } + let stages = stageIntervals(stagesJSON: f.stagesJSON, + sessionStart: f.effectiveStartTs, sessionEnd: f.endTs) + if stages.isEmpty { + intervals.append(StageInterval(start: f.effectiveStartTs, end: f.endTs, kind: .unspecified)) + } else { + intervals.append(contentsOf: stages) + } + prevEnd = max(prevEnd ?? f.endTs, f.endTs) + } + guard let first = frags.first, let spanEnd = frags.map(\.endTs).max(), spanEnd > first.effectiveStartTs + else { continue } + out.append(MergedSleepEntry( + keyStartTs: frags.map(\.startTs).min() ?? first.startTs, + spanStart: first.effectiveStartTs, + spanEnd: spanEnd, + intervals: intervals, + allKeyStartTs: frags.map(\.startTs).sorted())) + } + return out + } +``` + +- [ ] **Step 4: Run** — full `swift test --package-path Packages/StrandImport` green (existing stageIntervals suite must not regress from the `StageKind` addition — it's additive). +- [ ] **Step 5: Commit** — `feat(sleep): mergedSleepPlan — one write-back entry per bridged night, seams as wake (#364)` + +--- + +### Task 4: iOS consumer — `HealthKitBridge.writeSleep` groups by bridged night + +**Files:** +- Modify: `StrandiOS/Health/HealthKitBridge.swift:600-655` (`writeSleep`) + +**Interfaces:** consumes `SleepStageTotals.bridgedNightGroups` + `HealthWriteback.mergedSleepPlan`. Check the file's imports; add `import StrandAnalytics` if missing (the app target links it — SleepView already imports it). + +- [ ] **Step 1: Rewrite `writeSleep`** (doc comment updated to describe the one-entry-per-night shape and the #364 rationale): + +```swift + private func writeSleep(sessions: [CachedSleepSession]) async throws { + guard let type = HKObjectType.categoryType(forIdentifier: .sleepAnalysis), + store.authorizationStatus(for: type) == .sharingAuthorized else { return } + // Group fragments into bridged nights (#364): a night split by a brief mid-night wake exports + // as ONE .inBed span with the gap as an in-hypnogram wake segment — matching what the daily + // totals already score (#561) and what Oura / Apple Watch write. Naps never bridge. + let blocks = sessions.map { SleepStageTotals.NightBlock(start: $0.effectiveStartTs, end: $0.endTs) } + let offsetSec = TimeZone.current.secondsFromGMT() + let groups = SleepStageTotals.bridgedNightGroups(blocks, offsetSec: offsetSec).map { g in + g.indices.map { i -> HealthWriteback.SleepFragment in + let s = sessions[i] + return .init(startTs: s.startTs, effectiveStartTs: s.effectiveStartTs, + endTs: s.endTs, stagesJSON: s.stagesJSON) + } + } + var samples: [HKCategorySample] = [] + var keys: [String] = [] + for entry in HealthWriteback.mergedSleepPlan(groups: groups) { + let key = "noop:\(noopDeviceId):sleep:\(entry.keyStartTs)" + let meta = [HKMetadataKeyExternalUUID: key] + // Delete keys cover EVERY fragment, so a night previously written as two entries fully + // clears when it becomes one (the absorbed fragment's old key would otherwise orphan). + keys.append(contentsOf: entry.allKeyStartTs.map { "noop:\(noopDeviceId):sleep:\($0)" }) + samples.append(HKCategorySample(type: type, value: HKCategoryValueSleepAnalysis.inBed.rawValue, + start: Date(timeIntervalSince1970: TimeInterval(entry.spanStart)), + end: Date(timeIntervalSince1970: TimeInterval(entry.spanEnd)), + metadata: meta)) + for seg in entry.intervals { + let value: HKCategoryValueSleepAnalysis + switch seg.kind { + case .awake: value = .awake + case .light: value = .asleepCore + case .deep: value = .asleepDeep + case .rem: value = .asleepREM + case .unspecified: value = .asleepUnspecified + } + samples.append(HKCategorySample( + type: type, value: value.rawValue, + start: Date(timeIntervalSince1970: TimeInterval(seg.start)), + end: Date(timeIntervalSince1970: TimeInterval(seg.end)), + metadata: meta)) + } + } + guard !samples.isEmpty else { return } + let pred = NSCompoundPredicate(andPredicateWithSubpredicates: [ + HKQuery.predicateForObjects(from: HKSource.default()), + HKQuery.predicateForObjects(withMetadataKey: HKMetadataKeyExternalUUID, allowedValues: keys), + ]) + _ = try? await store.deleteObjects(of: type, predicate: pred) + try await store.save(samples) + } +``` + +- [ ] **Step 2: Compile** — `xcodegen generate && xcodebuild -project Strand.xcodeproj -scheme NOOPiOS -destination 'generic/platform=iOS' CODE_SIGNING_ALLOWED=NO build` (HealthKitBridge is iOS-only). Expected: BUILD SUCCEEDED. +- [ ] **Step 3: Commit** — `fix(health): export a wake-split night as ONE Apple Health entry with the gap as awake (#364)` + +--- + +### Task 5: macOS/iOS display — gap-wake segments + awake minutes in `SleepView.mergeDay` + +**Files:** +- Modify: `Strand/Screens/SleepView.swift:1981-2019` (`mergeDay`) + +- [ ] **Step 1: Splice the seams** — inside `mergeDay`, after the per-fragment loop and before `guard stages.asleep > 0`: + +```swift + // #364: the inter-fragment wake seams belong to the night — draw them as wake segments and + // count them as awake minutes, so the hero hypnogram has no hole and the card's minutes match + // the day totals (which already add the gap via interFragmentAwakeSeconds, #777/#705). + let ordered = Array(group) + for (prev, next) in zip(ordered, ordered.dropFirst()) { + let gapStart = prev.endTs, gapEnd = next.effectiveStartTs + guard gapEnd > gapStart else { continue } + stages.awake += Double(gapEnd - gapStart) / 60.0 + segs.append(SleepInterval(stage: .awake, start: TimeInterval(gapStart), end: TimeInterval(gapEnd))) + } +``` + + **Pin the exact `SleepInterval` init at implementation time** (`grep -n "struct SleepInterval" Strand/`): match its stage enum case (`.awake` vs `.wake`) and start/end types (TimeInterval vs Int) to the real definition — the snippet above assumes `TimeInterval`; adjust to whatever `decodeSegments` appends. `ordered` must use the SAME `group` (post-stub-drop) the fragment loop used. Gap uses `prev.endTs` (not running max) because group fragments are ascending and non-overlapping in practice; if Task 1 resolved containment as "bridged", guard with `guard gapEnd > gapStart` (already present) so contained fragments emit nothing. +- [ ] **Step 2: Check the in-bed subtitle** — find `stageCard`'s subtitle source (~line 635). If it renders `night.stages.total`, the gap now counts (correct — matches engine). If it renders the onset→wake span, it already included the gap. Either way NO further edit; just confirm which and note it in the PR body. +- [ ] **Step 3: Compile BOTH apps** — `xcodebuild -scheme Strand -destination 'platform=macOS' CODE_SIGNING_ALLOWED=NO build` AND the NOOPiOS build from Task 4 Step 2 (SleepView is shared). +- [ ] **Step 4: Commit** — `feat(sleep): draw the night's wake seams in the hero hypnogram + count them as awake (#364)` + +--- + +### Task 6: Android write path — grouping in `HealthExportPlan` + absorbed-id deletion in `HealthConnectWriter` + +**Files:** +- Modify: `android/app/src/main/java/com/noop/ingest/HealthExportPlan.kt` (sleep section, lines 77-128) +- Modify: `android/app/src/main/java/com/noop/ingest/HealthConnectWriter.kt` (`writeSleep`, ~lines 199-227) +- Test: `android/app/src/test/java/com/noop/ingest/HealthExportPlanTest.kt` (append) + +**Interfaces:** + +```kotlin +// SleepInput gains the immutable key / effective-onset split (iOS parity): +data class SleepInput(val keyStartTs: Long, val startTs: Long, val endTs: Long, val stagesJSON: String?) +data class SleepPlan( + val clientId: String, // "noop-sleep-" + val startSec: Long, val endSec: Long, + val stages: List, // fragments' AWAKE/SLEEPING plans + seams as StagePlan(asleep=false) + val absorbedClientIds: List, // non-representative fragments' old ids → writer deletes +) +fun sleepSessions(sessions: List, nowSec: Long, offsetSec: Long): List +``` + +- [ ] **Step 1: Write the JVM tests** (append to `HealthExportPlanTest.kt`; same fixture numbers as Task 3): + +```kotlin + @Test + fun sleepPlanFoldsBridgedFragmentsWithAwakeSeam() { + val t = 1_767_312_000L + val a = HealthExportPlan.SleepInput(t, t, t + 7_200, + """[{"start":$t,"end":${t + 7_200},"stage":"light"}]""") + val b = HealthExportPlan.SleepInput(t + 8_160, t + 8_160, t + 14_400, + """[{"start":${t + 8_160},"end":${t + 14_400},"stage":"deep"}]""") + val plans = HealthExportPlan.sleepSessions(listOf(a, b), nowSec = t + 86_400, offsetSec = 0) + assertEquals(1, plans.size) + val p = plans[0] + assertEquals("noop-sleep-$t", p.clientId) + assertEquals(t, p.startSec); assertEquals(t + 14_400, p.endSec) + assertEquals(listOf("noop-sleep-${t + 8_160}"), p.absorbedClientIds) + assertTrue(p.stages.contains(HealthExportPlan.StagePlan(t + 7_200, t + 8_160, asleep = false))) + } + + @Test + fun napStaysItsOwnPlanWithNoAbsorbedIds() { + val t = 1_767_312_000L + val night = HealthExportPlan.SleepInput(t, t, t + 21_600, null) + val nap = HealthExportPlan.SleepInput(t + 50_400, t + 50_400, t + 54_000, null) + val plans = HealthExportPlan.sleepSessions(listOf(night, nap), nowSec = t + 86_400, offsetSec = 0) + assertEquals(2, plans.size) + assertTrue(plans.all { it.absorbedClientIds.isEmpty() }) + } + + @Test + fun editedOnsetMovesSpanButClientIdStaysImmutable() { + val t = 1_767_312_000L + val edited = HealthExportPlan.SleepInput(t, t + 600, t + 7_200, null) + val p = HealthExportPlan.sleepSessions(listOf(edited), nowSec = t + 86_400, offsetSec = 0)[0] + assertEquals("noop-sleep-$t", p.clientId) + assertEquals(t + 600, p.startSec) + } +``` + +- [ ] **Step 2: Implement** — group via `com.noop.analytics.SleepStageTotals.bridgedNightGroups` (map inputs to `NightBlock(startTs, endTs)` — effective onset), fold each group: seams → `StagePlan(gapStart, gapEnd, asleep = false)`; unstaged fragments contribute no stages (HC convention unchanged); clientId from the earliest fragment's `keyStartTs`; absorbedClientIds = the rest. Update `writeSleep` call site: map `SleepInput(it.startTs, it.effectiveStartTs ?: it.startTs, it.endTs, it.stagesJSON)` — pin the actual Room entity's adjusted-onset field name (`grep -n "effectiveStart\|startTsAdjusted" android/app/src/main/java/com/noop/data/` at implementation). After `insertChunked`, delete absorbed ids: + +```kotlin + val absorbed = plans.flatMap { it.absorbedClientIds } + if (absorbed.isNotEmpty()) { + client.deleteRecords(SleepSessionRecord::class, recordIdsList = emptyList(), clientRecordIdsList = absorbed) + } +``` + +- [ ] **Step 3: Commit** — `fix(health): android — one Health Connect session per bridged night, absorbed fragments deleted (#364)` + +--- + +### Task 7: Android display — seams in `selectNight` + +**Files:** +- Modify: `android/app/src/main/java/com/noop/ui/SleepScreen.kt:2827-2851` (`groupSegments` / `sumGroupStages` / `groupInBedMin`) + +- [ ] **Step 1: Splice seams into `groupSegments`** — after the flatMap, insert a `PersistedSegment(stage = "wake", start = prev.endTs, end = next.effectiveStartTs)` between consecutive `heroGroup` fragments (guard `end > start`; pin `PersistedSegment`'s exact constructor by grepping its definition). Add the same seam minutes to `sumGroupStages`'s awake figure and to `groupInBedMin` (gaps now count — matching the engine's `interFragmentAwakeSeconds` day totals and iOS's span-based in-bed). +- [ ] **Step 2: Commit** — `feat(sleep): android — wake seams in the hero hypnogram + gap-inclusive minutes (#364)` + +--- + +### Task 8: Verification sweep + +- [ ] `swift test` — StrandAnalytics, StrandImport (and StrandDesign/WhoopStore if touched — they shouldn't be). +- [ ] `xcodegen generate && xcodebuild` — `Strand` (macOS) AND `NOOPiOS` (iOS), both `CODE_SIGNING_ALLOWED=NO build`. +- [ ] Push branch to fork; dispatch `android.yml` (assemble + JVM tests — covers Tasks 2/6 red-free) and `app-build.yml` (both app targets on clean `project.yml`). +- [ ] `swift-packages.yml` runs automatically on the PR. + +### Task 9: PR + +- [ ] One PR to `ryanbr/noop` from `vishk23:feat/sleep-night-stitch-364`, title `fix(sleep): present a wake-split night as one night — export + hypnogram, both platforms (#364)`. Body: Ryan's recommended shape restated (display-time via the existing bridge, `gapBridgeMaxMin` reused, one `.inBed`/`SleepSessionRecord` per bridged night, seams as wake), the dedup 2→1 rule, the in-bed/awake minutes now matching the engine's #777/#705 totals, Proposal 2 deferred, verification section (swift tests, both app builds, fork CI links). No attribution. + +### Task 10: Sync local main + +- [ ] `git merge feat/sleep-night-stitch-364` into VK's `main` (keeps his device builds current); run `swift test` on the two packages post-merge. diff --git a/docs/superpowers/specs/2026-07-10-noop-cloud-mcp-design.md b/docs/superpowers/specs/2026-07-10-noop-cloud-mcp-design.md new file mode 100644 index 0000000000..2d58061df1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-noop-cloud-mcp-design.md @@ -0,0 +1,113 @@ +# noop-cloud — personal biometrics MCP server (Fly.io) — design + +- **Date:** 2026-07-10 +- **Status:** Draft for review +- **Scope:** A new, standalone, self-hostable server project (sibling repo `noop-cloud/`, its own git) + minimal opt-in app-side hooks in this fork. Phased: read mirror → gated writes → sync-back. +- **Relation to upstream:** `CLAUDE.md` marks "no server / no cloud sync" a hard upstream constraint — this is deliberately a **personal-fork + sidecar** project, never a PR to `ryanbr/noop`. The server ships as an agnostic container anyone can self-host; nothing phones home by default. + +--- + +## 1. Summary + +The user's NOOP data (WHOOP live + Oura cloud import + Apple Health) lives in the iPhone app's SQLite. He wants an **agentic layer over it**: ask Claude/ChatGPT deep questions, get a plain-English **morning report**, have AI **find and fix mislogged data** (dedupe workouts, correct sleep bounds), **corroborate sources** (WHOOP vs Oura vs Apple baselines), and let his **other agents** read/write workout context — all without a Mac dependency. + +`noop-cloud` is a small remote **MCP server on Fly.io** holding a **mirror** of the phone's database. The phone ships its existing `.noopbak` backup to the server (Phase 1: an iOS Shortcut posts it — zero app code; Phase 1.5: an in-app "Sync to my server" button). Claude (web/iOS/Code/API), ChatGPT, and scheduled agents connect over Streamable HTTP. Writes are **staged server-side** (`propose_edit` → `confirm_edit`) because scheduled agents skip client approval UIs. Sync-back to the phone is Phase 3, under a dedicated source namespace so the app's recompute can never clobber (or be clobbered by) server edits. + +## 2. Goals / non-goals + +**Goals** +- **G1 — Mirror + query.** The full phone DB queryable via MCP tools from Claude + ChatGPT + agents, no Mac. +- **G2 — Morning report.** A scheduled agent produces a plain-English daily briefing from the mirror. +- **G3 — Gated cleanup writes.** AI proposes fixes (workouts, sleep bounds, bad points); nothing mutates without explicit confirmation; every mutation journaled + reversible. +- **G4 — Cross-source corroboration.** First-class tools to compare WHOOP/Oura/Apple per-day and surface discrepancies/baselines. +- **G5 — Self-hostable.** One container + one volume; secrets via env; no third-party data processors. Shareable with the community as "run your own." +- **G6 — Sync-back seam (Phase 3).** Server-side corrections eventually visible in the phone app without corruption. + +**Non-goals** +- No multi-user/hosted service. No upstream PR. No replacement of NOOP's scoring. No live BLE anything (phone stays the collector). No LiteFS/replication (single writer by design). + +## 3. Verified constraints the design is built on (2026-07-10) + +**MCP / clients** +- Current spec **2025-11-25**: remote transport = **Streamable HTTP** (SSE transport deprecated). A **stateless 2026-07-28 revision** is locked (removes `Mcp-Session-Id`/initialize); official guidance: existing servers keep working. → Build on **TypeScript SDK 1.29.x pinned** (Tier-1, ships `StreamableHTTPServerTransport` + OAuth plumbing); migrate post-7/28 deliberately. +- **Auth matrix:** ChatGPT developer-mode connectors = **OAuth 2.1 + PKCE (CIMD/DCR) only — static keys rejected**. claude.ai web custom connectors (individual) = OAuth or no-auth in practice. **Claude Code** = static bearer via `--header` works today; also auto-OAuth. **Messages API MCP connector** (beta `mcp-client-2025-11-20`) = any `https://` URL + static `authorization_token`, per-tool allowlists, tools-only. +- **Claude Code Routines** (cloud, cron ≥1h) include the user's connectors and **execute write tools without permission prompts** → server-side write gating is mandatory, client approval UI is not a defense. +- ChatGPT Deep Research requires exactly-shaped **`search`/`fetch`** tools; general connectors don't. + +**Fly.io** +- One `shared-cpu-1x` machine + one volume (`/data`), SQLite **WAL**, `auto_stop_machines="stop"` + `auto_start_machines=true` (wake on request). Cost ≈ $0.20–$2.20/mo. Native scheduled machines lack time-of-day precision → morning cron runs **outside** (GitHub Actions cron or a Claude Routine) and doubles as the wake-up. **No LiteFS** (unmaintained ~15 mo; Fly warns against autostop combos). + +**App-side facts (from code study)** +- **`.noopbak`** = ZIP{`noop-backup.sqlite` (+optional `settings.json`)}; export **checkpoints WAL** + `PRAGMA quick_check` first; restore is a **whole-DB destructive swap** with origin/integrity gates and rollback; fully available on iOS (share sheet / Files). +- **BackupSync** = one-way timestamped snapshots into a user folder (iCloud Drive capable) + on-launch daily catch-up. No merge logic exists anywhere. +- **No change tracking**: no `updatedAt` columns; the v5 `synced` flags are documented dead scaffolding from a removed upload feature (which failed once on ts-highwater ordering — lesson recorded). Delta sync = new columns; whole-DB ship is the honest Phase-1 transport. +- **Edit provenance is thin**: only `sleepSession.userEdited` (+`startTsAdjusted`) survives recompute, and only for bounds/stages — `efficiency/restingHr/avgHrv` are overwritten regardless. `dailyMetric`/`workout`/`metricSeries` have **zero** protection and `analyzeRecent` blindly re-upserts the trailing ~21 days every ~15 foreground minutes. **Sleep-delete tombstones live in UserDefaults, not the DB** — backups don't carry deletion state. +- Multi-source reconciliation already exists (per-source `deviceId` namespaces + `Repository.resolvedSeries`/`dayOwnership`) — the correct integration point for server-origin data (mirror the `"oura-api"` pattern; never mutate another source's rows in place). + +## 4. Architecture + +``` +iPhone (collector) Fly.io app "noop-cloud" (1 machine, 1 volume) +┌───────────────────────┐ HTTPS ┌──────────────────────────────────────────────┐ +│ NOOP app │ POST │ /ingest ← bearer(rw). .noopbak → verify → │ +│ ├ BackupSync writes │ .noopbak │ unzip → quick_check → atomic swap │ +│ │ .noopbak snapshot ├─────────────▶│ /data/mirror.sqlite (RO for tools) │ +│ └ (P1: iOS Shortcut │ │ /mcp ← Streamable HTTP MCP │ +│ posts it; P1.5: │ │ read tools · corroborate · search/fetch │ +│ in-app button) │ │ write tools → staging (propose/confirm) │ +└───────────▲───────────┘ │ /data/server.sqlite: edits journal, staging, │ + │ Phase 3: pull server │ tokens, ingest log │ + │ edits (userEdited rules) │ auth: OAuth2.1 shim (PKCE+CIMD/DCR) + 2 static │ + └──────────────────────────│ bearer tokens (ro / rw) │ + └──────▲───────────────▲───────────────────────┘ + Claude web/iOS/Code/API │ │ ChatGPT (OAuth) + Routines/cron (ro token)┘ │ other agents (MCP) +``` + +Two SQLite files on the volume: **`mirror.sqlite`** (verbatim phone DB, replaced atomically per ingest, opened read-only by tools) and **`server.sqlite`** (server-owned: staged edits, confirmed-edit journal, ingest history, token hashes). Server edits are **never** written into the mirror — the journal is the source of truth for Phase-3 sync-back, and survives every mirror swap. + +## 5. Phases + +**Phase 1 — Mirror + read tools + morning report (MVP; no app code changes)** +- Fly app (TS SDK 1.29.x, Express, better-sqlite3): `/ingest` + `/mcp` + `/healthz`. +- Upload path: the app's existing **BackupSync** snapshot + an **iOS Shortcut** ("Send NOOP to cloud": share/automation → POST file with `Authorization: Bearer `), so Phase 1 requires zero app-side code. Manual cadence is fine (data freshness tool reports staleness honestly). +- Read tools (schema from the repo study; multi-source aware): `health_snapshot`, `metric_series` (any `deviceId` source incl. `oura-api`), `sleep_summary`, `workout_summary`, `data_freshness` (incl. mirror age), **`compare_sources`** (per-day WHOOP/Oura/Apple side-by-side with deltas — G4's workhorse), plus ChatGPT-shaped `search`/`fetch`. Prompts: `morning_report`, `corroborate_sources`, `find_messy_data`. +- Auth v1: two static bearer tokens (ro/rw) → Claude Code + Messages API + cron work day 1. +- Morning report: **GitHub Actions cron** (free, exact-time) → Messages API with the MCP connector (ro token, tool-allowlisted) → delivers via ntfy/email push; a Claude Routine is the zero-infra alternative (ro token only — routines skip write approvals). +- **Exit criteria:** user asks Claude (phone or desktop) "how's my week?" against live-ish data with no Mac involved; morning report arrives on schedule. + +**Phase 1.5 — Chat-UI auth + in-app upload** +- OAuth 2.1 shim (SDK `mcpAuthRouter` + proxy provider; GitHub/Google as the single-user login; PKCE + CIMD, DCR fallback) so **claude.ai web/iOS and ChatGPT** connect properly. Static tokens remain for headless. +- Optional in-app **"Sync to my server"** (Settings; URL+token fields; POSTs the checkpointed `.noopbak` via the `OuraAPIClient` URLSession pattern; manual button first, on-launch catch-up like BackupSync later). Off by default, personal-fork only. + +**Phase 2 — Gated writes + corroboration cleanup** +- `server.sqlite` staging: `propose_edit(kind, payload, rationale)` → human-readable diff → `confirm_edit(id)` / `reject_edit(id)`; `list_pending`, `edit_journal`, `undo_edit`. Kinds: `fix_workout`, `delete_workout`, `add_workout`, `adjust_sleep_bounds`, `delete_metric_point`, `set_baseline_note`. +- Tool annotations (`readOnlyHint`/`destructiveHint`) so both chat UIs render approval framing; **enforcement stays server-side**: the ro token cannot call write tools at all; `confirm_edit` requires the rw credential and is never auto-invoked by prompts we ship. +- Confirmed edits apply to a **server overlay** consulted by read tools (mirror stays pristine) — the overlay + journal are exactly what Phase 3 ships to the phone. + +**Phase 3 — Sync-back to the phone (design constraints locked now, detail then)** +- Transport: app pulls `GET /edits?since=` (journal is append-only + idempotent by edit id). +- Apply rules (from the code study): sleep-bound edits → `applySleepEdit`-equivalent with `userEdited=1` (only path that survives recompute); workout/daily/metric corrections → written under a **new source namespace `"noop-cloud"`** + registered in the source resolver (mirror the `oura-api` integration), never in-place mutations of another source's rows; deletions → require a DB-backed tombstone mechanism (the UserDefaults one doesn't sync) — designed in the Phase-3 plan; every applied edit ack'd back so the journal tracks phone-applied state. + +## 6. Auth & security + +- HTTPS only (Fly-terminated). Tokens: ≥32-byte random, stored hashed in `server.sqlite`, `ro` + `rw` scopes; rotation = insert new/revoke old. OAuth shim auto-approves exactly one upstream identity (the user's GitHub/Google), everything else 403. +- PHI posture: single-tenant container, no analytics/telemetry, no third-party processors; Fly volume snapshots (default retention) are the backup; `/ingest` size-capped + zip-validated (`quick_check`, magic bytes, origin gate — mirroring `DataBackup.restore`'s own checks); rate-limited. +- Threat notes: a leaked **ro** token exposes read access (rotate; consider IP allowlist later); a leaked **rw** token is the crown jewel — it's entered exactly twice (Shortcut + Claude Code config). Routines/cron get **ro only**, always. + +## 7. Testing + +- Server: vitest — ingest pipeline (fixture `.noopbak` incl. corrupt/oversized/foreign-origin rejects), every tool against a fixture mirror (multi-source days), staging state machine (propose→confirm/reject/undo), auth middleware (scope enforcement, token hashing), `search`/`fetch` shape conformance. +- Contract smoke: MCP Inspector against local server; Claude Code `claude mcp add --transport http` end-to-end. +- Phase 3 app-side: XCTest against `WhoopStore.inMemory()` proving recompute-survival of applied edits (the `userEdited` + namespace rules). + +## 8. Open questions (answers assumed → flag if wrong) + +- **Q1 Report delivery:** assumed **ntfy.sh push** (free, no account, self-hostable) with email as fallback. OK? +- **Q2 Upload cadence P1:** manual Shortcut tap (or iOS automation e.g. daily at 6am while charging). OK to start manual? +- **Q3 Server repo home:** new sibling repo `/Users/vk/VKDEV/NOOP/noop-cloud` (own git, MIT/PolyForm — pick at creation). OK? +- **Q4 OAuth upstream identity:** GitHub login assumed (user has GH). OK? + +## 9. Out of scope + +Multi-user hosting, HIPAA formalities, WHOOP/Oura credential proxying (Oura import keeps running on-phone), replacing NOOP scoring, Android app changes, upstreaming any of this. diff --git a/docs/superpowers/specs/2026-07-12-sleep-night-stitch-design.md b/docs/superpowers/specs/2026-07-12-sleep-night-stitch-design.md new file mode 100644 index 0000000000..918628344f --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-sleep-night-stitch-design.md @@ -0,0 +1,129 @@ +# Sleep night-stitch: present a wake-split night as one night — design + +**Date:** 2026-07-12 +**Issue:** [#364](https://github.com/ryanbr/noop/issues/364) +**Scope:** Both platforms. Swift core in `Packages/StrandAnalytics` (`SleepStageTotals`) + two +app-target consumers (`StrandiOS/Health/HealthKitBridge.swift`, `Strand/Screens/SleepView.swift`); +Kotlin twins in `android/…/analytics/SleepStageTotals.kt` + `HealthConnectWriter.kt` + +`ui/SleepScreen.kt`. Detector and stored sessions are **untouched** — this is presentation only. + +## Problem + +When the sleep detector sees a sustained mid-night wake it ends the session and starts a new one, so +one night reaches storage as two `SleepSession` rows. A 16-minute wake at ~3 AM produces a main block +plus a second fragment. The daily totals are already correct — `SleepStageTotals.mainNightGroupIndices` +(#561/#525/#861) bridges fragments separated by a gap `< gapBridgeMaxMin` (60 min, plus a 90-min +night-tail bridge for overnight-onset fragments) into one **main-night group** for scoring. But two +consumers still read the raw split sessions: + +- **Apple Health write-back** (`HealthKitBridge.writeSleep`) iterates raw sessions → one `.inBed` + HealthKit entry per fragment → the reported two-entry export. +- **Sleep screen** (`SleepView` / Android `SleepScreen`) surfaces the second fragment as its own card + instead of a wake segment inside the one night. + +Prior art (WHOOP, Oura, Apple Watch) renders one night with the wake folded in as awake time; only +naps / genuinely separate sleeps split. + +## Approach + +One shared grouping/merge primitive in the analytics package; four thin consumers. The +correctness-critical logic — which fragments are one night, and the merged stage timeline with the +wake gap — lives where `swift test` / JVM tests cover it. Consumers render/emit only. + +Rejected: (B) group inline in each consumer — duplicates the bridge across four app-target surfaces +with no CI, guaranteed drift; (C) merge at the detector/storage layer — breaks sleep-time edits +(#318), union dedup, and the Apple Health round-trip. Ryan ruled out (C) explicitly on #364. + +## The primitive + +`mainNightGroupIndices` today builds bridged groups then scores and returns only the **winning** main +night. The consumers need **every** night grouped (main night merged, naps standalone, prior nights +merged), so the grouping pass ([`SleepStageTotals.swift:333-356`](../../../Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStageTotals.swift)) is extracted into a reusable, pure function: + +``` +groupBridgedNights(sessions) -> [DisplayedNight] +``` + +where each `DisplayedNight` carries only **pure grouping/time data — no stage decoding** (so the +grouping layer gains no dependency on the stage vocabulary or the HealthKit decoder): +- `fragmentIndices: [Int]` — indices into the input, ascending; +- `spanStart / spanEnd` — earliest `effectiveStartTs` → latest `endTs` across the group; +- `gapWakes: [(start, end)]` — the inter-fragment seams, one per gap (`start = prevFragment.end`, + `end = nextFragment.start`). These, folded in as `wake`, are the *only* new stage content a merged + night introduces; each fragment's own decoded stages are unchanged. + +Grouping uses the **same** `gapBridgeMaxMin` + night-tail overnight rule the scorer already applies — +no new threshold — so display/export group *consistently with the corrected daily totals*. Naps and +genuinely separate sleeps do not bridge, so they remain their own `DisplayedNight` (own card / own +HealthKit entry). Pure and deterministic; shared verbatim by the Kotlin twin for cross-platform +parity. + +The existing `mainNightGroupIndices` is refactored to call `groupBridgedNights` and then score+pick, +so there is exactly one grouping implementation and the #561 tests still pin it. + +Assembling a merged night's timeline is then "concatenate each fragment's existing decoded stages, +then splice in `gapWakes`" over `[spanStart, spanEnd]`. For the **write path** this assembly is a pure +helper in `Packages/StrandImport` `HealthWriteback` (reusing the tested `stageIntervals` per-fragment +decoder), so it gets `swift test` coverage. The **display path** does the same interleave in its +hypnogram builder (UI layer, compile-checked). A group whose fragments all lack decoded stages yields +one `asleepUnspecified` span over `[spanStart, spanEnd]` plus the `gapWakes`, exactly as `writeSleep` +degrades today. + +## The four consumers + +1. **`writeSleep`** (`HealthKitBridge`) — iterate `groupBridgedNights(sessions)` instead of raw + sessions; one `.inBed` sample over `spanStart→spanEnd` plus one `HKCategorySample` per + `mergedStages` segment (`wake→.awake`, `light→.asleepCore`, `deep→.asleepDeep`, `rem→.asleepREM`). +2. **`SleepView`** (macOS+iOS, shared) — list `DisplayedNight`s; the secondary fragment renders as a + wake segment within the night's hypnogram instead of a separate card. The delicate surface: nap + cards, sleep-time edit/undo (#318), and per-session selection all live here and must keep working — + naps stay their own `DisplayedNight`, and edit/undo still act on the underlying stored session. +3. **`HealthConnectWriter`** (Android) — twin of #1 (`SleepSessionRecord` + stage records). +4. **`SleepScreen`** (Android Compose) — twin of #2. + +## Correctness subtlety: HealthKit dedup across the 2→1 transition + +The idempotency key is `noop::sleep:` per session. A night previously written as +**two** entries must not orphan the second when it becomes **one**. So: + +- The merged entry's samples are keyed by the group's **representative** `startTs` — the earliest + fragment's immutable detected onset (not `effectiveStartTs`, so a user edit keeps identity). +- The delete-then-write predicate's `allowedValues` must include **every** fragment's per-`startTs` + key across all groups (not just the representatives), so a prior per-fragment write is fully + cleared before the merged write. + +Same care on the Health Connect side (its records are keyed analogously). + +## Verification + +- **Core primitive** — `swift test` (StrandAnalytics) + JVM test (Android `SleepStageTotals`), both in + default/fork CI. Cases: short-wake bridge merges two fragments into one night with a `wake` gap + interval at the seam; a nap stays a separate `DisplayedNight`; night-tail overnight bridge; an edited + `effectiveStartTs` moves `spanStart`; unspecified-stage fragments; a three-fragment night. Assert the + refactored `mainNightGroupIndices` still returns byte-identical groups (the #561 regression pins). +- **`HealthWriteback` merged-timeline assembly** — the pure write-path helper (concatenate decoded + fragment stages + splice `gapWakes`, degrade to `asleepUnspecified`) gets `swift test` coverage: the + seam-`wake` interval sits exactly at `[prevEnd, nextStart]`, stages stay clamped to `[spanStart, + spanEnd]`, and the multi-fragment delete-key set is complete. +- **Consumers** — app-target Swift (`NOOPiOS` + `Strand` macOS) compiled via on-demand `app-build.yml`; + Android via fork `android.yml` (the #368 play). HealthKit / Health Connect themselves are not + unit-testable. + +## Deliberately excluded + +- **Proposal 2 — motion-aware wake classification** (restless-in-bed vs out-of-bed from + `gravitySample` / `stepSample`). The gravity stream is too sparse on WHOOP 4.0 to trust (same + sparse-motion limit behind the 4.0 sleep-staging work), and per the repo's derived-biosignal + validation standard an unproven per-record signal ships as instrumentation or behind a default-off + toggle, never as the default. Deferred to an honestly-labelled 5.0/MG follow-up; the 60-min bridge + already decides "same night" for v1. +- **Detector / stored sessions** — unchanged. Sessions stay split in storage; grouping is read-time. +- **Daily totals / scoring** — already correct via the existing bridge; not touched beyond the + no-op-preserving `mainNightGroupIndices` refactor. + +## Cross-platform parity + +`groupBridgedNights` and its `DisplayedNight` output are byte-identical Swift↔Kotlin (same bridge +constants, same tie/order rules, same gap-as-wake construction). The two writers emit the same +one-span-per-night shape; the two sleep screens render the same folded hypnogram. UI is feature-level, +not pixel-level (SwiftUI Charts vs Compose Canvas), per the parity contract. diff --git a/project.yml b/project.yml index 16cf2e5e43..38b3600db4 100644 --- a/project.yml +++ b/project.yml @@ -111,6 +111,11 @@ targets: OURA_CLIENT_ID: $(OURA_CLIENT_ID) OURA_CLIENT_SECRET: $(OURA_CLIENT_SECRET) OURA_REDIRECT_URI: $(OURA_REDIRECT_URI) + # Cloud Sync bundle credentials (Phase 3.5 zero-touch), same untracked-xcconfig chain as the + # OURA_* keys above; read at runtime via CloudSyncSettings.effectiveURL/effectiveToken. Absent/ + # blank values leave the lane on manual Keychain configuration only. + CLOUDSYNC_URL: $(CLOUDSYNC_URL) + CLOUDSYNC_TOKEN: $(CLOUDSYNC_TOKEN) # Registers the noop:// URL scheme so ASWebAuthenticationSession's OAuth redirect # (noop://oura/callback) reaches OuraOAuthProvider instead of bouncing off the OS. CFBundleURLTypes: @@ -139,6 +144,13 @@ targets: # distributed (ad-hoc) build. The app uses no JIT / unsigned-exec memory, so it's safe. ENABLE_HARDENED_RUNTIME: YES COMBINE_HIDPI_IMAGES: YES + # liters (page-replication trial). All three variables are defined EMPTY in the tracked + # Config/Liters.xcconfig and only filled in by the untracked Config/LitersLocal.xcconfig that + # Rust/build-ios.sh writes, so on a checkout that has never built the Rust side these three + # lines append nothing and this target links exactly what it links today. See Config/Liters.xcconfig. + LIBRARY_SEARCH_PATHS: $(inherited) $(LITERS_LIB_DIR) + SWIFT_INCLUDE_PATHS: $(inherited) $(LITERS_HEADER_DIR) + OTHER_LDFLAGS: $(inherited) $(LITERS_LDFLAGS) dependencies: - package: WhoopProtocol - package: WhoopStore @@ -175,6 +187,19 @@ targets: # NOTE: must track the app target's PRODUCT_NAME ("NOOP Staging" since the staging rename). TEST_HOST: "$(BUILT_PRODUCTS_DIR)/NOOP Staging.app/Contents/MacOS/NOOP Staging" BUNDLE_LOADER: "$(TEST_HOST)" + # liters: SWIFT_INCLUDE_PATHS ONLY, deliberately — no LIBRARY_SEARCH_PATHS and no + # OTHER_LDFLAGS, unlike the two app targets. + # + # The include path is required. `Strand.swiftmodule` records `liters_ffiFFI` as a dependency + # (the generated bindings import it), and `@testable import Strand` makes StrandTests resolve + # that transitively — without this it fails with "Unable to resolve module dependency: + # 'liters_ffiFFI'" even though the app itself compiled fine. + # + # The link flags are required to be ABSENT. This bundle already resolves liters' symbols + # through BUNDLE_LOADER, so adding -lnoop_liters would pull a SECOND copy of the Rust + # staticlib into the test bundle. UniFFI keeps its object handle map in static storage; two + # copies means two maps, and a handle minted by one is meaningless to the other. + SWIFT_INCLUDE_PATHS: $(inherited) $(LITERS_HEADER_DIR) # --------------------------------------------------------------------------- # iOS app target (community build-from-source; never distributed via the App @@ -255,14 +280,31 @@ targets: # backgrounded app near the user's chosen time to write the overnight strap-log drop. BGAppRefresh # needs the "fetch" background mode; the task id below must also be permitted (and registered at # launch in StrandiOSApp.init) or the submit fails gracefully and only the foreground/"Run now" - # paths run. + # paths run. Cloud Sync v2's own BGAppRefreshTaskRequest (CloudSyncBackgroundRefresh) shares this + # SAME "fetch" mode — one background mode covers every BGAppRefreshTask identifier in the app. - fetch - # #510: permit the scheduled debug auto-export's BGAppRefreshTask identifier. Built from - # $(PRODUCT_BUNDLE_IDENTIFIER) so it tracks BUNDLE_ID_PREFIX automatically; MUST still match - # ScheduledDebugExport.bgTaskIdentifier EXACTLY (which derives it from Bundle.main the same - # way) and be registered at launch (StrandiOSApp.init), or iOS refuses to schedule/deliver it. + # Cloud Sync v2: required for iOS to wake the backgrounded app on a silent (content-available) + # APNs push (CloudSyncAppDelegate.didReceiveRemoteNotification). Code-complete ahead of the + # server/entitlement: `NOOP.entitlements` doesn't carry `aps-environment` in this branch yet + # (see CloudSyncAppDelegate's doc comment), so no push can actually arrive until that lands — + # this mode alone is inert without it, but harmless to declare early. + - remote-notification + # #510: permit the scheduled debug auto-export's BGAppRefreshTask identifier (and, alongside it, + # Cloud Sync v2's). Both still have to be registered at launch (StrandiOSApp.init) or iOS refuses + # to schedule or deliver them. + # + # These derive from $(PRODUCT_BUNDLE_IDENTIFIER) rather than being spelled out, for the same + # reason APP_GROUP_ID is a single setting: iOS matches the identifier a handler registers against + # THIS list as an exact string, and the entries are namespaced under the bundle id — so a build + # under a different Apple ID rewrote this list and left the hardcoded Swift literals stale, + # silently killing BOTH background lanes with no symptom. The Swift side now resolves from the + # bundle too (BGTaskIdentifier.resolve, Strand/System/BGTaskIdentifier.swift), so the two sides + # track the bundle id together and cannot drift apart again. Xcode expands the variable during + # the Process-Info.plist phase — verified in the BUILT app's Info.plist, not just the generated + # source, since expansion inside a plist ARRAY is the part worth confirming. BGTaskSchedulerPermittedIdentifiers: - $(PRODUCT_BUNDLE_IDENTIFIER).debugexport + - $(PRODUCT_BUNDLE_IDENTIFIER).cloudsync.refresh NSBluetoothAlwaysUsageDescription: "NOOP connects directly to your WHOOP strap over Bluetooth to read heart rate, R-R intervals, battery, and sensor data locally on your iPhone. Nothing leaves your device." NSLocationWhenInUseUsageDescription: "NOOP records your route during an outdoor workout to show distance, pace, and a map, and keeps recording while the screen is off. The route stays on your device. Nothing leaves it." NSHealthShareUsageDescription: "NOOP reads your own Apple Health data on-device to compute recovery, strain, and sleep. Nothing leaves your device." @@ -284,6 +326,11 @@ targets: OURA_CLIENT_ID: $(OURA_CLIENT_ID) OURA_CLIENT_SECRET: $(OURA_CLIENT_SECRET) OURA_REDIRECT_URI: $(OURA_REDIRECT_URI) + # Cloud Sync bundle credentials (Phase 3.5 zero-touch), same untracked-xcconfig chain as the + # OURA_* keys above; read at runtime via CloudSyncSettings.effectiveURL/effectiveToken. Absent/ + # blank values leave the lane on manual Keychain configuration only. + CLOUDSYNC_URL: $(CLOUDSYNC_URL) + CLOUDSYNC_TOKEN: $(CLOUDSYNC_TOKEN) # Registers the noop:// URL scheme so ASWebAuthenticationSession's OAuth redirect # (noop://oura/callback) reaches OuraOAuthProvider instead of bouncing off the OS. CFBundleURLTypes: @@ -306,6 +353,12 @@ targets: GENERATE_INFOPLIST_FILE: NO TARGETED_DEVICE_FAMILY: "1,2" SWIFT_EMIT_LOC_STRINGS: YES + # liters (page-replication trial) — same inert-by-default seam as the macOS target above. + # The slice these resolve to is chosen per-SDK inside Config/LitersLocal.xcconfig, so the + # device and simulator builds pick up their own archive without a second setting here. + LIBRARY_SEARCH_PATHS: $(inherited) $(LITERS_LIB_DIR) + SWIFT_INCLUDE_PATHS: $(inherited) $(LITERS_HEADER_DIR) + OTHER_LDFLAGS: $(inherited) $(LITERS_LDFLAGS) dependencies: - package: WhoopProtocol - package: WhoopStore