diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SkinTempBackfill.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SkinTempBackfill.swift new file mode 100644 index 0000000000..61f29be71c --- /dev/null +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SkinTempBackfill.swift @@ -0,0 +1,159 @@ +import Foundation +import WhoopProtocol + +/// #1853: backfill the nightly ABSOLUTE skin temperature (`skinTempC`) for nights the 21-night +/// `analyzeRecent` rescore window never reaches. +/// +/// `skinTempC` is written only by the scoring pass (`IntelligenceEngine`, #1663, 2026-08-27), which +/// covers a rolling 21-night window. Nights already outside that window when the column shipped were +/// never revisited, and nothing walks them since — so a night is missing its absolute if it fell +/// outside every window that ran after 27 Aug, regardless of whether its raw `skinTempSample` rows +/// are still on disk (they are: `skinTempSample` has no age-based retention). +/// +/// This file holds the PURE, database-free half of the backfill: the session-attribution rule and the +/// per-night plan. The walker that reads from / writes to `WhoopStore` lives in `Strand/Data`. +/// +/// Three correctness rules (all re-review findings from the Android twin, #1852 — reproduced verbatim +/// or the Swift backfill will be wrong in the same ways): +/// +/// 1. **Attribute sessions by END.** The night window is 54 h wide (30 h before local midnight through +/// the next), so it contains the PREVIOUS night as well. The engine's rule is +/// `matched = allSessions.filter { tsInDay(it.end) }`; handing the funnel the whole window averages +/// two nights into one temperature and stores it as the wrong day's. +/// +/// 2. **Page the candidates.** A plain "oldest N" returns the same nights every pass, and the oldest +/// are the ones most likely to have lost their raw samples — the sweep latches on page one and never +/// reaches the newer nights that can fill. +/// +/// 3. **The WHOOP 4.0 anchor comes from the CURRENT window.** It is learned window-wide, and the +/// engine's note says that is safe because the offset "cancels in the deviation" — true for a delta, +/// not for an absolute. A re-learned anchor puts backfilled nights on a different offset from the +/// stored ones and the chart plots two scales as one line. **No anchor ⇒ decline, never the global +/// default.** +public enum SkinTempBackfill { + + /// One night the backfill is considering: the day key, the night's read window (from/to in unix + /// seconds), and the device owner whose raw streams should be read. The walker builds these from + /// `DailyMetric` rows that have a `skinTempDevC` but no `skinTempC`; the pure plan below decides + /// which sessions and samples belong to each. + public struct NightCandidate: Equatable, Sendable { + public let day: String // yyyy-MM-dd (local-day key, matching DailyMetric.day) + public let from: Int // unix seconds: dayStart - StreamReadCap.lookbackSeconds + public let to: Int // unix seconds: dayStart + 24h (or now for today) + public let owner: String // the device id whose raw streams wrote this night + public init(day: String, from: Int, to: Int, owner: String) { + self.day = day; self.from = from; self.to = to; self.owner = owner + } + } + + /// The result of attempting one night's backfill. + public struct NightResult: Equatable, Sendable { + public let day: String + /// The re-derived absolute (°C), or nil when the night declined (no anchor, too few worn + /// samples, no sessions, etc.). The walker writes `skinTempC` ONLY when this is non-nil. + public let skinTempC: Double? + /// Why the night declined, for the diagnostic report. nil when it filled. + public let declineReason: String? + public init(day: String, skinTempC: Double?, declineReason: String? = nil) { + self.day = day; self.skinTempC = skinTempC; self.declineReason = declineReason + } + } + + /// Rule 1: attribute sessions by END timestamp. The night window is 54 h wide and contains the + /// PREVIOUS night as well, so handing the funnel every session in the window averages two nights + /// into one temperature. The engine's rule is `matched = allSessions.filter { tsInDay(it.end) }`: + /// a session belongs to `day` iff its END falls in `[dayStart, dayStart + 86400)`. + /// + /// `dayStart` is the LOCAL-midnight unix seconds for the day key (the same `dayStart` the engine's + /// per-day loop computes). `sessions` are the raw `SleepSession`s read from the 54 h window. + /// Pure + deterministic; the walker tests this directly. + public static func sessionsForDay(_ sessions: [SleepSession], dayStart: Int) -> [SleepSession] { + let dayEnd = dayStart + 86_400 + return sessions.filter { $0.end >= dayStart && $0.end < dayEnd } + } + + /// Rule 3: the WHOOP 4.0 anchor MUST come from the current scoring window. If no per-device anchor + /// could be learned (too few in-band samples), DECLINE the night — do NOT fall back to the global + /// 826 anchor. A re-learned anchor puts backfilled nights on a different offset from the stored + /// ones and the chart plots two scales as one line. + /// + /// Returns the anchor to use, or nil to decline. For a 5/MG (`.whoop5`) the anchor is always nil + /// and the night proceeds (centidegree path, no anchor). + public static func resolveAnchor(family: DeviceFamily, windowAnchorRaw: Double?) -> Double? { + switch family { + case .whoop4: + // No anchor ⇒ decline. The global 826 is NOT a fallback here (rule 3). + return windowAnchorRaw + case .whoop5: + // Centidegree path: no anchor, no decline on anchor grounds. + return nil + } + } + + /// Compute one night's backfill absolute, applying all three rules. Pure + deterministic; the + /// walker supplies the raw inputs and writes the result. + /// + /// - Parameters: + /// - candidate: the night to backfill (day, window, owner). + /// - sessions: ALL sleep sessions in the 54 h window (the walker reads them once); rule 1 + /// filters to this day's by end timestamp. + /// - hr: HR samples for the night's window. + /// - skinTemp: raw skin-temp samples for the night's window. + /// - family: the device family that wrote the owner's skin-temp rows (`.whoop4` vs `.whoop5`). + /// - windowAnchorRaw: the per-device WHOOP 4.0 anchor learned from the CURRENT scoring window + /// (rule 3). nil for a 5/MG or when the current window couldn't learn one. + /// - dayStart: the LOCAL-midnight unix seconds for `candidate.day`. + /// - wornToleranceSec: the worn-gate timestamp tolerance (0 for WHOOP, >0 for Oura). + /// - Returns: the nightly mean (°C) or nil with a decline reason. + public static func computeNight( + candidate: NightCandidate, + sessions: [SleepSession], + hr: [HRSample], + skinTemp: [SkinTempSample], + family: DeviceFamily, + windowAnchorRaw: Double?, + dayStart: Int, + wornToleranceSec: Int = 0 + ) -> NightResult { + // Rule 1: attribute sessions by END. Don't hand the funnel the whole 54 h window. + let matched = sessionsForDay(sessions, dayStart: dayStart) + if matched.isEmpty { + return NightResult(day: candidate.day, skinTempC: nil, + declineReason: "no sleep session ends on this day") + } + // Rule 3: the WHOOP 4.0 anchor comes from the current window. No anchor ⇒ decline. + let anchor = resolveAnchor(family: family, windowAnchorRaw: windowAnchorRaw) + if family == .whoop4 && anchor == nil { + return NightResult(day: candidate.day, skinTempC: nil, + declineReason: "no per-device 4.0 anchor from the current window (rule 3: decline, not global fallback)") + } + // Run the SAME funnel the scoring pass uses — identical inputs, identical result, no second + // derivation that could disagree. + let diag = AnalyticsEngine.skinTempFunnel( + matched, hr: hr, skinTemp: skinTemp, family: family, + anchorRaw: anchor, wornToleranceSec: wornToleranceSec) + if let mean = diag.mean { + return NightResult(day: candidate.day, skinTempC: mean) + } + // Decline with the funnel's own explanation so the diagnostic report is self-attesting. + let reason = diag.kept < diag.minSamples + ? "only \(diag.kept)/\(diag.minSamples) worn samples" + : "funnel returned no mean" + return NightResult(day: candidate.day, skinTempC: nil, declineReason: reason) + } + + /// Rule 2: page the candidates so the sweep doesn't latch on the oldest N (which are the most + /// likely to have lost their raw samples). Returns the candidates for one page, oldest-first + /// within the page. The walker calls this repeatedly with increasing `page` (0-based) until a page + /// comes back empty. + /// + /// `pageSize` caps the per-pass cost (one sleep-window sample read per night). The issue's open + /// question on cost is answered by keeping this modest (default 50) and letting the caller decide + /// how many pages to run. + public static func page(_ candidates: [NightCandidate], page: Int, pageSize: Int = 50) -> [NightCandidate] { + let start = page * pageSize + guard start < candidates.count else { return [] } + let end = min(start + pageSize, candidates.count) + return Array(candidates[start.. Int { + try syncWrite { db in + try db.execute(sql: """ + UPDATE dailyMetric + SET skinTempC = ? + WHERE deviceId = ? AND day = ? AND skinTempC IS NULL + """, arguments: [skinTempC, deviceId, day]) + return db.changesCount + } + } + // MARK: - Reads /// Cached sleep sessions overlapping [from, to] (by startTs), oldest first. diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/DailySkinTempAbsoluteTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/DailySkinTempAbsoluteTests.swift index 6a43ebdbdb..f6d9f884f3 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/DailySkinTempAbsoluteTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/DailySkinTempAbsoluteTests.swift @@ -70,4 +70,58 @@ final class DailySkinTempAbsoluteTests: XCTestCase { XCTAssertEqual(try XCTUnwrap(rows.first?.skinTempC), 34.4, accuracy: 0.001, "a re-score must be able to correct the stored absolute") } + + // MARK: - #1853: fill-only write (`fillSkinTempC`) + + /// The fill-only write fills a NULL `skinTempC` and touches NOTHING else — the deviation, recovery, + /// HRV, and every other scored field stay exactly as the scoring pass left them. This is the + /// non-negotiable: a single-column write that can only fill a NULL, never an upsert of a rebuilt row. + func testFillSkinTempC_fillsNullAndTouchesNothingElse() async throws { + let store = try await WhoopStore.inMemory() + // A scored night with a deviation but no absolute — the backfill's exact candidate. + let d = DailyMetric(day: "2026-08-14", totalSleepMin: 415, efficiency: 0.9, + deepMin: 88, remMin: 108, lightMin: 219, disturbances: 2, + restingHr: 50, avgHrv: 62.0, recovery: 0.06, strain: 10.5, + exerciseCount: 1, spo2Pct: 96.2, skinTempDevC: 0.52, respRateBpm: 14.7, + steps: 7_900, activeKcalEst: 2_180.0, avgSdnn: 88.4, skinTempC: nil) + try await store.upsertDailyMetrics([d], deviceId: "devA") + + let filled = try await store.fillSkinTempC(deviceId: "devA", day: "2026-08-14", skinTempC: 34.6) + XCTAssertEqual(filled, 1, "exactly one row filled") + + let rows = try await store.dailyMetrics(deviceId: "devA", from: "2026-08-14", to: "2026-08-14") + let row = try XCTUnwrap(rows.first) + XCTAssertEqual(try XCTUnwrap(row.skinTempC), 34.6, accuracy: 0.001, "the absolute was filled") + // Every other field is untouched — the fill-only write touched one column. + XCTAssertEqual(try XCTUnwrap(row.skinTempDevC), 0.52, accuracy: 0.001, "deviation untouched") + XCTAssertEqual(try XCTUnwrap(row.recovery), 0.06, accuracy: 0.001, "recovery untouched") + XCTAssertEqual(try XCTUnwrap(row.avgHrv), 62.0, accuracy: 0.001, "HRV untouched") + XCTAssertEqual(try XCTUnwrap(row.restingHr), 50, "resting HR untouched") + XCTAssertEqual(try XCTUnwrap(row.totalSleepMin), 415, accuracy: 0.001, "sleep untouched") + } + + /// The fill-only write REFUSES to overwrite a measured value. A night that already has an absolute + /// is left alone — the backfill must never clobber a value the scoring pass stored. + func testFillSkinTempC_refusesToOverwriteAnExistingAbsolute() async throws { + let store = try await WhoopStore.inMemory() + let d = DailyMetric(day: "2026-08-14", totalSleepMin: 415, efficiency: 0.9, + deepMin: 88, remMin: 108, lightMin: 219, disturbances: 2, + restingHr: 50, avgHrv: 62.0, recovery: 0.06, strain: 10.5, + exerciseCount: 1, skinTempDevC: 0.52, skinTempC: 33.9) + try await store.upsertDailyMetrics([d], deviceId: "devA") + + let filled = try await store.fillSkinTempC(deviceId: "devA", day: "2026-08-14", skinTempC: 34.6) + XCTAssertEqual(filled, 0, "an existing absolute must NOT be overwritten") + + let rows = try await store.dailyMetrics(deviceId: "devA", from: "2026-08-14", to: "2026-08-14") + XCTAssertEqual(try XCTUnwrap(rows.first?.skinTempC), 33.9, accuracy: 0.001, + "the original measured value survives") + } + + /// The fill-only write is a no-op when the row doesn't exist (no orphan rows created). + func testFillSkinTempC_noOpWhenRowDoesNotExist() async throws { + let store = try await WhoopStore.inMemory() + let filled = try await store.fillSkinTempC(deviceId: "devA", day: "2026-08-14", skinTempC: 34.6) + XCTAssertEqual(filled, 0, "no row to fill — no orphan created") + } } diff --git a/Strand/Data/SkinTempBackfillWalker.swift b/Strand/Data/SkinTempBackfillWalker.swift new file mode 100644 index 0000000000..55b5131783 --- /dev/null +++ b/Strand/Data/SkinTempBackfillWalker.swift @@ -0,0 +1,264 @@ +import Foundation +import StrandAnalytics +import WhoopStore +import WhoopProtocol + +/// #1853: the Apple walker that backfills `skinTempC` for nights the 21-night `analyzeRecent` rescore +/// window never reaches. The pure half (session attribution, the plan, rule 3) lives in +/// `StrandAnalytics/SkinTempBackfill.swift`; this file is the I/O layer that reads from / writes to +/// `WhoopStore`. +/// +/// The walker mirrors the scoring pass's per-night reads (same window, same stream limits, same +/// device-family / anchor / tolerance resolution) and calls the SAME `skinTempFunnel` the engine +/// uses, so a backfilled absolute is byte-identical to what the scoring pass would have stored. +/// +/// Trigger: ON-DEMAND from the Test Centre (the issue's "diagnostic first" + safest trigger). Doing +/// it automatically on upgrade risks a long first launch on a deep history. The caller decides how +/// many pages to run; one page is 50 nights by default. +/// +/// Non-negotiable: a fill-only write (`WhoopStore.fillSkinTempC`) that can only fill a NULL, never an +/// upsert of a rebuilt row. `skinTempDevC` and every other scored field are untouched. +/// An `actor`, deliberately, not a `@MainActor` class. This walks up to ten pages of fifty nights and +/// reads three streams for each, and while every `await` yields the main thread, each RESUMPTION and all +/// the per-night work between them would land on it. There is no UI state here - the store is an actor, +/// `DeviceRegistryStore` is a Sendable struct, and the result types are Sendable - so nothing needs the +/// main actor, and a long diagnostic pass has no business competing with the UI for it. +actor SkinTempBackfillWalker { + + /// One night that filled — the day and the re-derived absolute. + struct FilledNight: Sendable { + let day: String + let skinTempC: Double + } + + /// One night that declined — the day and the reason, for the diagnostic report. + struct DeclinedNight: Sendable { + let day: String + let reason: String + } + + /// The result of one backfill pass (one or more pages). + struct PassResult: Sendable { + /// Nights that filled (day → absolute °C). + let filled: [FilledNight] + /// Nights that declined (day → reason), for the diagnostic report. + let declined: [DeclinedNight] + /// Nights skipped because no raw `skinTempSample` rows exist for the window (the data is gone, + /// not the backfill's fault — see the issue's "Diagnostic first" note). + let noRawData: [String] + /// Whether another page might yield more fills (false when the last page was empty). + let moreRemaining: Bool + + var totalAttempted: Int { filled.count + declined.count + noRawData.count } + } + + private let store: WhoopStore + private let computedId: String + + init(store: WhoopStore) { + self.store = store + self.computedId = Repository.whoopSource + "-noop" + } + + /// Run one page of the backfill. The caller calls this repeatedly with increasing `page` until + /// `moreRemaining` is false or a budget is reached. + /// + /// - Parameters: + /// - page: 0-based page index (rule 2: page the candidates so the sweep doesn't latch on the + /// oldest N). + /// - pageSize: nights per page (default 50). + /// - windowAnchorRaw: the per-device WHOOP 4.0 anchor learned from the CURRENT scoring window + /// (rule 3). The caller obtains this from the same window-wide scan the scoring pass uses. + /// nil for a 5/MG or when the current window couldn't learn one. + /// - tzOffsetSeconds: seconds east of UTC, for local-day → unix-seconds conversion. + func runPage(page: Int, pageSize: Int = 50, + windowAnchorRaw: Double?, + tzOffsetSeconds: Int) async -> PassResult { + // 1. Find candidate nights: computed rows with a deviation but no absolute. + let candidates = await candidateNights() + let pageCandidates = SkinTempBackfill.page(candidates, page: page, pageSize: pageSize) + if pageCandidates.isEmpty { + return PassResult(filled: [], declined: [], noRawData: [], moreRemaining: false) + } + + // 2. Resolve the device family + worn tolerance for the owner (single-owner install: the + // canonical "my-whoop"). The registry is read once, mirroring the scoring pass. + let registry = DeviceRegistryStore(dbQueue: store.registryWriter) + let regDevices = (try? registry.all()) ?? [] + let activeId = (try? registry.activeDeviceId()) ?? Repository.whoopSource + + var filled: [FilledNight] = [] + var declined: [DeclinedNight] = [] + var noRawData: [String] = [] + + for candidate in pageCandidates { + // Resolve the day owner for this night (same resolver the scoring pass uses). + let owner = await IntelligenceEngine.resolveDayOwner( + day: candidate.day, from: candidate.from, to: candidate.to, + store: store, devices: regDevices, activeId: activeId, + registry: registry, fallbackDeviceId: Repository.whoopSource) + + let family = IntelligenceEngine.skinTempFamily(forOwner: owner, devices: regDevices) + let tolerance = IntelligenceEngine.skinTempWornToleranceSec(forOwner: owner, devices: regDevices) + + // Read the raw streams for this night's window — same limits the scoring pass uses. + let skin = (try? await store.skinTempSamples( + deviceId: owner, from: candidate.from, to: candidate.to, + limit: StreamReadCap.skin)) ?? [] + if skin.isEmpty { + noRawData.append(candidate.day) + continue + } + let hr = (try? await store.hrSamples( + deviceId: owner, from: candidate.from, to: candidate.to, + limit: StreamReadCap.hr)) ?? [] + // Read stored sleep sessions for the window (the backfill takes the night's stored sleep + // window, no re-detection — the issue's "take the night's stored sleep window from + // sleepSession"). + let persisted = (try? await store.sleepSessions( + deviceId: owner, from: candidate.from, to: candidate.to, + limit: 4000)) ?? [] + let sessions = persisted.compactMap { AnalyticsEngine.sleepSession(fromProvided: $0) } + + // Compute the local-midnight unix seconds for this day (the day key is local, so the + // midnight is the UTC midnight of (dayStart + tzOffset)). The scoring pass uses the same + // shift: `dayString(ts, offsetSec:) == day` ⇔ `(ts + offsetSec) ∈ [dayStart, +86400)`. + let dayStart = Self.localMidnightUnix(forDay: candidate.day, tzOffsetSeconds: tzOffsetSeconds) + + let result = SkinTempBackfill.computeNight( + candidate: candidate, + sessions: sessions, + hr: hr, + skinTemp: skin, + family: family, + windowAnchorRaw: windowAnchorRaw, + dayStart: dayStart, + wornToleranceSec: tolerance) + + if let abs = result.skinTempC { + // Fill-only write: can only fill a NULL, never overwrite a measured value. + _ = try? await store.fillSkinTempC(deviceId: computedId, day: candidate.day, skinTempC: abs) + filled.append(FilledNight(day: candidate.day, skinTempC: abs)) + } else { + declined.append(DeclinedNight(day: candidate.day, reason: result.declineReason ?? "unknown")) + } + } + + let moreRemaining = pageCandidates.count == pageSize + return PassResult(filled: filled, declined: declined, noRawData: noRawData, + moreRemaining: moreRemaining) + } + + /// Find computed `DailyMetric` rows that have a `skinTempDevC` but no `skinTempC` — the nights + /// the backfill can fill. Oldest-first (the issue pages oldest→newest so the sweep reaches the + /// newer nights that can fill). + private func candidateNights() async -> [SkinTempBackfill.NightCandidate] { + // Read all computed rows (the backfill window is unbounded — the issue's "how far back" is + // every night with samples). A wide day range covers the full local-history span. + let rows = (try? await store.dailyMetrics( + deviceId: computedId, from: "0000-01-01", to: "9999-12-31")) ?? [] + let tzOffset = TimeZone.current.secondsFromGMT() + let now = Int(Date().timeIntervalSince1970) + // Candidate: has a deviation (the night was scored) but no absolute (the column was null). + // Build the night's read window the same way the scoring pass does. + return rows + .filter { $0.skinTempDevC != nil && $0.skinTempC == nil } + .sorted { $0.day < $1.day } + .map { row in + let dayStart = Self.localMidnightUnix(forDay: row.day, tzOffsetSeconds: tzOffset) + let from = dayStart - StreamReadCap.lookbackSeconds + let to = min(dayStart + 86_400, now) // don't read past now for today's partial night + return SkinTempBackfill.NightCandidate(day: row.day, from: from, to: to, + owner: Repository.whoopSource) + } + } + + /// Local-midnight unix seconds for a yyyy-MM-dd day key. The day key is LOCAL, so midnight is the + /// UTC midnight of `(dayStart + tzOffset)`: `dayString(ts, offsetSec:) == day` ⇔ + /// `(ts + offsetSec) ∈ [dayStart, +86400)`, so `dayStart = utcMidnight(day) - tzOffset`. + static func localMidnightUnix(forDay day: String, tzOffsetSeconds: Int) -> Int { + // Parse the yyyy-MM-dd key to a UTC midnight, then shift by the tz offset to get local midnight. + let parts = day.split(separator: "-") + guard parts.count == 3, + let y = Int(parts[0]), let m = Int(parts[1]), let d = Int(parts[2]) else { return 0 } + var comps = DateComponents() + comps.year = y; comps.month = m; comps.day = d + comps.hour = 0; comps.minute = 0; comps.second = 0 + comps.timeZone = TimeZone(identifier: "UTC") + let utcMidnight = Calendar(identifier: .gregorian).date(from: comps) ?? Date() + return Int(utcMidnight.timeIntervalSince1970) - tzOffsetSeconds + } + + // MARK: - Self-contained entry point (the call site) + + /// The aggregate result of a full backfill pass (one or more pages), for the Test Centre report. + struct BackfillResult: Sendable { + let filled: [FilledNight] + let declined: [DeclinedNight] + let noRawData: [String] + /// The window anchor that was used (nil for a 5/MG or when the current window couldn't learn one). + let windowAnchorRaw: Double? + /// Whether the pass reached the end (false when a page was empty) or hit the page budget. + let reachedEnd: Bool + var filledCount: Int { filled.count } + var declinedCount: Int { declined.count } + var noRawDataCount: Int { noRawData.count } + } + + /// Run the full backfill: resolve the WHOOP 4.0 window anchor from the CURRENT scoring window + /// (rule 3 — the same window-wide scan the engine uses), then page through the candidate nights + /// until a page is empty or `maxPages` is reached. This is the single entry point the Test Centre + /// action calls; it owns the anchor scan so the call site does not duplicate engine internals. + /// + /// `maxDays` mirrors the scoring pass's window (default 21). `maxPages` caps the per-pass cost + /// (default 10 pages × 50 nights = 500 nights, plenty for any install). + func runBackfill(maxDays: Int = 21, maxPages: Int = 10) async -> BackfillResult { + let tzOffset = TimeZone.current.secondsFromGMT() + let now = Int(Date().timeIntervalSince1970) + let nowLocalMidnight = Self.localMidnightUnix( + forDay: Self.dayKey(now: now, tzOffsetSeconds: tzOffset), tzOffsetSeconds: tzOffset) + // Rule 3: the anchor comes from the CURRENT scoring window, learned window-wide (not per-night). + // Same window bounds the engine's `analyzeRecent` uses for `skinAnchorScanFrom`/`skinAnchorScanTo`. + let scanFrom = nowLocalMidnight - (maxDays - 1) * 86_400 - StreamReadCap.lookbackSeconds + let scanTo = nowLocalMidnight + 18 * 3_600 + let registry = DeviceRegistryStore(dbQueue: store.registryWriter) + let activeId = (try? registry.activeDeviceId()) ?? Repository.whoopSource + let regDevices = (try? registry.all()) ?? [] + let family = IntelligenceEngine.skinTempFamily(forOwner: activeId, devices: regDevices) + // Only a 4.0 needs the anchor; a 5/MG (centidegree path) passes nil and proceeds. + let windowAnchorRaw: Double? + if family == .whoop4 { + let windowSkin = (try? await store.skinTempSamples( + deviceId: activeId, from: scanFrom, to: scanTo, + limit: StreamReadCap.skin)) ?? [] + windowAnchorRaw = Whoop4SkinTemp.deviceAnchorRaw(windowSkin.map { $0.raw }) + } else { + windowAnchorRaw = nil + } + var filled: [FilledNight] = [] + var declined: [DeclinedNight] = [] + var noRawData: [String] = [] + var reachedEnd = false + for page in 0.. String { + let comps = Calendar(identifier: .gregorian).dateComponents( + [.year, .month, .day], from: Date(timeIntervalSince1970: TimeInterval(now + tzOffsetSeconds))) + let y = comps.year ?? 0, m = comps.month ?? 0, d = comps.day ?? 0 + return String(format: "%04d-%02d-%02d", y, m, d) + } +} diff --git a/Strand/Resources/Localizable.xcstrings b/Strand/Resources/Localizable.xcstrings index 9f45660284..755687b563 100644 --- a/Strand/Resources/Localizable.xcstrings +++ b/Strand/Resources/Localizable.xcstrings @@ -210074,6 +210074,18 @@ "ru": {"stringUnit": {"state": "translated", "value": "%lld из последних %lld ночей не записали ВСР. Проверьте, что ремешок надет ночью и синхронизируется."}}, "zh-Hans": {"stringUnit": {"state": "translated", "value": "最近 %2$lld 晚中有 %1$lld 晚未记录 HRV。请检查夜间是否佩戴并已同步。"}}, "zh-Hant": {"stringUnit": {"state": "translated", "value": "最近 %2$lld 晚中有 %1$lld 晚未記錄 HRV。請檢查夜間是否配戴並已同步。"}} + } }, + "Re-derives the absolute °C for nights the 21-night rescore window never reaches, using the same funnel as the scoring pass. Fill-only: it never overwrites a measured value. Watch the strap log for the result.": { "localizations": { + "de": {"stringUnit": {"state": "translated", "value": "Leitet das absolute °C für Nächte neu ab, die das 21-Nächte-Fenster nie erreicht, mit demselben Funnel wie der Scoring-Durchlauf. Nur Auffüllen: ein gemessener Wert wird nie überschrieben. Das Ergebnis steht im Strap-Log."}}, + "en": {"stringUnit": {"state": "translated", "value": "Re-derives the absolute °C for nights the 21-night rescore window never reaches, using the same funnel as the scoring pass. Fill-only: it never overwrites a measured value. Watch the strap log for the result."}}, + "es": {"stringUnit": {"state": "translated", "value": "Vuelve a derivar el °C absoluto de las noches que la ventana de 21 noches nunca alcanza, con el mismo embudo que la pasada de puntuación. Solo rellena: nunca sobrescribe un valor medido. Consulta el resultado en el registro de la banda."}}, + "fr": {"stringUnit": {"state": "translated", "value": "Recalcule le °C absolu des nuits que la fenêtre de 21 nuits n’atteint jamais, avec le même entonnoir que la passe de calcul. Remplissage seul : une valeur mesurée n’est jamais écrasée. Le résultat s’affiche dans le journal du bracelet."}}, + "it": {"stringUnit": {"state": "translated", "value": "Ricalcola il °C assoluto per le notti che la finestra di 21 notti non raggiunge mai, con lo stesso funnel della passata di calcolo. Solo riempimento: non sovrascrive mai un valore misurato. Il risultato è nel log della fascia."}}, + "pl": {"stringUnit": {"state": "translated", "value": "Ponownie wylicza bezwzględne °C dla nocy, których nie obejmuje okno 21 nocy, tym samym lejkiem co przebieg oceniający. Tylko uzupełnia: nigdy nie nadpisuje zmierzonej wartości. Wynik znajdziesz w dzienniku opaski."}}, + "pt-PT": {"stringUnit": {"state": "translated", "value": "Volta a derivar o °C absoluto das noites que a janela de 21 noites nunca alcança, com o mesmo funil da passagem de pontuação. Apenas preenche: nunca substitui um valor medido. Vê o resultado no registo da fita."}}, + "ru": {"stringUnit": {"state": "translated", "value": "Пересчитывает абсолютные °C для ночей, до которых не доходит окно в 21 ночь, тем же алгоритмом, что и обычный расчёт. Только заполняет: измеренное значение никогда не перезаписывается. Результат смотрите в журнале ремешка."}}, + "zh-Hans": {"stringUnit": {"state": "translated", "value": "用与评分流程相同的漏斗，为 21 晚窗口未覆盖的夜晚重新推导绝对温度（°C）。仅填充：绝不覆盖已测量的值。结果见腕带日志。"}}, + "zh-Hant": {"stringUnit": {"state": "translated", "value": "以與評分流程相同的漏斗，為 21 晚視窗未涵蓋的夜晚重新推導絕對溫度（°C）。僅填充：絕不覆寫已測量的值。結果見腕帶日誌。"}} } } }, "version": "1.0" diff --git a/Strand/Screens/TestCentreView.swift b/Strand/Screens/TestCentreView.swift index 941ee50d07..8699f38446 100644 --- a/Strand/Screens/TestCentreView.swift +++ b/Strand/Screens/TestCentreView.swift @@ -30,6 +30,11 @@ struct TestCentreView: View { @State private var infoMessage = "" @State private var showInfo = false + // #1853: skin-temp absolute backfill (on-demand, diagnostic-first). Runs the walker that fills + // `skinTempC` for nights outside the 21-night rescore window. In-progress flag + last result line. + @State private var skinTempBackfillRunning = false + @State private var skinTempBackfillStatus: String? + // Section 3: scheduled daily auto-export, the same ScheduledDebugExport store the Settings card uses. @State private var debugExportOn = ScheduledDebugExport.isEnabled @State private var debugExportMinutes = ScheduledDebugExport.timeMinutes @@ -277,6 +282,25 @@ struct TestCentreView: View { Divider().overlay(StrandPalette.hairline) + // #1853: skin-temp absolute backfill (on-demand, diagnostic-first). Fills `skinTempC` + // for nights outside the 21-night rescore window that never got an absolute. Fill-only: + // it can only fill a NULL, never overwrite a measured value or touch the deviation. + NoopButton("Backfill skin-temp absolutes", systemImage: "thermometer.medium", + kind: .secondary) { + runSkinTempBackfill() + } + .disabled(skinTempBackfillRunning) + Text("Re-derives the absolute °C for nights the 21-night rescore window never reaches, using the same funnel as the scoring pass. Fill-only: it never overwrites a measured value. Watch the strap log for the result.") + .font(StrandFont.caption).foregroundStyle(StrandPalette.textTertiary) + .fixedSize(horizontal: false, vertical: true) + if let status = skinTempBackfillStatus { + Text(status) + .font(StrandFont.caption).foregroundStyle(StrandPalette.textSecondary) + .fixedSize(horizontal: false, vertical: true) + } + + Divider().overlay(StrandPalette.hairline) + // Environment dump: the IOSDiagnostics-backed block exportableLogText already carries, // surfaced as a copyable readout (spec section 3.4). NoopButton("Copy environment dump", systemImage: "info.circle", kind: .secondary) { @@ -522,6 +546,37 @@ struct TestCentreView: View { showInfo = true } + /// #1853: run the skin-temp absolute backfill on demand (diagnostic-first trigger). The walker + /// resolves the WHOOP 4.0 window anchor from the current scoring window, pages through the + /// candidate nights, and fills each NULL `skinTempC`. The result is logged to the strap log and + /// shown under the button so the user sees exactly what filled, declined, and had no raw data. + private func runSkinTempBackfill() { + guard !skinTempBackfillRunning else { return } + skinTempBackfillRunning = true + skinTempBackfillStatus = String(localized: "Backfilling…") + Task { @MainActor in + guard let store = await model.repo.storeHandle() else { + skinTempBackfillStatus = String(localized: "No on-device store yet.") + skinTempBackfillRunning = false + return + } + let walker = SkinTempBackfillWalker(store: store) + let result = await walker.runBackfill() + let anchorNote = result.windowAnchorRaw != nil + ? "anchor from current window" + : "no 4.0 anchor (5/MG or too few worn samples)" + let endNote = result.reachedEnd ? "reached end" : "hit page budget" + let line = "skin-temp backfill (#1853): filled \(result.filledCount), " + + "declined \(result.declinedCount), no raw data \(result.noRawDataCount) " + + "(\(anchorNote), \(endNote))" + live.append(log: line) + skinTempBackfillStatus = line + skinTempBackfillRunning = false + // Refresh so the newly-filled absolutes show in the charts/explorer. + await model.repo.refresh() + } + } + private var debugExportTimeBinding: Binding { Binding( get: { diff --git a/StrandTests/SkinTempBackfillTests.swift b/StrandTests/SkinTempBackfillTests.swift new file mode 100644 index 0000000000..09ed34925c --- /dev/null +++ b/StrandTests/SkinTempBackfillTests.swift @@ -0,0 +1,229 @@ +import XCTest +import StrandAnalytics +import WhoopProtocol + +/// #1853: tests for the PURE half of the skin-temp absolute backfill — the session-attribution rule +/// (rule 1), the anchor rule (rule 3), and the pagination rule (rule 2). The I/O walker that reads +/// from / writes to `WhoopStore` is covered separately. +final class SkinTempBackfillTests: XCTestCase { + + // MARK: Rule 1 — attribute sessions by END + + /// A session whose END falls in the day is matched; one whose END falls in the previous day is not. + /// The night window is 54 h wide and contains the previous night, so filtering by end is the only + /// thing that keeps two nights from being averaged into one temperature. + func testSessionsForDay_matchesByEndNotStart() { + // dayStart = 2026-08-25 00:00 UTC = 1_789_632_000 + let dayStart = 1_789_632_000 + let prevNightEnd = dayStart - 3_600 // 23:00 the PREVIOUS day → NOT matched + let thisNightEnd = dayStart + 7 * 3_600 // 07:00 this day → matched + let nextNightEnd = dayStart + 86_400 + 3_600 // 01:00 NEXT day → NOT matched + let sessions = [ + SleepSession(start: dayStart - 8 * 3_600, end: prevNightEnd, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + SleepSession(start: dayStart - 1 * 3_600, end: thisNightEnd, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + SleepSession(start: dayStart + 20 * 3_600, end: nextNightEnd, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + let matched = SkinTempBackfill.sessionsForDay(sessions, dayStart: dayStart) + XCTAssertEqual(matched.count, 1) + XCTAssertEqual(matched.first?.end, thisNightEnd) + } + + /// A session ending exactly at midnight (dayStart) is matched (>= dayStart). + func testSessionsForDay_inclusiveAtDayStart() { + let dayStart = 1_789_632_000 + let sessions = [ + SleepSession(start: dayStart - 8 * 3_600, end: dayStart, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + XCTAssertEqual(SkinTempBackfill.sessionsForDay(sessions, dayStart: dayStart).count, 1) + } + + /// A session ending exactly at next midnight (dayStart + 86400) is NOT matched (< dayEnd). + func testSessionsForDay_exclusiveAtDayEnd() { + let dayStart = 1_789_632_000 + let sessions = [ + SleepSession(start: dayStart + 12 * 3_600, end: dayStart + 86_400, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + XCTAssertEqual(SkinTempBackfill.sessionsForDay(sessions, dayStart: dayStart).count, 0) + } + + /// Empty sessions → empty match (the night declines with "no sleep session ends on this day"). + func testSessionsForDay_emptySessionsReturnsEmpty() { + XCTAssertEqual(SkinTempBackfill.sessionsForDay([], dayStart: 0).count, 0) + } + + // MARK: Rule 3 — the WHOOP 4.0 anchor comes from the current window + + /// A WHOOP 4.0 with a per-device anchor from the current window uses it. + func testResolveAnchor_whoop4WithAnchorReturnsIt() { + let anchor = SkinTempBackfill.resolveAnchor(family: .whoop4, windowAnchorRaw: 1290.0) + XCTAssertEqual(anchor, 1290.0) + } + + /// A WHOOP 4.0 with NO per-device anchor returns nil → the night DECLINES. The global 826 is NOT + /// a fallback here — a re-learned anchor puts backfilled nights on a different offset from the + /// stored ones and the chart plots two scales as one line. + func testResolveAnchor_whoop4WithoutAnchorDeclines() { + let anchor = SkinTempBackfill.resolveAnchor(family: .whoop4, windowAnchorRaw: nil) + XCTAssertNil(anchor, "rule 3: no anchor ⇒ decline, never the global default") + } + + /// A WHOOP 5/MG always returns nil anchor (centidegree path, no anchor) — and does NOT decline on + /// anchor grounds (the decline check in `computeNight` is `family == .whoop4 && anchor == nil`). + func testResolveAnchor_whoop5ReturnsNilButDoesNotDecline() { + let anchor = SkinTempBackfill.resolveAnchor(family: .whoop5, windowAnchorRaw: nil) + XCTAssertNil(anchor) + // The computeNight test below confirms a 5/MG night proceeds without an anchor. + } + + // MARK: computeNight — the full per-night plan + + /// A 5/MG night with enough worn samples fills. The centidegree path needs no anchor. + func testComputeNight_whoop5WithEnoughSamplesFills() { + let dayStart = 1_789_632_000 + let candidate = SkinTempBackfill.NightCandidate( + day: "2026-08-25", from: dayStart - 30 * 3_600, to: dayStart + 24 * 3_600, owner: "my-whoop") + // A session ending in this day. + let sessions = [ + SleepSession(start: dayStart - 1 * 3_600, end: dayStart + 7 * 3_600, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + // 400 worn skin-temp samples (3057 raw = 30.57 °C on 5/MG) at 1 Hz, with matching HR. + let skinTemp = (0..<400).map { i in + SkinTempSample(ts: dayStart - 1 * 3_600 + i, raw: 3057) + } + let hr = (0..<400).map { i in HRSample(ts: dayStart - 1 * 3_600 + i, bpm: 60) } + let result = SkinTempBackfill.computeNight( + candidate: candidate, sessions: sessions, hr: hr, skinTemp: skinTemp, + family: .whoop5, windowAnchorRaw: nil, dayStart: dayStart) + XCTAssertNotNil(result.skinTempC) + XCTAssertEqual(result.skinTempC!, 30.57, accuracy: 0.01) + XCTAssertNil(result.declineReason) + } + + /// A WHOOP 4.0 night with no per-device anchor DECLINES (rule 3). Even with enough samples the + /// night is not filled — the global 826 is NOT a fallback. + func testComputeNight_whoop4WithoutAnchorDeclines() { + let dayStart = 1_789_632_000 + let candidate = SkinTempBackfill.NightCandidate( + day: "2026-08-25", from: dayStart - 30 * 3_600, to: dayStart + 24 * 3_600, owner: "my-whoop") + let sessions = [ + SleepSession(start: dayStart - 1 * 3_600, end: dayStart + 7 * 3_600, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + // Enough samples to fill IF the anchor were allowed — but rule 3 declines before the funnel. + let skinTemp = (0..<400).map { i in + SkinTempSample(ts: dayStart - 1 * 3_600 + i, raw: 826) + } + let hr = (0..<400).map { i in HRSample(ts: dayStart - 1 * 3_600 + i, bpm: 60) } + let result = SkinTempBackfill.computeNight( + candidate: candidate, sessions: sessions, hr: hr, skinTemp: skinTemp, + family: .whoop4, windowAnchorRaw: nil, dayStart: dayStart) + XCTAssertNil(result.skinTempC, "rule 3: no anchor ⇒ decline, never the global default") + XCTAssertNotNil(result.declineReason) + XCTAssertTrue(result.declineReason!.contains("anchor"), "decline reason names the anchor") + } + + /// A night with no sessions ending on the day declines. + func testComputeNight_noSessionsDeclines() { + let dayStart = 1_789_632_000 + let candidate = SkinTempBackfill.NightCandidate( + day: "2026-08-25", from: dayStart - 30 * 3_600, to: dayStart + 24 * 3_600, owner: "my-whoop") + let result = SkinTempBackfill.computeNight( + candidate: candidate, sessions: [], hr: [], skinTemp: [], + family: .whoop5, windowAnchorRaw: nil, dayStart: dayStart) + XCTAssertNil(result.skinTempC) + XCTAssertEqual(result.declineReason, "no sleep session ends on this day") + } + + /// A night with too few worn samples (< minSkinTempSamples = 300) declines. + func testComputeNight_tooFewSamplesDeclines() { + let dayStart = 1_789_632_000 + let candidate = SkinTempBackfill.NightCandidate( + day: "2026-08-25", from: dayStart - 30 * 3_600, to: dayStart + 24 * 3_600, owner: "my-whoop") + let sessions = [ + SleepSession(start: dayStart - 1 * 3_600, end: dayStart + 7 * 3_600, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + // Only 100 samples — below the 300 floor. + let skinTemp = (0..<100).map { i in + SkinTempSample(ts: dayStart - 1 * 3_600 + i, raw: 3057) + } + let hr = (0..<100).map { i in HRSample(ts: dayStart - 1 * 3_600 + i, bpm: 60) } + let result = SkinTempBackfill.computeNight( + candidate: candidate, sessions: sessions, hr: hr, skinTemp: skinTemp, + family: .whoop5, windowAnchorRaw: nil, dayStart: dayStart) + XCTAssertNil(result.skinTempC) + XCTAssertNotNil(result.declineReason) + XCTAssertTrue(result.declineReason!.contains("worn samples"), "decline reason names the sample floor") + } + + /// A night whose sessions END in the PREVIOUS day is not matched (rule 1), even if the session + /// span overlaps this day's window. This is the exact bug rule 1 prevents: two nights averaged + /// into one temperature. + func testComputeNight_previousNightSessionNotMatched() { + let dayStart = 1_789_632_000 + let candidate = SkinTempBackfill.NightCandidate( + day: "2026-08-25", from: dayStart - 30 * 3_600, to: dayStart + 24 * 3_600, owner: "my-whoop") + // A session that ends in the PREVIOUS day (23:00 on Aug 24) but whose span overlaps this day's + // 54 h window. Without rule 1 this would be matched and the night would fill. + let sessions = [ + SleepSession(start: dayStart - 9 * 3_600, end: dayStart - 1 * 3_600, efficiency: 0.9, + stages: [], restingHR: nil, avgHRV: nil, hrOnly: false), + ] + let skinTemp = (0..<400).map { i in + SkinTempSample(ts: dayStart - 9 * 3_600 + i, raw: 3057) + } + let hr = (0..<400).map { i in HRSample(ts: dayStart - 9 * 3_600 + i, bpm: 60) } + let result = SkinTempBackfill.computeNight( + candidate: candidate, sessions: sessions, hr: hr, skinTemp: skinTemp, + family: .whoop5, windowAnchorRaw: nil, dayStart: dayStart) + XCTAssertNil(result.skinTempC, "rule 1: a session ending in the previous day is not this day's") + XCTAssertEqual(result.declineReason, "no sleep session ends on this day") + } + + // MARK: Rule 2 — page the candidates + + /// Page 0 returns the first `pageSize` candidates, oldest-first. + func testPage_firstPageReturnsOldestN() { + let candidates = (0..<120).map { i in + SkinTempBackfill.NightCandidate(day: "2026-08-\(String(format: "%02d", i + 1))", + from: 0, to: 0, owner: "my-whoop") + } + let page0 = SkinTempBackfill.page(candidates, page: 0, pageSize: 50) + XCTAssertEqual(page0.count, 50) + XCTAssertEqual(page0.first?.day, "2026-08-01") + XCTAssertEqual(page0.last?.day, "2026-08-50") + } + + /// Page 2 returns the next chunk; page 3 is empty (only 120 candidates, 50 per page → 3 pages). + func testPage_thirdPageEmpty() { + let candidates = (0..<120).map { i in + SkinTempBackfill.NightCandidate(day: "2026-08-\(String(format: "%02d", i + 1))", + from: 0, to: 0, owner: "my-whoop") + } + let page2 = SkinTempBackfill.page(candidates, page: 2, pageSize: 50) + XCTAssertEqual(page2.count, 20) + let page3 = SkinTempBackfill.page(candidates, page: 3, pageSize: 50) + XCTAssertTrue(page3.isEmpty, "page 3 is empty — the walker stops here") + } + + /// An empty candidate list returns an empty page on every page. + func testPage_emptyCandidatesReturnsEmpty() { + XCTAssertTrue(SkinTempBackfill.page([], page: 0).isEmpty) + } + + /// A partial last page returns exactly the remainder. + func testPage_partialLastPage() { + let candidates = (0..<75).map { i in + SkinTempBackfill.NightCandidate(day: "2026-08-\(String(format: "%02d", i + 1))", + from: 0, to: 0, owner: "my-whoop") + } + let page1 = SkinTempBackfill.page(candidates, page: 1, pageSize: 50) + XCTAssertEqual(page1.count, 25) + } +}