Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 76 additions & 27 deletions android/app/src/main/java/com/noop/analytics/SkinTempBackfill.kt
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,22 @@ object SkinTempBackfill {
val noSamples: Int = 0,
/** Nights with no stored sleep session ending on them — nothing to compute a nightly mean over. */
val noSessions: Int = 0,
/** The newest day this page examined; the caller's cursor for the next page. Empty when none. */
val lastDay: String = "",
/** COMPOSITE cursor (`day|deviceId`) for the next page — a day alone would skip a same-day row on
* the far side of a page boundary. Empty when the page was empty. */
val lastCursor: String = "",
/** True when the page came back short — every outstanding night has now been visited this sweep. */
val sweepComplete: Boolean = false,
val declinedNoAnchor: Boolean = false,
/**
* Nights skipped because their strap could not be converted — a WHOOP 4.0 with no resolvable
* anchor. Counted, not silent: this log line exists to answer "why did nothing fill?", and a
* candidates/examined pair that does not reconcile sends the reader looking in the wrong place.
*/
val declined: Int = 0,
) {
val examined: Int get() = filled + noMean + noSamples + noSessions
val examined: Int get() = filled + noMean + noSamples + noSessions + declined

/** Whether ANY strap was declined this page. Derived, so it cannot drift from [declined]. */
val declinedNoAnchor: Boolean get() = declined > 0
}

/**
Expand Down Expand Up @@ -177,43 +186,61 @@ object SkinTempBackfill {
fun computedIdFor(deviceId: String): String =
if (deviceId.endsWith("-noop")) deviceId else "$deviceId-noop"

/**
* The STRAP id a scored row's raw samples are banked under — its own id with the computed suffix
* removed. Derived from the row rather than from whatever device happens to be active, because the
* two are not reliably the same (#1303 serial adoption, a second strap, an imported history).
*/
fun sampleIdFor(rowDeviceId: String): String = rowDeviceId.removeSuffix("-noop")

/** One PAGE of candidate day keys, strictly after [afterDay]. See the DAO's note on why this pages. */
suspend fun candidates(
repo: WhoopRepository,
deviceId: String,
afterDay: String,
afterCursor: String,
max: Int = DEFAULT_MAX_NIGHTS,
): List<String> = repo.daysMissingSkinTempAbsolute(deviceId, afterDay, max)
): List<com.noop.data.SkinTempBackfillRow> = repo.daysMissingSkinTempAbsolute(afterCursor, max)

/**
* Everything a strap's nights need to convert raw to °C. Null from the resolver DECLINES that strap —
* used when a WHOOP 4.0 has no anchor, where writing on the global default would bank temperatures
* outside human range.
*/
data class DeviceContext(
val family: DeviceFamily,
val anchorRaw: Double?,
val wornToleranceSec: Long,
)

/**
* Walk the candidate nights and fill what can be re-derived.
*
* [currentAnchorRaw] must be the anchor the engine is using NOW, not one learned from the window being
* filled — see the anchor note on this object. A 4.0 without one is declined outright.
* Every id comes off the ROW, and every conversion context off [deviceContext] for that row's strap —
* nothing here predicts which device a night belongs to. A 4.0 whose anchor will not resolve is
* declined for that strap alone, so one unusable device cannot stall the rest.
*
* Reads one night at a time rather than one big span: the deep-history case is exactly where a single
* unbounded read would be worst, and a per-night read is also what lets a cancelled run leave every
* night it already filled correctly filled.
*/
suspend fun run(
repo: WhoopRepository,
/** The STRAP id. Row reads/writes use its computed twin; raw sample reads use it directly. */
deviceId: String,
family: DeviceFamily,
currentAnchorRaw: Double?,
wornToleranceSec: Long,
/**
* Resolves a STRAP id to its conversion context, or null to decline it.
*
* A lambda rather than three fixed values because rows can belong to ANY device — a second strap,
* an imported history, an id #1303's serial adoption has moved. Resolving once from whatever
* device is "active" and applying it to every row is the same predicted-id mistake that made two
* earlier releases of this find nothing, just one level up: here it would not find nothing, it
* would convert one strap's raw register on another strap's scale.
*/
deviceContext: suspend (String) -> DeviceContext?,
zone: java.time.ZoneId = java.time.ZoneId.systemDefault(),
nowSeconds: Long = System.currentTimeMillis() / 1000,
afterDay: String = "",
afterCursor: String = "",
max: Int = DEFAULT_MAX_NIGHTS,
limit: Int = STREAM_LIMIT,
): Report {
if (requiresAnchor(family) && currentAnchorRaw == null) {
return Report(declinedNoAnchor = true)
}
val anchor = anchorFor(family, currentAnchorRaw)
val rowId = computedIdFor(deviceId)
val days = candidates(repo, rowId, afterDay, max)
val days = candidates(repo, afterCursor, max)
if (days.isEmpty()) return Report(sweepComplete = true)

val nowLocalMidnight = java.time.Instant.ofEpochSecond(nowSeconds)
Expand All @@ -222,7 +249,19 @@ object SkinTempBackfill {
var noMean = 0
var noSamples = 0
var noSessions = 0
for (day in days) {
var declined = 0
// One resolve per strap, not per night.
val contexts = HashMap<String, DeviceContext?>()
for (row in days) {
val day = row.day
// Both namespaces come FROM THE ROW: the scored row's own id, and the strap id its raw
// samples sit under. Nothing here predicts an id.
val rowId = row.deviceId
val sampleId = sampleIdFor(rowId)
val ctx = contexts.getOrPut(sampleId) { deviceContext(sampleId) }
if (ctx == null) { declined++; continue }
if (requiresAnchor(ctx.family) && ctx.anchorRaw == null) { declined++; continue }
val anchor = anchorFor(ctx.family, ctx.anchorRaw)
val dayStart = runCatching {
java.time.LocalDate.parse(day).atStartOfDay(zone).toEpochSecond()
}.getOrNull() ?: continue
Expand All @@ -239,18 +278,18 @@ object SkinTempBackfill {
if (sessions.isEmpty()) { noSessions++; continue }
val readFrom = sessions.minOf { it.start }
val readTo = sessions.maxOf { it.end }
val skin = repo.skinTempSamples(deviceId, readFrom, readTo, limit)
val skin = repo.skinTempSamples(sampleId, readFrom, readTo, limit)
if (skin.isEmpty()) { noSamples++; continue }
val hr = repo.hrSamplesForDevice(deviceId, readFrom, readTo, limit)
val mean = nightlyAbsolute(sessions, hr, skin, family, anchor, wornToleranceSec)
val hr = repo.hrSamplesForDevice(sampleId, readFrom, readTo, limit)
val mean = nightlyAbsolute(sessions, hr, skin, ctx.family, anchor, ctx.wornToleranceSec)
if (mean == null) { noMean++; continue }
// Fill-only by construction; a row that gained an absolute meanwhile reports 0 and is left be.
if (repo.fillSkinTempAbsolute(rowId, day, mean) > 0) filled++ else noMean++
}
return Report(
candidates = days.size, filled = filled, noMean = noMean,
noSamples = noSamples, noSessions = noSessions,
lastDay = days.last(),
noSamples = noSamples, noSessions = noSessions, declined = declined,
lastCursor = cursorOf(days.last()),
// A short page means this sweep has seen every outstanding night.
sweepComplete = days.size < max,
)
Expand All @@ -259,6 +298,16 @@ object SkinTempBackfill {
/** Per-night read cap, matching the engine's stream limit. */
const val STREAM_LIMIT = 200_000

/**
* The composite paging cursor for a row — see [Report.lastCursor].
*
* Safe as a plain string comparison because [com.noop.data.DailyMetric.day] is FIXED-WIDTH ISO
* (`yyyy-MM-dd`): the separator always lands at the same offset, so comparing the concatenation is
* exactly comparing `(day, deviceId)`, which is the order the query pages in. A variable-width key
* here would silently reorder the sweep.
*/
fun cursorOf(row: com.noop.data.SkinTempBackfillRow): String = "${row.day}|${row.deviceId}"

/** Seconds in a local day, for the end-of-day bound the engine attributes sessions by. */
const val SECONDS_PER_DAY: Long = 86_400L
}
13 changes: 13 additions & 0 deletions android/app/src/main/java/com/noop/data/Entities.kt
Original file line number Diff line number Diff line change
Expand Up @@ -745,3 +745,16 @@ data class V18AuxSampleEntity(
return result
}
}

/**
* #1855: one candidate row for the skin-temp absolutes backfill — the day AND the device namespace it
* actually lives in.
*
* The id is READ, never assumed. Scored rows sit under the engine's `<strap>-noop` namespace while raw
* samples sit under the strap id, and #1303's serial adoption can move the strap id underneath both. Two
* separate releases of this backfill found nothing because they predicted the id instead of reading it.
*/
data class SkinTempBackfillRow(
val deviceId: String,
val day: String,
)
25 changes: 17 additions & 8 deletions android/app/src/main/java/com/noop/data/WhoopDao.kt
Original file line number Diff line number Diff line change
Expand Up @@ -167,25 +167,34 @@ interface WhoopDao : DeviceRegistryDao {
* #1851: the day keys that have a skin-temp DEVIATION but no ABSOLUTE beside it — the nights the
* 21-night `analyzeRecent` window never revisited after `skinTempC` shipped (2026-08-27).
*
* PAGED on [afterDay], not just capped. A plain "oldest N" returns the SAME nights every pass, so a
* NOT filtered by device, deliberately. The id a scored row carries is not something a caller can
* safely predict: the engine writes under `<strap>-noop`, and #1303's serial adoption can move the
* strap id itself. Guessing it wrong finds nothing and is indistinguishable from "already done" —
* which is exactly how this shipped twice. The row carries its own id; take it from there.
*
* The cursor is COMPOSITE (`day|deviceId`), not the day alone. Without a device filter a single day
* can hold rows for several devices, and a page boundary falling between them would skip the row on
* the far side for the whole sweep — a night silently never attempted.
*
* PAGED, not just capped. A plain "oldest N" returns the SAME nights every pass, so a
* run whose first page cannot fill — and the oldest nights are exactly the ones most likely to have
* lost their raw samples — would latch on that page and never reach the newer nights that CAN fill.
* The caller advances a cursor, so one sweep visits every candidate exactly once.
*/
@Query(
"SELECT day FROM dailyMetric WHERE deviceId = :deviceId " +
"AND skinTempC IS NULL AND skinTempDevC IS NOT NULL AND day > :afterDay " +
"ORDER BY day ASC LIMIT :limit"
"SELECT deviceId, day FROM dailyMetric " +
"WHERE skinTempC IS NULL AND skinTempDevC IS NOT NULL " +
"AND (day || '|' || deviceId) > :afterCursor " +
"ORDER BY day ASC, deviceId ASC LIMIT :limit"
)
suspend fun daysMissingSkinTempAbsolute(deviceId: String, afterDay: String, limit: Int): List<String>
suspend fun daysMissingSkinTempAbsolute(afterCursor: String, limit: Int): List<SkinTempBackfillRow>

/** #1851: how many nights are outstanding in total — the uncapped count the "nothing left to try"
* watermark keys on, so a newly imported night re-arms the sweep. */
@Query(
"SELECT COUNT(*) FROM dailyMetric WHERE deviceId = :deviceId " +
"AND skinTempC IS NULL AND skinTempDevC IS NOT NULL"
"SELECT COUNT(*) FROM dailyMetric WHERE skinTempC IS NULL AND skinTempDevC IS NOT NULL"
)
suspend fun countDaysMissingSkinTempAbsolute(deviceId: String): Int
suspend fun countDaysMissingSkinTempAbsolute(): Int

/**
* #1851: fill ONE night's absolute, and only when it is absent.
Expand Down
10 changes: 5 additions & 5 deletions android/app/src/main/java/com/noop/data/WhoopRepository.kt
Original file line number Diff line number Diff line change
Expand Up @@ -1176,12 +1176,12 @@ class WhoopRepository(
dao.spo2Samples(deviceId, from, to, limit)

/** #1851: nights holding a skin-temp deviation but no absolute, oldest first. See [SkinTempBackfill]. */
suspend fun daysMissingSkinTempAbsolute(deviceId: String, afterDay: String, limit: Int): List<String> =
dao.daysMissingSkinTempAbsolute(deviceId, afterDay, limit)
suspend fun daysMissingSkinTempAbsolute(afterCursor: String, limit: Int): List<SkinTempBackfillRow> =
dao.daysMissingSkinTempAbsolute(afterCursor, limit)

/** #1851: total outstanding nights, uncapped. See [daysMissingSkinTempAbsolute]. */
suspend fun countDaysMissingSkinTempAbsolute(deviceId: String): Int =
dao.countDaysMissingSkinTempAbsolute(deviceId)
/** #1851: total outstanding nights, uncapped and across every device. See the DAO's note on why this
* is not filtered by id. */
suspend fun countDaysMissingSkinTempAbsolute(): Int = dao.countDaysMissingSkinTempAbsolute()

/**
* #1851: fill ONE night's absolute, only where none exists. Returns rows changed (0 = already had one).
Expand Down
83 changes: 46 additions & 37 deletions android/app/src/main/java/com/noop/ui/AppViewModel.kt
Original file line number Diff line number Diff line change
Expand Up @@ -738,59 +738,68 @@ class AppViewModel(app: Application) : AndroidViewModel(app) {
*/
private suspend fun backfillSkinTempAbsolutes(deviceId: String) {
val prefs = NoopPrefs.of(appContext)
// Scored rows live in the COMPUTED namespace; raw samples under the strap id. Counting the strap
// id here found nothing at all, so the sweep looked finished before it began.
val outstanding = repository.countDaysMissingSkinTempAbsolute(
com.noop.analytics.SkinTempBackfill.computedIdFor(deviceId),
)
// #1855: not filtered by device. Two releases of this found nothing because they PREDICTED the id
// a scored row carries — the engine's "-noop" namespace, and a strap id that #1303's serial
// adoption can move underneath it. Each row now carries the id it actually lives under.
val outstanding = repository.countDaysMissingSkinTempAbsolute()
if (outstanding == 0) return
if (prefs.getInt(skinTempBackfillStuckKey, -1) == outstanding) return

val ownerSource = RegistryDayOwnerSource(noopApp.deviceRegistry)
val family = ownerSource.skinTempFamily(deviceId)
val tolerance = ownerSource.skinTempWornToleranceSec(deviceId)
val now = System.currentTimeMillis() / 1000
val anchor = if (family == com.noop.protocol.DeviceFamily.WHOOP4) {
// Same span the engine learns its anchor over, NOT the window being filled.
val scanFrom = now - 21L * 24 * 3_600
val windowSkin = repository.skinTempSamples(
deviceId, scanFrom, now, com.noop.analytics.SkinTempBackfill.STREAM_LIMIT,
)
com.noop.protocol.Whoop4SkinTemp.deviceAnchorRaw(windowSkin.map { it.raw })
} else {
null
}
val cursor = prefs.getString(skinTempBackfillCursorKey, "") ?: ""
val report = com.noop.analytics.SkinTempBackfill.run(
repo = repository,
deviceId = deviceId,
family = family,
currentAnchorRaw = anchor,
wornToleranceSec = tolerance,
// Resolved per STRAP the rows actually belong to, not once from whatever device is active —
// converting one strap's raw register on another strap's scale is a wrong number, not a
// missing one.
deviceContext = { strapId ->
val family = ownerSource.skinTempFamily(strapId)
val anchor = if (family == com.noop.protocol.DeviceFamily.WHOOP4) {
// The span the engine learns its anchor over, NOT the window being filled.
val windowSkin = repository.skinTempSamples(
strapId, now - 21L * 24 * 3_600, now,
com.noop.analytics.SkinTempBackfill.STREAM_LIMIT,
)
com.noop.protocol.Whoop4SkinTemp.deviceAnchorRaw(windowSkin.map { it.raw })
} else {
null
}
com.noop.analytics.SkinTempBackfill.DeviceContext(
family = family,
anchorRaw = anchor,
wornToleranceSec = ownerSource.skinTempWornToleranceSec(strapId),
)
},
nowSeconds = now,
afterDay = cursor,
afterCursor = cursor,
)
// A decline records nothing: a 4.0 whose anchor resolves later must still get its backfill.
if (!report.declinedNoAnchor) {
val sweepFills = prefs.getInt(skinTempBackfillSweepFillsKey, 0) + report.filled
val edit = prefs.edit()
if (report.sweepComplete) {
// Every outstanding night has been visited. Start the next sweep from the top, and only
// stop paying for reads when the WHOLE sweep — not just its last page — found nothing.
edit.remove(skinTempBackfillCursorKey).remove(skinTempBackfillSweepFillsKey)
if (sweepFills == 0) edit.putInt(skinTempBackfillStuckKey, outstanding)
} else {
edit.putString(skinTempBackfillCursorKey, report.lastDay)
.putInt(skinTempBackfillSweepFillsKey, sweepFills)
// The cursor ALWAYS advances. `declinedNoAnchor` now means "some strap was declined", not "the run
// did nothing" — withholding the bookkeeping on it would freeze the cursor and re-process page one
// on every tick, forever, for any install holding one un-anchored 4.0. A decline only withholds
// the STUCK watermark, so the sweep retries once that strap's anchor resolves.
val sweepFills = prefs.getInt(skinTempBackfillSweepFillsKey, 0) + report.filled
val edit = prefs.edit()
if (report.sweepComplete) {
// Every outstanding night has been visited. Start the next sweep from the top, and only stop
// paying for reads when the WHOLE sweep — not just its last page — found nothing AND nothing
// was declined.
edit.remove(skinTempBackfillCursorKey).remove(skinTempBackfillSweepFillsKey)
if (sweepFills == 0 && !report.declinedNoAnchor) {
edit.putInt(skinTempBackfillStuckKey, outstanding)
}
if (report.filled > 0) edit.remove(skinTempBackfillStuckKey)
edit.apply()
} else {
edit.putString(skinTempBackfillCursorKey, report.lastCursor)
.putInt(skinTempBackfillSweepFillsKey, sweepFills)
}
if (report.filled > 0) edit.remove(skinTempBackfillStuckKey)
edit.apply()
ble.externalLog(
"skinTempBackfill: page=${report.candidates} filled=${report.filled} noMean=${report.noMean} " +
"noSamples=${report.noSamples} noSessions=${report.noSessions} " +
"declined=${report.declined} " +
"outstanding=$outstanding sweepComplete=${report.sweepComplete} " +
"declined=${report.declinedNoAnchor}",
"sweepFills=$sweepFills",
com.noop.testcentre.TestDomain.UNIVERSAL,
)
}
Expand Down
Loading
Loading