From 3c546305d4a9e1735e21fb77c9ee2dd8b65dab7b Mon Sep 17 00:00:00 2001 From: fearthedear Date: Tue, 6 Oct 2026 12:18:08 +0000 Subject: [PATCH 1/2] feat(push): export daily sleep performance in protocol 1.1 Co-authored-by: fearthedear <12115297+fearthedear@users.noreply.github.com> --- .../java/com/noop/push/PushCapabilities.kt | 9 +- .../java/com/noop/push/PushCoordinator.kt | 28 +++-- .../src/main/java/com/noop/push/PushDao.kt | 43 ++++++-- .../java/com/noop/push/PushHttpTransport.kt | 2 +- .../src/main/java/com/noop/push/PushModels.kt | 9 ++ .../main/java/com/noop/push/PushProtocol.kt | 102 +++++++++++++----- .../java/com/noop/push/PushCoordinatorTest.kt | 31 ++++++ .../noop/push/PushDaoSleepPerformanceTest.kt | 85 +++++++++++++++ .../noop/push/PushHttpTransportPolicyTest.kt | 12 ++- .../java/com/noop/push/PushProtocolTest.kt | 49 +++++++++ docs/PUSH_PROTOCOL.md | 47 +++++--- 11 files changed, 353 insertions(+), 64 deletions(-) create mode 100644 android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt diff --git a/android/app/src/main/java/com/noop/push/PushCapabilities.kt b/android/app/src/main/java/com/noop/push/PushCapabilities.kt index 7494bd30fa..9d0291bcf6 100644 --- a/android/app/src/main/java/com/noop/push/PushCapabilities.kt +++ b/android/app/src/main/java/com/noop/push/PushCapabilities.kt @@ -3,7 +3,7 @@ package com.noop.push import org.json.JSONArray import org.json.JSONObject -/** A receiver may only narrow the fixed protocol 1.0 registry, never name new data. */ +/** A receiver may only narrow the fixed v1 registry for its selected minor version, never name new data. */ data class PushCapabilities( val appendTables: Set, val mutableTables: Set, @@ -32,12 +32,13 @@ data class PushCapabilities( val required = setOf("type", "protocolVersion", "receiverStateId", "streams") val actualMembers = obj.keys().asSequence().toSet() if (!actualMembers.containsAll(required)) { - throw PushProtocolException("capabilities are missing required protocol 1.0 members") + throw PushProtocolException("capabilities are missing required members") } if (actualMembers.any { it in PushProtocol.FORBIDDEN_REMOTE_CONTROL_MEMBERS }) { throw PushProtocolException("capabilities contain forbidden remote-control metadata") } - if (obj.opt("type") != "capabilities" || obj.opt("protocolVersion") != PushProtocol.VERSION) { + val protocolVersion = obj.opt("protocolVersion") as? String + if (obj.opt("type") != "capabilities" || protocolVersion == null || protocolVersion !in PushProtocol.SUPPORTED_VERSIONS) { throw PushProtocolException("unsupported capability document") } val receiverStateId = (obj.opt("receiverStateId") as? String)?.takeIf(::isCanonicalUuid) @@ -59,7 +60,7 @@ data class PushCapabilities( else -> throw PushProtocolException("unknown capability stream") } } - return PushCapabilities(append, mutable, PushProtocol.VERSION, receiverStateId) + return PushCapabilities(append, mutable, protocolVersion!!, receiverStateId) } private fun isCanonicalUuid(value: String): Boolean = runCatching { diff --git a/android/app/src/main/java/com/noop/push/PushCoordinator.kt b/android/app/src/main/java/com/noop/push/PushCoordinator.kt index 9d7dce4281..0e028beb06 100644 --- a/android/app/src/main/java/com/noop/push/PushCoordinator.kt +++ b/android/app/src/main/java/com/noop/push/PushCoordinator.kt @@ -15,7 +15,11 @@ class PushCoordinator( private val zoneId: ZoneId, private val destinationStillCurrent: () -> Boolean = { true }, ) { - suspend fun pushAppend(table: PushAppendTable, deviceId: String): PushResult { + suspend fun pushAppend( + table: PushAppendTable, + deviceId: String, + protocolVersion: String = PushProtocol.VERSION, + ): PushResult { val stored = try { progress.cursor(table, deviceId) } catch (cancelled: CancellationException) { @@ -49,7 +53,7 @@ class PushCoordinator( } if (rows.isEmpty()) return PushResult.NoData val batch = try { - PushProtocol.appendBatch(table, sourceId, deviceId, effective, rows) + PushProtocol.appendBatch(table, sourceId, deviceId, effective, rows, protocolVersion) } catch (t: PushProtocolException) { return rejected(PushFailure(PushFailureCode.LOCAL_DATA)) } @@ -69,11 +73,15 @@ class PushCoordinator( } } - suspend fun pushMutable(table: PushMutableTable, deviceId: String): PushResult { + suspend fun pushMutable( + table: PushMutableTable, + deviceId: String, + protocolVersion: String = PushProtocol.VERSION, + ): PushResult { val fullWindow = PushWindow.ending(today(), zoneId) val rows = try { source.mutableRows( - table, deviceId, fullWindow, PushProtocol.MAX_MUTABLE_SNAPSHOT_RECORDS + 1, + table, deviceId, fullWindow, PushProtocol.MAX_MUTABLE_SNAPSHOT_RECORDS + 1, protocolVersion, ) } catch (cancelled: CancellationException) { throw cancelled @@ -92,7 +100,7 @@ class PushCoordinator( val recordsByDay = days.associateWith { mutableListOf() } for (record in rows) { val size = try { - PushProtocol.mutableRecordEncodedSize(table, record) + PushProtocol.mutableRecordEncodedSize(table, record, protocolVersion) } catch (t: PushProtocolException) { return rejected(PushFailure(PushFailureCode.LOCAL_DATA)) } @@ -111,7 +119,7 @@ class PushCoordinator( } val currentHashes = try { recordsByDay.mapKeys { (day, _) -> day.toString() } - .mapValues { (_, dayRows) -> PushProtocol.mutableSnapshotHash(table, dayRows) } + .mapValues { (_, dayRows) -> PushProtocol.mutableSnapshotHash(table, dayRows, protocolVersion) } } catch (_: PushProtocolException) { return rejected(PushFailure(PushFailureCode.LOCAL_DATA)) } @@ -130,7 +138,7 @@ class PushCoordinator( .flatMap { recordsByDay.getValue(it).asSequence() } .toList() val batches = try { - PushProtocol.mutableBatches(table, sourceId, deviceId, window, changedRows) + PushProtocol.mutableBatches(table, sourceId, deviceId, window, changedRows, protocolVersion) } catch (t: PushProtocolException) { return rejected(PushFailure(PushFailureCode.LOCAL_DATA)) } @@ -215,7 +223,7 @@ class PushCoordinator( var selectedFailure: PushFailure? = null for (deviceId in selectedDevices) { for (table in PushAppendTable.entries.filter { it in capabilities.appendTables }) { - when (val result = pushAppend(table, deviceId)) { + when (val result = pushAppend(table, deviceId, capabilities.protocolVersion)) { is PushResult.Accepted -> { accepted += result.batchCount acceptedRecords += result.recordCount @@ -232,7 +240,7 @@ class PushCoordinator( } } for (table in PushMutableTable.entries.filter { it in capabilities.mutableTables }) { - when (val result = pushMutable(table, deviceId)) { + when (val result = pushMutable(table, deviceId, capabilities.protocolVersion)) { is PushResult.Accepted -> { accepted += result.batchCount acceptedRecords += result.recordCount @@ -282,7 +290,7 @@ class PushCoordinator( ) } val ack = try { - PushAck.parse(response.body) + PushAck.parse(response.body, batch.protocolVersion) } catch (t: PushProtocolException) { return rejected(PushFailure(PushFailureCode.ACK_INVALID)) } diff --git a/android/app/src/main/java/com/noop/push/PushDao.kt b/android/app/src/main/java/com/noop/push/PushDao.kt index e8c73b1460..8b2e47e17f 100644 --- a/android/app/src/main/java/com/noop/push/PushDao.kt +++ b/android/app/src/main/java/com/noop/push/PushDao.kt @@ -22,9 +22,14 @@ internal object PushSnapshotPreflight { } } - fun query(table: String, columns: List, predicate: String, orderBy: String): String = - "SELECT COALESCE(SUM(${rowEstimateExpression(columns)}), 0) FROM " + - "(SELECT ${columns.joinToString()} FROM $table WHERE $predicate ORDER BY $orderBy LIMIT ?)" + fun query( + table: String, + columns: List, + predicate: String, + orderBy: String, + projections: List = columns, + ): String = "SELECT COALESCE(SUM(${rowEstimateExpression(columns)}), 0) FROM " + + "(SELECT ${projections.joinToString()} FROM $table WHERE $predicate ORDER BY $orderBy LIMIT ?)" } internal object PushDeviceDiscovery { @@ -44,7 +49,7 @@ internal object PushDeviceDiscovery { class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshotSource { override suspend fun knownDeviceIds(capabilities: PushCapabilities): List = db.withTransaction { val supportedTables = capabilities.appendTables.map(::appendSpec) + - capabilities.mutableTables.map(::mutableSpec) + capabilities.mutableTables.map { mutableSpec(it, capabilities.protocolVersion) } val sql = PushDeviceDiscovery.query(supportedTables.map(TableSpec::sqlName)) db.query(SimpleSQLiteQuery(sql)).use { cursor -> buildList { @@ -102,9 +107,17 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot deviceId: String, window: PushWindow, limit: Int, + ): List = mutableRows(table, deviceId, window, limit, PushProtocol.VERSION) + + override suspend fun mutableRows( + table: PushMutableTable, + deviceId: String, + window: PushWindow, + limit: Int, + protocolVersion: String, ): List { require(limit in 1..(PushProtocol.MAX_MUTABLE_SNAPSHOT_RECORDS + 1)) - val spec = mutableSpec(table) + val spec = mutableSpec(table, protocolVersion) return db.withTransaction { val (predicate, bounds) = when (table) { PushMutableTable.DAILY_METRIC, PushMutableTable.JOURNAL -> @@ -113,7 +126,7 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot "startTs >= ? AND startTs < ?" to arrayOf(window.startTsInclusive, window.endTsExclusive) } - val sql = "SELECT ${spec.columns.joinToString()} FROM ${spec.sqlName} " + + val sql = "SELECT ${spec.projections.joinToString()} FROM ${spec.sqlName} " + "WHERE deviceId = ? AND $predicate ORDER BY ${spec.keyColumns.joinToString()} ASC LIMIT ?" val args = arrayOfNulls(bounds.size + 2) args[0] = deviceId @@ -125,6 +138,7 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot spec.columns, "deviceId = ? AND $predicate", "${spec.keyColumns.joinToString()} ASC", + spec.projections, ), args, ) @@ -182,8 +196,10 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot val keyColumns: List, val dataColumns: List, val booleanColumns: Set = emptySet(), + val projectionOverrides: Map = emptyMap(), ) { val columns: List = keyColumns + dataColumns + val projections: List = columns.map { projectionOverrides[it] ?: it } } private fun appendSpec(table: PushAppendTable): TableSpec = when (table) { @@ -197,8 +213,11 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot PushAppendTable.GRAVITY_SAMPLE -> GRAVITY } - private fun mutableSpec(table: PushMutableTable): TableSpec = when (table) { - PushMutableTable.DAILY_METRIC -> DAILY + private fun mutableSpec( + table: PushMutableTable, + protocolVersion: String = PushProtocol.VERSION, + ): TableSpec = when (table) { + PushMutableTable.DAILY_METRIC -> if (protocolVersion == PushProtocol.LATEST_VERSION) DAILY_V11 else DAILY PushMutableTable.SLEEP_SESSION -> SLEEP PushMutableTable.WORKOUT -> WORKOUT PushMutableTable.JOURNAL -> JOURNAL @@ -237,6 +256,14 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot "skinTempDevC", "respRateBpm", "steps", "activeKcalEst", "spo2Red", "spo2Ir", ), ) + val DAILY_V11 = DAILY.copy( + dataColumns = DAILY.dataColumns + "sleepPerformance", + projectionOverrides = mapOf( + "sleepPerformance" to "(SELECT value FROM metricSeries AS pushScore " + + "WHERE pushScore.deviceId = dailyMetric.deviceId AND pushScore.day = dailyMetric.day " + + "AND pushScore.key = 'sleep_performance' LIMIT 1) AS sleepPerformance", + ), + ) val SLEEP = TableSpec( "sleepSession", listOf("startTs"), diff --git a/android/app/src/main/java/com/noop/push/PushHttpTransport.kt b/android/app/src/main/java/com/noop/push/PushHttpTransport.kt index 9f26262d83..adece7350c 100644 --- a/android/app/src/main/java/com/noop/push/PushHttpTransport.kt +++ b/android/app/src/main/java/com/noop/push/PushHttpTransport.kt @@ -30,7 +30,7 @@ class PushHttpTransport( .url(endpoint.url) .header("Authorization", "Bearer $bearerToken") .header("Accept", "application/json") - .header(ACCEPT_VERSION_HEADER, PushProtocol.VERSION) + .header(ACCEPT_VERSION_HEADER, PushProtocol.SUPPORTED_VERSIONS.joinToString(", ")) .get() .build() val response = try { diff --git a/android/app/src/main/java/com/noop/push/PushModels.kt b/android/app/src/main/java/com/noop/push/PushModels.kt index c42c69dc7e..54385d19d3 100644 --- a/android/app/src/main/java/com/noop/push/PushModels.kt +++ b/android/app/src/main/java/com/noop/push/PushModels.kt @@ -158,6 +158,15 @@ interface PushSnapshotSource { window: PushWindow, limit: Int, ): List + + /** Newer wire versions may add an explicit projection without broadening older snapshots. */ + suspend fun mutableRows( + table: PushMutableTable, + deviceId: String, + window: PushWindow, + limit: Int, + protocolVersion: String, + ): List = mutableRows(table, deviceId, window, limit) } sealed interface PushResult { diff --git a/android/app/src/main/java/com/noop/push/PushProtocol.kt b/android/app/src/main/java/com/noop/push/PushProtocol.kt index 212d415b9f..edc5d69a8a 100644 --- a/android/app/src/main/java/com/noop/push/PushProtocol.kt +++ b/android/app/src/main/java/com/noop/push/PushProtocol.kt @@ -6,9 +6,11 @@ import java.util.UUID class PushProtocolException(message: String) : IllegalArgumentException(message) -/** Deterministic, bounded NDJSON encoder and acknowledgement codec for protocol 1.0. */ +/** Deterministic, bounded NDJSON encoder and acknowledgement codec for the negotiated push version. */ object PushProtocol { const val VERSION = "1.0" + const val LATEST_VERSION = "1.1" + val SUPPORTED_VERSIONS = listOf(LATEST_VERSION, VERSION) const val MAX_RECORDS = 5_000 /** Hard limit for the decoded UTF-8 NDJSON entity, before optional content coding. */ const val MAX_BODY_BYTES = 4 * 1024 * 1024 @@ -29,13 +31,15 @@ object PushProtocol { deviceId: String, startCursor: PushCursor?, records: List, + protocolVersion: String = VERSION, ): PushBatch { + requireSupportedVersion(protocolVersion) validateUuid(sourceId, "sourceId") if (records.isEmpty()) throw PushProtocolException("append batch must contain a record") require(records.zipWithNext().all { (a, b) -> a.rowId < b.rowId }) { "append records must be strictly ordered by rowid" } - records.forEach { validateRecord(table, it.key, it.data) } + records.forEach { validateRecord(table, it.key, it.data, protocolVersion) } val candidates = records.take(MAX_RECORDS) val selectedRows = ArrayList(candidates.size) val selectedLines = ArrayList(candidates.size) @@ -48,7 +52,7 @@ object PushProtocol { val end = cursorFor(table, deviceId, candidate) val candidateCount = selectedRows.size + 1 val headerSize = appendHeader( - sourceId, table, deviceId, startCursor, end, candidateCount, UUID_PLACEHOLDER, + sourceId, table, deviceId, startCursor, end, candidateCount, UUID_PLACEHOLDER, protocolVersion, ).size if (headerSize + rowBytes + encodedRow.size > MAX_BODY_BYTES) break selectedRows += candidate @@ -58,13 +62,13 @@ object PushProtocol { if (selectedRows.isEmpty()) throw PushProtocolException("first append record exceeds the 4 MiB decoded batch limit") val endCursor = cursorFor(table, deviceId, selectedRows.last()) - val identity = appendIdentity(sourceId, table, deviceId, startCursor, endCursor, selectedRows.size) + val identity = appendIdentity(sourceId, table, deviceId, startCursor, endCursor, selectedRows.size, protocolVersion) val batchId = stableUuid(identity, selectedLines) - val header = appendHeader(sourceId, table, deviceId, startCursor, endCursor, selectedRows.size, batchId) + val header = appendHeader(sourceId, table, deviceId, startCursor, endCursor, selectedRows.size, batchId, protocolVersion) val body = concatenate(header, selectedLines) check(body.size <= MAX_BODY_BYTES) return PushBatch( - protocolVersion = VERSION, + protocolVersion = protocolVersion, batchId = batchId, sourceId = sourceId, table = table, @@ -85,16 +89,18 @@ object PushProtocol { deviceId: String, window: PushWindow, records: List, + protocolVersion: String = VERSION, ): List { + requireSupportedVersion(protocolVersion) validateUuid(sourceId, "sourceId") - records.forEach { validateRecord(table, it.key, it.data) } + records.forEach { validateRecord(table, it.key, it.data, protocolVersion) } val duplicate = records.groupingBy { orderedObjectJson(it.key) }.eachCount().any { it.value > 1 } if (duplicate) throw PushProtocolException("replace_window contains a duplicate key") val lines = records.map(::encodeRecordLine) val replacementIdentity = mapOf( "deviceId" to deviceId, "delivery" to "replace_window", - "protocolVersion" to VERSION, + "protocolVersion" to protocolVersion, "sourceId" to sourceId, "stream" to table.wireName, "window" to selectorBounds(table, window), @@ -116,6 +122,7 @@ object PushProtocol { parts = Int.MAX_VALUE, count = nextCount, batchId = UUID_PLACEHOLDER, + protocolVersion = protocolVersion, ) if (nextCount > MAX_RECORDS || conservativeHeader.size + currentBytes + line.size > MAX_BODY_BYTES) { if (current.isEmpty()) throw PushProtocolException("first replace_window record exceeds the 4 MiB decoded batch limit") @@ -125,7 +132,7 @@ object PushProtocol { } val oneHeader = mutableHeader( sourceId, table, deviceId, window, replacementId, - Int.MAX_VALUE, Int.MAX_VALUE, 1, UUID_PLACEHOLDER, + Int.MAX_VALUE, Int.MAX_VALUE, 1, UUID_PLACEHOLDER, protocolVersion, ) if (oneHeader.size + line.size > MAX_BODY_BYTES) { throw PushProtocolException("replace_window record exceeds the 4 MiB decoded batch limit") @@ -139,16 +146,16 @@ object PushProtocol { return chunks.mapIndexed { index, partLines -> val part = index + 1 val identity = mutableIdentity( - sourceId, table, deviceId, window, replacementId, part, parts, partLines.size, + sourceId, table, deviceId, window, replacementId, part, parts, partLines.size, protocolVersion, ) val batchId = stableUuid(identity, partLines) val header = mutableHeader( - sourceId, table, deviceId, window, replacementId, part, parts, partLines.size, batchId, + sourceId, table, deviceId, window, replacementId, part, parts, partLines.size, batchId, protocolVersion, ) val body = concatenate(header, partLines) check(partLines.size <= MAX_RECORDS && body.size <= MAX_BODY_BYTES) PushBatch( - protocolVersion = VERSION, + protocolVersion = protocolVersion, batchId = batchId, sourceId = sourceId, table = table, @@ -172,7 +179,8 @@ object PushProtocol { deviceId: String, window: PushWindow, records: List, - ): PushBatch = mutableBatches(table, sourceId, deviceId, window, records).singleOrNull() + protocolVersion: String = VERSION, + ): PushBatch = mutableBatches(table, sourceId, deviceId, window, records, protocolVersion).singleOrNull() ?: throw PushProtocolException("replace_window requires multiple parts") /** SHA-256(stream LF device LF compact-natural-key), matching cursor invalidation contract. */ @@ -181,8 +189,12 @@ object PushProtocol { return sha256Hex("${table.wireName}\n$deviceId\n${orderedObjectJson(key)}".toByteArray(Charsets.UTF_8)) } - internal fun mutableRecordEncodedSize(table: PushMutableTable, record: PushMutableRecord): Int { - validateRecord(table, record.key, record.data) + internal fun mutableRecordEncodedSize( + table: PushMutableTable, + record: PushMutableRecord, + protocolVersion: String = VERSION, + ): Int { + validateRecord(table, record.key, record.data, protocolVersion) return encodeRecordLine(record).size } @@ -190,13 +202,14 @@ object PushProtocol { internal fun mutableSnapshotHash( table: PushMutableTable, records: List, + protocolVersion: String = VERSION, ): String { val lines = records.map { record -> - validateRecord(table, record.key, record.data) + validateRecord(table, record.key, record.data, protocolVersion) encodeRecordLine(record) }.sortedWith { left, right -> compareBytes(left, right) } val digest = MessageDigest.getInstance("SHA-256") - digest.update("noop-push-day-hash\n$VERSION\n${table.wireName}\n".toByteArray(Charsets.UTF_8)) + digest.update("noop-push-day-hash\n$protocolVersion\n${table.wireName}\n".toByteArray(Charsets.UTF_8)) lines.forEach(digest::update) return digest.digest().joinToString("") { "%02x".format(it.toInt() and 0xff) } } @@ -277,11 +290,12 @@ object PushProtocol { start: PushCursor?, end: PushCursor, count: Int, + protocolVersion: String, ): Map = mapOf( "delivery" to "append", "deviceId" to deviceId, "endCursor" to cursorJson(end), - "protocolVersion" to VERSION, + "protocolVersion" to protocolVersion, "recordCount" to count, "sourceId" to sourceId, "startCursor" to start?.let(::cursorJson), @@ -297,7 +311,10 @@ object PushProtocol { end: PushCursor, count: Int, batchId: String, - ): ByteArray = encodeLine(appendIdentity(sourceId, table, deviceId, start, end, count) + ("batchId" to batchId)) + protocolVersion: String, + ): ByteArray = encodeLine( + appendIdentity(sourceId, table, deviceId, start, end, count, protocolVersion) + ("batchId" to batchId), + ) private fun mutableIdentity( sourceId: String, @@ -308,11 +325,12 @@ object PushProtocol { part: Int, parts: Int, count: Int, + protocolVersion: String, ): Map = mapOf( "delivery" to "replace_window", "deviceId" to deviceId, "endCursor" to null, - "protocolVersion" to VERSION, + "protocolVersion" to protocolVersion, "recordCount" to count, "sourceId" to sourceId, "startCursor" to null, @@ -335,8 +353,9 @@ object PushProtocol { parts: Int, count: Int, batchId: String, + protocolVersion: String, ): ByteArray = encodeLine( - mutableIdentity(sourceId, table, deviceId, window, replacementId, part, parts, count) + + mutableIdentity(sourceId, table, deviceId, window, replacementId, part, parts, count, protocolVersion) + ("batchId" to batchId), ) @@ -394,17 +413,43 @@ object PushProtocol { return left.size - right.size } - private fun validateRecord(table: PushTable, key: Map, data: Map) { + private fun validateRecord( + table: PushTable, + key: Map, + data: Map, + protocolVersion: String, + ) { + requireSupportedVersion(protocolVersion) val spec = REGISTRY.getValue(table.wireName) + val dataMembers = dataMembers(table, protocolVersion) if (key.keys.toList() != spec.first) throw PushProtocolException("${table.wireName} key does not match registry") - if (data.keys.toSet() != spec.second.toSet() || data.size != spec.second.size) { + if (data.keys.toSet() != dataMembers.toSet() || data.size != dataMembers.size) { throw PushProtocolException("${table.wireName} data does not match registry") } + if (table == PushMutableTable.DAILY_METRIC && protocolVersion == LATEST_VERSION) { + val score = data["sleepPerformance"] + if (score != null && (score !is Number || !score.toDouble().isFinite() || score.toDouble() !in 0.0..100.0)) { + throw PushProtocolException("dailyMetric.sleepPerformance must be null or within 0..100") + } + } if ("deviceId" in key || "deviceId" in data || "synced" in data) { throw PushProtocolException("batch-scoped or local-only column in record") } } + private fun dataMembers(table: PushTable, protocolVersion: String): List { + val members = REGISTRY.getValue(table.wireName).second + return if (table == PushMutableTable.DAILY_METRIC && protocolVersion == LATEST_VERSION) { + members + "sleepPerformance" + } else members + } + + private fun requireSupportedVersion(protocolVersion: String) { + if (protocolVersion !in SUPPORTED_VERSIONS) { + throw PushProtocolException("unsupported protocol version") + } + } + private fun validateRecordKeys(table: PushAppendTable, key: Map) { if (key.keys.toList() != REGISTRY.getValue(table.wireName).first) { throw PushProtocolException("${table.wireName} key does not match registry") @@ -482,20 +527,23 @@ data class PushAck( batch.endCursor, batch.recordCount, "accepted", ) - fun parse(bytes: ByteArray): PushAck { + fun parse(bytes: ByteArray, expectedVersion: String = PushProtocol.VERSION): PushAck { if (bytes.size > PushProtocol.MAX_ACK_BYTES) throw PushProtocolException("ack exceeds size limit") val obj = try { JSONObject(bytes.toString(Charsets.UTF_8)) } catch (_: Throwable) { throw PushProtocolException("ack is not valid JSON") } + val protocolVersion = (obj.opt("protocolVersion") as? String) + ?.takeIf { it == expectedVersion } + ?: throw PushProtocolException("ack protocolVersion does not match the negotiated version") val expectedMembers = setOf( "protocolVersion", "batchId", "stream", "deviceId", "endCursor", "acceptedRows", "status", ) val actualMembers = obj.keys().asSequence().toSet() if (!actualMembers.containsAll(expectedMembers)) { - throw PushProtocolException("ack is missing required protocol 1.0 members") + throw PushProtocolException("ack is missing required members") } if (actualMembers.any { it in PushProtocol.FORBIDDEN_REMOTE_CONTROL_MEMBERS }) { throw PushProtocolException("ack contains forbidden remote-control metadata") @@ -514,7 +562,7 @@ data class PushAck( null, JSONObject.NULL -> null is JSONObject -> { if (!raw.keys().asSequence().toSet().containsAll(setOf("rowId", "keySha256"))) { - throw PushProtocolException("ack.endCursor is missing required protocol 1.0 members") + throw PushProtocolException("ack.endCursor is missing required members") } val row = raw.opt("rowId") as? Number ?: throw PushProtocolException("ack.endCursor.rowId must be an integer") @@ -529,7 +577,7 @@ data class PushAck( else -> throw PushProtocolException("ack.endCursor must be an object or null") } return PushAck( - protocolVersion = string("protocolVersion"), + protocolVersion = protocolVersion, batchId = string("batchId"), stream = string("stream"), deviceId = string("deviceId"), diff --git a/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt b/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt index c0f2f499fb..5b19fcfaff 100644 --- a/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt +++ b/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt @@ -513,6 +513,37 @@ class PushCoordinatorTest { assertEquals(listOf("journal"), transport.batches.map { it.table.wireName }) } + @Test + fun negotiatedV11IsUsedForSnapshotEncodingAndAcknowledgement() = runBlocking { + val day = "2026-08-18" + val score = PushMutableRecord( + linkedMapOf("day" to day), + linkedMapOf( + "totalSleepMin" to null, "efficiency" to null, "deepMin" to null, "remMin" to null, + "lightMin" to null, "disturbances" to null, "restingHr" to null, "avgHrv" to null, + "recovery" to null, "strain" to null, "exerciseCount" to null, "spo2Pct" to null, + "skinTempDevC" to null, "respRateBpm" to null, "steps" to null, "activeKcalEst" to null, + "spo2Red" to null, "spo2Ir" to null, "sleepPerformance" to 88.0, + ), + ) + val source = FakePushSource( + mutable = mutableMapOf(key(PushMutableTable.DAILY_METRIC, "strap-noop") to mutableListOf(score)), + ) + val transport = AckingTransport() + val capabilities = PushCapabilities( + appendTables = emptySet(), + mutableTables = setOf(PushMutableTable.DAILY_METRIC), + protocolVersion = PushProtocol.LATEST_VERSION, + ) + + PushCoordinator(source, transport, MemoryProgress(), SOURCE_A, pinnedToday, ZoneId.of("UTC")) + .pushKnownDevices(capabilities = capabilities) + + val batch = transport.batches.single() + assertEquals(PushProtocol.LATEST_VERSION, batch.protocolVersion) + assertTrue(batch.body.toString(Charsets.UTF_8).contains("\"sleepPerformance\":88.0")) + } + @Test fun emptyCapabilitiesAvoidEvenDeviceDiscovery() = runBlocking { val source = FakePushSource().apply { knownDeviceIdsFailure = AssertionError("Room must stay unopened") } diff --git a/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt b/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt new file mode 100644 index 0000000000..edd6d752d8 --- /dev/null +++ b/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt @@ -0,0 +1,85 @@ +package com.noop.push + +import androidx.room.Room +import com.noop.data.DailyMetric +import com.noop.data.MetricSeriesRow +import com.noop.data.WhoopDatabase +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34], manifest = Config.NONE) +class PushDaoSleepPerformanceTest { + @Test + fun v11ProjectsOnlyTheMatchingMetricSeriesScoreAndV10StaysUnchanged() = runBlocking { + val database = Room.inMemoryDatabaseBuilder( + RuntimeEnvironment.getApplication(), + WhoopDatabase::class.java, + ).allowMainThreadQueries().build() + try { + val day = "2026-08-18" + database.whoopDao().upsertDailyMetrics( + listOf( + DailyMetric(deviceId = "strap-noop", day = day, totalSleepMin = 430.0), + DailyMetric(deviceId = "other-noop", day = day, totalSleepMin = 420.0), + ), + ) + database.whoopDao().upsertMetricSeries( + listOf( + MetricSeriesRow("strap-noop", day, "sleep_performance", 91.5), + MetricSeriesRow("other-noop", day, "sleep_performance", 12.0), + MetricSeriesRow("strap-noop", day, "recovery", 70.0), + ), + ) + val window = PushWindow(day, day, 1L, 2L) + val source = database.pushDao() + + val legacy = source.mutableRows( + PushMutableTable.DAILY_METRIC, "strap-noop", window, 10, PushProtocol.VERSION, + ).single() + val current = source.mutableRows( + PushMutableTable.DAILY_METRIC, "strap-noop", window, 10, PushProtocol.LATEST_VERSION, + ).single() + + assertFalse(legacy.data.containsKey("sleepPerformance")) + assertTrue(current.data.containsKey("sleepPerformance")) + assertEquals(91.5, current.data["sleepPerformance"] as Double, 0.0) + assertNull(current.data["recovery"]) + } finally { + database.close() + } + } + + @Test + fun v11EmitsNullWhenTheSameSourceHasNoSleepPerformancePoint() = runBlocking { + val database = Room.inMemoryDatabaseBuilder( + RuntimeEnvironment.getApplication(), + WhoopDatabase::class.java, + ).allowMainThreadQueries().build() + try { + val day = "2026-08-18" + database.whoopDao().upsertDailyMetrics( + listOf(DailyMetric(deviceId = "strap-noop", day = day, totalSleepMin = 430.0)), + ) + val current = database.pushDao().mutableRows( + PushMutableTable.DAILY_METRIC, + "strap-noop", + PushWindow(day, day, 1L, 2L), + 10, + PushProtocol.LATEST_VERSION, + ).single() + + assertNull(current.data["sleepPerformance"]) + } finally { + database.close() + } + } +} diff --git a/android/app/src/test/java/com/noop/push/PushHttpTransportPolicyTest.kt b/android/app/src/test/java/com/noop/push/PushHttpTransportPolicyTest.kt index a40c9ae17c..dd056fd0b9 100644 --- a/android/app/src/test/java/com/noop/push/PushHttpTransportPolicyTest.kt +++ b/android/app/src/test/java/com/noop/push/PushHttpTransportPolicyTest.kt @@ -90,7 +90,7 @@ class PushHttpTransportPolicyTest { assertEquals("GET", method) assertEquals("Bearer secret", authorization) - assertEquals("1.0", acceptedVersions) + assertEquals("1.1, 1.0", acceptedVersions) assertEquals( PushCapabilitiesResult.Available( PushCapabilities( @@ -103,6 +103,16 @@ class PushHttpTransportPolicyTest { ) } + @Test fun v11CapabilitiesSelectTheNewDailyMetricProjection() { + val result = PushCapabilities.parse( + """{"type":"capabilities","protocolVersion":"1.1","receiverStateId":"00000000-0000-4000-8000-000000000099","streams":["dailyMetric"]}""" + .toByteArray(), + ) + + assertEquals("1.1", result.protocolVersion) + assertEquals(setOf(PushMutableTable.DAILY_METRIC), result.mutableTables) + } + @Test fun receiverWithoutCapabilitiesFailsClosedBeforeHealthDataIsSent() = runBlocking { val client = OkHttpClient.Builder().addInterceptor { chain -> Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1) diff --git a/android/app/src/test/java/com/noop/push/PushProtocolTest.kt b/android/app/src/test/java/com/noop/push/PushProtocolTest.kt index f882f1a715..10dd08a020 100644 --- a/android/app/src/test/java/com/noop/push/PushProtocolTest.kt +++ b/android/app/src/test/java/com/noop/push/PushProtocolTest.kt @@ -24,6 +24,51 @@ class PushProtocolTest { assertTrue(PushAck.parse(extended.toByteArray()).exactlyMatches(batch)) } + @Test + fun version11AddsOnlyTheExplicitDailySleepPerformanceProjection() { + val score = PushMutableRecord( + linkedMapOf("day" to "2026-08-18"), + linkedMapOf( + "totalSleepMin" to null, "efficiency" to null, "deepMin" to null, "remMin" to null, + "lightMin" to null, "disturbances" to null, "restingHr" to null, "avgHrv" to null, + "recovery" to null, "strain" to null, "exerciseCount" to null, "spo2Pct" to null, + "skinTempDevC" to null, "respRateBpm" to null, "steps" to null, "activeKcalEst" to null, + "spo2Red" to null, "spo2Ir" to null, "sleepPerformance" to 87.5, + ), + ) + val batch = PushProtocol.mutableBatch( + PushMutableTable.DAILY_METRIC, SOURCE_A, "strap-noop", testWindow(), listOf(score), + PushProtocol.LATEST_VERSION, + ) + val lines = batch.body.toString(Charsets.UTF_8).trimEnd().lines() + assertEquals("1.1", JSONObject(lines.first()).getString("protocolVersion")) + assertEquals(87.5, JSONObject(lines[1]).getJSONObject("data").getDouble("sleepPerformance"), 0.0) + assertTrue(PushAck.parse(PushAck.fromBatch(batch).encode(), PushProtocol.LATEST_VERSION).exactlyMatches(batch)) + + var rejectedByV10 = false + try { + PushProtocol.mutableBatch( + PushMutableTable.DAILY_METRIC, SOURCE_A, "strap-noop", testWindow(), listOf(score), + PushProtocol.VERSION, + ) + } catch (_: PushProtocolException) { + rejectedByV10 = true + } + assertTrue("protocol 1.0 must not emit the 1.1 member", rejectedByV10) + + var rejectedOutOfRangeScore = false + try { + PushProtocol.mutableBatch( + PushMutableTable.DAILY_METRIC, SOURCE_A, "strap-noop", testWindow(), + listOf(score.copy(data = score.data + ("sleepPerformance" to 101.0))), + PushProtocol.LATEST_VERSION, + ) + } catch (_: PushProtocolException) { + rejectedOutOfRangeScore = true + } + assertTrue("sleepPerformance must be bounded to 0..100", rejectedOutOfRangeScore) + } + @Test fun appendRegistryIsExactlyTheEightDocumentedStreams() { assertEquals( @@ -175,6 +220,10 @@ class PushProtocolTest { PushProtocol.mutableSnapshotHash(PushMutableTable.JOURNAL, listOf(first)), PushProtocol.mutableSnapshotHash(PushMutableTable.JOURNAL, emptyList()), ) + assertNotEquals( + PushProtocol.mutableSnapshotHash(PushMutableTable.JOURNAL, listOf(first), PushProtocol.VERSION), + PushProtocol.mutableSnapshotHash(PushMutableTable.JOURNAL, listOf(first), PushProtocol.LATEST_VERSION), + ) } private fun testWindow() = PushWindow("2026-08-05", "2026-08-18", 1_754_348_400L, 1_755_558_000L) diff --git a/docs/PUSH_PROTOCOL.md b/docs/PUSH_PROTOCOL.md index 9c06c8be77..9c30ff9419 100644 --- a/docs/PUSH_PROTOCOL.md +++ b/docs/PUSH_PROTOCOL.md @@ -1,10 +1,12 @@ # Self-hosted push protocol This document specifies the wire contract for NOOP's **Experimental**, default-off export to a -user-owned HTTP(S) endpoint. Protocol version **1.0** covers the Android-first client. It is a -one-way export protocol: the on-device database is authoritative, the receiver acknowledges writes -and may advertise only which fixed v1 streams it accepts. NOOP never reads health data, commands, -URLs, field names, or other configuration back from the receiver. +user-owned HTTP(S) endpoint. Protocol versions **1.0** and **1.1** cover the Android-first client. +Version 1.1 adds only the explicit nullable daily `sleepPerformance` projection; it does not add a +stream or export the general metric-series table. This is a one-way export protocol: the on-device +database is authoritative, the receiver acknowledges writes and may advertise only which fixed v1 +streams it accepts. NOOP never reads health data, commands, URLs, field names, or other configuration +back from the receiver. NOOP does not ship, operate, or endorse a receiver. A receiver is not part of this repository, and this contract must not be interpreted as an account, hosted-sync, restore, or two-way-sync API. @@ -19,7 +21,7 @@ does not open the health database or send a batch. GET /the/user-configured-path HTTP/1.1 Accept: application/json Authorization: Bearer -NOOP-Push-Accept-Version: 1.0 +NOOP-Push-Accept-Version: 1.1, 1.0 ``` A successful capability response has these required members: @@ -293,7 +295,7 @@ null. The database's `deviceId` is supplied by the header and `synced` is intent | `stream` | Key | Window selector | `data` members | |---|---|---|---| -| `dailyMetric` | `day` | `day` | `totalSleepMin`, `efficiency`, `deepMin`, `remMin`, `lightMin`, `disturbances`, `restingHr`, `avgHrv`, `recovery`, `strain`, `exerciseCount`, `spo2Pct`, `skinTempDevC`, `respRateBpm`, `steps`, `activeKcalEst`, `spo2Red`, `spo2Ir` (all nullable) | +| `dailyMetric` | `day` | `day` | `totalSleepMin`, `efficiency`, `deepMin`, `remMin`, `lightMin`, `disturbances`, `restingHr`, `avgHrv`, `recovery`, `strain`, `exerciseCount`, `spo2Pct`, `skinTempDevC`, `respRateBpm`, `steps`, `activeKcalEst`, `spo2Red`, `spo2Ir` (all nullable); v1.1 adds nullable `sleepPerformance` | | `sleepSession` | `startTs` | `startTs` | `endTs`, `efficiency`, `restingHr`, `avgHrv`, `stagesJSON`, `userEdited`, `startTsAdjusted`, `motionJSON`, `sleepStateJSON`, `stagingSparse` | | `workout` | `startTs`, `sport` | `startTs` | `endTs`, `source`, `durationS`, `energyKcal`, `avgHr`, `maxHr`, `strain`, `distanceM`, `zonesJSON`, `notes`, `routePolyline`, `steps` | | `journal` | `day`, `question` | `day` | `answeredYes`, `notes`, `numericValue` | @@ -305,9 +307,25 @@ are integers; metric and measurement fields are finite numbers. See [DATA_MODEL. and `android/app/src/main/java/com/noop/data/Entities.kt` for the local meanings and units. The wire registry, not automatic reflection over either database, determines what is sent. -Newer tables such as `ppgHrSample`, `stepSample`, `sleepStateSample`, `metricSeries`, raw waveform / -IMU tables, and any future schema additions are not silently exported by v1. Adding a stream or an -optional `data` member requires a documented registry update and protocol minor version. +The `dailyMetric` field list in protocol 1.0 is unchanged. Protocol 1.1 adds only the nullable +`data.sleepPerformance` number (0–100), sourced from the matching `metricSeries` row whose key is +`sleep_performance` and whose source/day match the enclosing `dailyMetric` row. It is not mapped from +`recovery`. When there is no matching score point, the field is JSON `null`. + +The value is source-local: it may be an imported Sleep Performance value or NOOP's locally computed +approximate Rest composite. The latter is not WHOOP's proprietary score. The exporter does not merge +values across device/source scopes; receivers can distinguish scopes from the batch `deviceId` and +should label the score honestly. This field is never a recovery score and is not recomputed or inferred +by the exporter. + +A receiver that selects protocol 1.0 receives the original field set. It does not receive +`sleepPerformance`, preserving compatibility with existing receivers. + +Newer tables such as `ppgHrSample`, `stepSample`, `sleepStateSample`, raw waveform / IMU tables, +and any future schema additions are not silently exported. The general `metricSeries` table is not +a stream: v1.1 projects only the single `sleep_performance` value that matches each exported +`dailyMetric` source/day. Other metric-series keys remain local. Adding a stream or optional `data` +member requires a documented registry update and protocol minor version. ## Acceptance, errors, and retry idempotency @@ -373,7 +391,8 @@ opened; senders never optimistically emit a version the receiver did not select. reject an unsupported major version. - A minor version may add an optional header/data member or a registry stream. Receivers supporting the same major must ignore unknown object members. They may reject an unknown stream without - rejecting batches for supported streams. + rejecting batches for supported streams. Specifically, v1.1 adds only the nullable + `dailyMetric.data.sleepPerformance` field described above. - A sender must not emit a new stream or field while claiming an older minor version. Removing or changing the meaning/type of a field, or changing a stream key or delivery mode, requires a new major version. @@ -388,6 +407,8 @@ endpoint, request diagnostics, set cadence, or otherwise control NOOP. The contract is platform-neutral. NOOP on iOS/macOS uses GRDB/SQLite with the same natural keys and logical v1 streams, but every implementation uses explicit registry projections rather than reflection -or `SELECT *`. A platform lacking a nullable exported column emits `null`; platform-only columns stay -absent until a later negotiated registry version. Scheduling and credential storage are platform -concerns and do not change NDJSON, acknowledgement, idempotency, or replacement semantics. +or `SELECT *`. A sender may continue to negotiate protocol 1.0. A sender that advertises 1.1 must +project the matching `sleep_performance` metric-series point into `dailyMetric.data.sleepPerformance` +using the same source and day as the daily row; a missing point emits `null`. Other metric-series keys +and platform-only columns stay absent. Scheduling and credential storage are platform concerns and do +not change NDJSON, acknowledgement, idempotency, or replacement semantics. From 7d3a0ffef93ee93b06cdc6bf1e2a71ac2e19c28f Mon Sep 17 00:00:00 2001 From: fearthedear Date: Fri, 9 Oct 2026 08:21:59 +0000 Subject: [PATCH 2/2] feat(push): export sleep consistency in protocol 1.1 Co-authored-by: fearthedear <12115297+fearthedear@users.noreply.github.com> --- .../src/main/java/com/noop/push/PushDao.kt | 82 +++++++++++++- .../main/java/com/noop/push/PushProtocol.kt | 12 +- .../push/SleepConsistencyPushProjection.kt | 43 +++++++ .../java/com/noop/push/PushCoordinatorTest.kt | 2 + .../noop/push/PushDaoSleepPerformanceTest.kt | 107 +++++++++++++++++- .../java/com/noop/push/PushProtocolTest.kt | 16 ++- docs/PUSH_PROTOCOL.md | 59 ++++++---- 7 files changed, 289 insertions(+), 32 deletions(-) create mode 100644 android/app/src/main/java/com/noop/push/SleepConsistencyPushProjection.kt diff --git a/android/app/src/main/java/com/noop/push/PushDao.kt b/android/app/src/main/java/com/noop/push/PushDao.kt index 8b2e47e17f..56c08ea829 100644 --- a/android/app/src/main/java/com/noop/push/PushDao.kt +++ b/android/app/src/main/java/com/noop/push/PushDao.kt @@ -4,6 +4,8 @@ import android.database.Cursor import androidx.room.withTransaction import androidx.sqlite.db.SimpleSQLiteQuery import com.noop.data.WhoopDatabase +import java.time.LocalDate +import java.time.ZoneId /** SQL-side upper bound evaluated before Android materializes any unrestricted TEXT value. */ internal object PushSnapshotPreflight { @@ -142,12 +144,85 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot ), args, ) - db.query(SimpleSQLiteQuery(sql, args)).use { cursor -> + val records = db.query(SimpleSQLiteQuery(sql, args)).use { cursor -> buildList { while (cursor.moveToNext()) add(cursor.mutableRecord(spec)) } } + if (table != PushMutableTable.DAILY_METRIC || + protocolVersion != PushProtocol.LATEST_VERSION || records.isEmpty() + ) { + records + } else if (records.last().data["sleepConsistency"] != null) { + // SleepModel prefers the imported consistency series as a whole when its latest day has a + // point; don't mix imported values and the locally calculated fallback in one series. + records + } else { + // When the latest day has no imported consistency score, SleepModel uses the local + // bedtime-spread calculation for the series. Never substitute recovery or performance. + val computedConsistency = sleepConsistencyByWakeDay(deviceId, window) + records.map { record -> + val day = record.key["day"] as? String + record.copy(data = record.data + ("sleepConsistency" to day?.let(computedConsistency::get))) + } + } + } + } + + /** + * Compute the existing trailing-14 bedtime-onset score for each local wake day in the export window. + * Fetching the previous 13 sessions before the first in-window wake keeps the first exported score + * identical to the Sleep screen without loading unbounded sleep history. + */ + private fun sleepConsistencyByWakeDay(deviceId: String, window: PushWindow): Map { + val zone = ZoneId.systemDefault() + val start = LocalDate.parse(window.fromDay).atStartOfDay(zone).toEpochSecond() + val end = LocalDate.parse(window.toDay).plusDays(1).atStartOfDay(zone).toEpochSecond() + val inWindow = db.query( + SimpleSQLiteQuery( + "SELECT startTs, endTs, startTsAdjusted FROM sleepSession " + + "WHERE deviceId = ? AND endTs >= ? AND endTs < ? ORDER BY startTs ASC LIMIT ?", + arrayOf(deviceId, start, end, PushProtocol.MAX_MUTABLE_SNAPSHOT_RECORDS + 1), + ), + ).use { cursor -> + buildList { + while (cursor.moveToNext()) { + val adjusted = cursor.getColumnIndexOrThrow("startTsAdjusted") + add( + SleepConsistencySession( + startTs = cursor.getLong(cursor.getColumnIndexOrThrow("startTs")), + endTs = cursor.getLong(cursor.getColumnIndexOrThrow("endTs")), + startTsAdjusted = if (cursor.isNull(adjusted)) null else cursor.getLong(adjusted), + ), + ) + } + } } + if (inWindow.size > PushProtocol.MAX_MUTABLE_SNAPSHOT_RECORDS) { + throw PushProtocolException("sleep consistency snapshot exceeds local row limit") + } + val firstStartTs = inWindow.firstOrNull()?.startTs ?: return emptyMap() + val prior = db.query( + SimpleSQLiteQuery( + "SELECT startTs, endTs, startTsAdjusted FROM sleepSession " + + "WHERE deviceId = ? AND startTs < ? ORDER BY startTs DESC LIMIT 13", + arrayOf(deviceId, firstStartTs), + ), + ).use { cursor -> + buildList { + while (cursor.moveToNext()) { + val adjusted = cursor.getColumnIndexOrThrow("startTsAdjusted") + add( + SleepConsistencySession( + startTs = cursor.getLong(cursor.getColumnIndexOrThrow("startTs")), + endTs = cursor.getLong(cursor.getColumnIndexOrThrow("endTs")), + startTsAdjusted = if (cursor.isNull(adjusted)) null else cursor.getLong(adjusted), + ), + ) + } + } + }.asReversed() + return SleepConsistencyPushProjection.byWakeDay(prior + inWindow, zone) } private fun ensureSnapshotBounded(sql: String, args: Array) { @@ -257,11 +332,14 @@ class PushDao internal constructor(private val db: WhoopDatabase) : PushSnapshot ), ) val DAILY_V11 = DAILY.copy( - dataColumns = DAILY.dataColumns + "sleepPerformance", + dataColumns = DAILY.dataColumns + listOf("sleepPerformance", "sleepConsistency"), projectionOverrides = mapOf( "sleepPerformance" to "(SELECT value FROM metricSeries AS pushScore " + "WHERE pushScore.deviceId = dailyMetric.deviceId AND pushScore.day = dailyMetric.day " + "AND pushScore.key = 'sleep_performance' LIMIT 1) AS sleepPerformance", + "sleepConsistency" to "(SELECT value FROM metricSeries AS pushConsistency " + + "WHERE pushConsistency.deviceId = dailyMetric.deviceId AND pushConsistency.day = dailyMetric.day " + + "AND pushConsistency.key = 'sleep_consistency' LIMIT 1) AS sleepConsistency", ), ) val SLEEP = TableSpec( diff --git a/android/app/src/main/java/com/noop/push/PushProtocol.kt b/android/app/src/main/java/com/noop/push/PushProtocol.kt index edc5d69a8a..e7ede7f80a 100644 --- a/android/app/src/main/java/com/noop/push/PushProtocol.kt +++ b/android/app/src/main/java/com/noop/push/PushProtocol.kt @@ -427,9 +427,13 @@ object PushProtocol { throw PushProtocolException("${table.wireName} data does not match registry") } if (table == PushMutableTable.DAILY_METRIC && protocolVersion == LATEST_VERSION) { - val score = data["sleepPerformance"] - if (score != null && (score !is Number || !score.toDouble().isFinite() || score.toDouble() !in 0.0..100.0)) { - throw PushProtocolException("dailyMetric.sleepPerformance must be null or within 0..100") + for (name in listOf("sleepPerformance", "sleepConsistency")) { + val score = data[name] + if (score != null && + (score !is Number || !score.toDouble().isFinite() || score.toDouble() !in 0.0..100.0) + ) { + throw PushProtocolException("dailyMetric.$name must be null or within 0..100") + } } } if ("deviceId" in key || "deviceId" in data || "synced" in data) { @@ -440,7 +444,7 @@ object PushProtocol { private fun dataMembers(table: PushTable, protocolVersion: String): List { val members = REGISTRY.getValue(table.wireName).second return if (table == PushMutableTable.DAILY_METRIC && protocolVersion == LATEST_VERSION) { - members + "sleepPerformance" + members + listOf("sleepPerformance", "sleepConsistency") } else members } diff --git a/android/app/src/main/java/com/noop/push/SleepConsistencyPushProjection.kt b/android/app/src/main/java/com/noop/push/SleepConsistencyPushProjection.kt new file mode 100644 index 0000000000..b230f265a0 --- /dev/null +++ b/android/app/src/main/java/com/noop/push/SleepConsistencyPushProjection.kt @@ -0,0 +1,43 @@ +package com.noop.push + +import java.time.Instant +import java.time.ZoneId +import kotlin.math.sqrt + +/** The small sleep-session shape needed to reproduce the Sleep screen's consistency series for export. */ +internal data class SleepConsistencySession( + val startTs: Long, + val endTs: Long, + val startTsAdjusted: Long?, +) { + val effectiveStartTs: Long get() = startTsAdjusted ?: startTs +} + +/** Source-local twin of SleepModelLogic.consistencySeries, keyed by each session's local wake day. */ +internal object SleepConsistencyPushProjection { + fun byWakeDay(sessions: List, zoneId: ZoneId): Map { + val ordered = sessions.sortedBy { it.startTs } + if (ordered.size < 3) return emptyMap() + + val bedtimeMinutes = ordered.map { session -> + val bedtime = Instant.ofEpochSecond(session.effectiveStartTs).atZone(zoneId) + var minutes = (bedtime.hour * 60 + bedtime.minute).toDouble() + if (minutes < 12 * 60) minutes += 24 * 60 // keep evening and after-midnight times continuous + minutes + } + val scores = linkedMapOf() + for (index in ordered.indices) { + val first = maxOf(0, index - 13) + val window = bedtimeMinutes.subList(first, index + 1) + if (window.size < 3) continue + val mean = window.average() + val variance = window.sumOf { (it - mean) * (it - mean) } / window.size + val score = (100.0 * (1.0 - sqrt(variance) / 120.0)).coerceIn(0.0, 100.0) + val wakeDay = Instant.ofEpochSecond(ordered[index].endTs).atZone(zoneId).toLocalDate().toString() + // SleepModel orders by startTs; if a day has multiple sessions, its latest session supplies + // the one daily point, matching the unique dailyMetric day key. + scores[wakeDay] = score + } + return scores + } +} diff --git a/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt b/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt index 5b19fcfaff..74eb2f6139 100644 --- a/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt +++ b/android/app/src/test/java/com/noop/push/PushCoordinatorTest.kt @@ -524,6 +524,7 @@ class PushCoordinatorTest { "recovery" to null, "strain" to null, "exerciseCount" to null, "spo2Pct" to null, "skinTempDevC" to null, "respRateBpm" to null, "steps" to null, "activeKcalEst" to null, "spo2Red" to null, "spo2Ir" to null, "sleepPerformance" to 88.0, + "sleepConsistency" to 73.0, ), ) val source = FakePushSource( @@ -542,6 +543,7 @@ class PushCoordinatorTest { val batch = transport.batches.single() assertEquals(PushProtocol.LATEST_VERSION, batch.protocolVersion) assertTrue(batch.body.toString(Charsets.UTF_8).contains("\"sleepPerformance\":88.0")) + assertTrue(batch.body.toString(Charsets.UTF_8).contains("\"sleepConsistency\":73.0")) } @Test diff --git a/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt b/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt index edd6d752d8..ba5ccb9b87 100644 --- a/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt +++ b/android/app/src/test/java/com/noop/push/PushDaoSleepPerformanceTest.kt @@ -3,6 +3,7 @@ package com.noop.push import androidx.room.Room import com.noop.data.DailyMetric import com.noop.data.MetricSeriesRow +import com.noop.data.SleepSession import com.noop.data.WhoopDatabase import kotlinx.coroutines.runBlocking import org.junit.Assert.assertEquals @@ -14,6 +15,8 @@ import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.RuntimeEnvironment import org.robolectric.annotation.Config +import java.time.LocalDateTime +import java.time.ZoneId @RunWith(RobolectricTestRunner::class) @Config(sdk = [34], manifest = Config.NONE) @@ -37,6 +40,8 @@ class PushDaoSleepPerformanceTest { MetricSeriesRow("strap-noop", day, "sleep_performance", 91.5), MetricSeriesRow("other-noop", day, "sleep_performance", 12.0), MetricSeriesRow("strap-noop", day, "recovery", 70.0), + MetricSeriesRow("strap-noop", day, "sleep_consistency", 77.0), + MetricSeriesRow("other-noop", day, "sleep_consistency", 11.0), ), ) val window = PushWindow(day, day, 1L, 2L) @@ -50,8 +55,10 @@ class PushDaoSleepPerformanceTest { ).single() assertFalse(legacy.data.containsKey("sleepPerformance")) + assertFalse(legacy.data.containsKey("sleepConsistency")) assertTrue(current.data.containsKey("sleepPerformance")) assertEquals(91.5, current.data["sleepPerformance"] as Double, 0.0) + assertEquals(77.0, current.data["sleepConsistency"] as Double, 0.0) assertNull(current.data["recovery"]) } finally { database.close() @@ -59,7 +66,7 @@ class PushDaoSleepPerformanceTest { } @Test - fun v11EmitsNullWhenTheSameSourceHasNoSleepPerformancePoint() = runBlocking { + fun v11EmitsNullWhenTheSourceHasNoScoreOrConsistencyHistory() = runBlocking { val database = Room.inMemoryDatabaseBuilder( RuntimeEnvironment.getApplication(), WhoopDatabase::class.java, @@ -78,6 +85,104 @@ class PushDaoSleepPerformanceTest { ).single() assertNull(current.data["sleepPerformance"]) + assertNull(current.data["sleepConsistency"]) + } finally { + database.close() + } + } + + @Test + fun v11DoesNotMixImportedConsistencyWithTheComputedSeries() = runBlocking { + val database = Room.inMemoryDatabaseBuilder( + RuntimeEnvironment.getApplication(), + WhoopDatabase::class.java, + ).allowMainThreadQueries().build() + try { + val firstDay = "2026-08-17" + val latestDay = "2026-08-18" + val zone = ZoneId.systemDefault() + fun ts(date: String, hour: Int) = + LocalDateTime.parse("${date}T%02d:00".format(hour)).atZone(zone).toEpochSecond() + database.whoopDao().upsertDailyMetrics( + listOf( + DailyMetric(deviceId = "strap-noop", day = firstDay, totalSleepMin = 430.0), + DailyMetric(deviceId = "strap-noop", day = latestDay, totalSleepMin = 430.0), + ), + ) + database.whoopDao().upsertMetricSeries( + listOf(MetricSeriesRow("strap-noop", latestDay, "sleep_consistency", 77.0)), + ) + database.whoopDao().upsertSleepSessions( + listOf( + SleepSession("strap-noop", ts("2026-08-14", 22), ts("2026-08-15", 6)), + SleepSession("strap-noop", ts("2026-08-15", 22), ts("2026-08-16", 6)), + SleepSession("strap-noop", ts("2026-08-16", 22), ts("2026-08-17", 6)), + SleepSession("strap-noop", ts("2026-08-17", 22), ts("2026-08-18", 6)), + ), + ) + + val rows = database.pushDao().mutableRows( + PushMutableTable.DAILY_METRIC, + "strap-noop", + PushWindow(firstDay, latestDay, ts(firstDay, 0), ts("2026-08-19", 0)), + 10, + PushProtocol.LATEST_VERSION, + ).associateBy { it.key["day"] } + + // The local calculation could produce a point for firstDay, but the Sleep screen selects + // the imported series for the entire range when its latest day has an imported point. + assertNull(rows[firstDay]?.data?.get("sleepConsistency")) + assertEquals(77.0, rows[latestDay]?.data?.get("sleepConsistency") as Double, 0.0) + } finally { + database.close() + } + } + + @Test + fun v11ComputesTheExistingBedtimeConsistencyForMatchingWakeDays() = runBlocking { + val database = Room.inMemoryDatabaseBuilder( + RuntimeEnvironment.getApplication(), + WhoopDatabase::class.java, + ).allowMainThreadQueries().build() + try { + val day = "2026-08-18" + val zone = ZoneId.systemDefault() + fun ts(date: String, hour: Int, minute: Int = 0) = + LocalDateTime.parse("${date}T%02d:%02d".format(hour, minute)).atZone(zone).toEpochSecond() + database.whoopDao().upsertDailyMetrics( + listOf(DailyMetric(deviceId = "strap-noop", day = day, totalSleepMin = 430.0)), + ) + database.whoopDao().upsertSleepSessions( + listOf( + SleepSession("strap-noop", ts("2026-08-15", 22), ts("2026-08-16", 6)), + SleepSession("strap-noop", ts("2026-08-16", 23), ts("2026-08-17", 6)), + // The corrected onset is the value used by the Sleep screen's consistency formula. + SleepSession( + "strap-noop", ts("2026-08-17", 23), ts("2026-08-18", 7), + startTsAdjusted = ts("2026-08-18", 0), + ), + // A second source must not influence the rating for strap-noop. + SleepSession("other-noop", ts("2026-08-15", 18), ts("2026-08-16", 6)), + SleepSession("other-noop", ts("2026-08-16", 20), ts("2026-08-17", 6)), + SleepSession("other-noop", ts("2026-08-17", 22), ts("2026-08-18", 7)), + ), + ) + + val current = database.pushDao().mutableRows( + PushMutableTable.DAILY_METRIC, + "strap-noop", + PushWindow(day, day, ts(day, 0), ts("2026-08-19", 0)), + 10, + PushProtocol.LATEST_VERSION, + ).single() + + // 22:00, 23:00, and corrected 00:00 have a population SD of sqrt(2400) minutes: + // exactly the existing SleepModel formula 100 * (1 - SD / 120). + assertEquals( + 100.0 * (1.0 - kotlin.math.sqrt(2400.0) / 120.0), + current.data["sleepConsistency"] as Double, + 1e-9, + ) } finally { database.close() } diff --git a/android/app/src/test/java/com/noop/push/PushProtocolTest.kt b/android/app/src/test/java/com/noop/push/PushProtocolTest.kt index 10dd08a020..13fbb11ff5 100644 --- a/android/app/src/test/java/com/noop/push/PushProtocolTest.kt +++ b/android/app/src/test/java/com/noop/push/PushProtocolTest.kt @@ -25,7 +25,7 @@ class PushProtocolTest { } @Test - fun version11AddsOnlyTheExplicitDailySleepPerformanceProjection() { + fun version11AddsOnlyTheExplicitDailySleepScoreProjections() { val score = PushMutableRecord( linkedMapOf("day" to "2026-08-18"), linkedMapOf( @@ -34,6 +34,7 @@ class PushProtocolTest { "recovery" to null, "strain" to null, "exerciseCount" to null, "spo2Pct" to null, "skinTempDevC" to null, "respRateBpm" to null, "steps" to null, "activeKcalEst" to null, "spo2Red" to null, "spo2Ir" to null, "sleepPerformance" to 87.5, + "sleepConsistency" to 82.0, ), ) val batch = PushProtocol.mutableBatch( @@ -43,6 +44,7 @@ class PushProtocolTest { val lines = batch.body.toString(Charsets.UTF_8).trimEnd().lines() assertEquals("1.1", JSONObject(lines.first()).getString("protocolVersion")) assertEquals(87.5, JSONObject(lines[1]).getJSONObject("data").getDouble("sleepPerformance"), 0.0) + assertEquals(82.0, JSONObject(lines[1]).getJSONObject("data").getDouble("sleepConsistency"), 0.0) assertTrue(PushAck.parse(PushAck.fromBatch(batch).encode(), PushProtocol.LATEST_VERSION).exactlyMatches(batch)) var rejectedByV10 = false @@ -67,6 +69,18 @@ class PushProtocolTest { rejectedOutOfRangeScore = true } assertTrue("sleepPerformance must be bounded to 0..100", rejectedOutOfRangeScore) + + var rejectedOutOfRangeConsistency = false + try { + PushProtocol.mutableBatch( + PushMutableTable.DAILY_METRIC, SOURCE_A, "strap-noop", testWindow(), + listOf(score.copy(data = score.data + ("sleepConsistency" to 101.0))), + PushProtocol.LATEST_VERSION, + ) + } catch (_: PushProtocolException) { + rejectedOutOfRangeConsistency = true + } + assertTrue("sleepConsistency must be bounded to 0..100", rejectedOutOfRangeConsistency) } @Test diff --git a/docs/PUSH_PROTOCOL.md b/docs/PUSH_PROTOCOL.md index 9c30ff9419..bcf795bbba 100644 --- a/docs/PUSH_PROTOCOL.md +++ b/docs/PUSH_PROTOCOL.md @@ -2,8 +2,8 @@ This document specifies the wire contract for NOOP's **Experimental**, default-off export to a user-owned HTTP(S) endpoint. Protocol versions **1.0** and **1.1** cover the Android-first client. -Version 1.1 adds only the explicit nullable daily `sleepPerformance` projection; it does not add a -stream or export the general metric-series table. This is a one-way export protocol: the on-device +Version 1.1 adds explicit nullable daily `sleepPerformance` and `sleepConsistency` projections; it does +not add a stream or export the general metric-series table. This is a one-way export protocol: the on-device database is authoritative, the receiver acknowledges writes and may advertise only which fixed v1 streams it accepts. NOOP never reads health data, commands, URLs, field names, or other configuration back from the receiver. @@ -295,7 +295,7 @@ null. The database's `deviceId` is supplied by the header and `synced` is intent | `stream` | Key | Window selector | `data` members | |---|---|---|---| -| `dailyMetric` | `day` | `day` | `totalSleepMin`, `efficiency`, `deepMin`, `remMin`, `lightMin`, `disturbances`, `restingHr`, `avgHrv`, `recovery`, `strain`, `exerciseCount`, `spo2Pct`, `skinTempDevC`, `respRateBpm`, `steps`, `activeKcalEst`, `spo2Red`, `spo2Ir` (all nullable); v1.1 adds nullable `sleepPerformance` | +| `dailyMetric` | `day` | `day` | `totalSleepMin`, `efficiency`, `deepMin`, `remMin`, `lightMin`, `disturbances`, `restingHr`, `avgHrv`, `recovery`, `strain`, `exerciseCount`, `spo2Pct`, `skinTempDevC`, `respRateBpm`, `steps`, `activeKcalEst`, `spo2Red`, `spo2Ir` (all nullable); v1.1 adds nullable `sleepPerformance` and `sleepConsistency` | | `sleepSession` | `startTs` | `startTs` | `endTs`, `efficiency`, `restingHr`, `avgHrv`, `stagesJSON`, `userEdited`, `startTsAdjusted`, `motionJSON`, `sleepStateJSON`, `stagingSparse` | | `workout` | `startTs`, `sport` | `startTs` | `endTs`, `source`, `durationS`, `energyKcal`, `avgHr`, `maxHr`, `strain`, `distanceM`, `zonesJSON`, `notes`, `routePolyline`, `steps` | | `journal` | `day`, `question` | `day` | `answeredYes`, `notes`, `numericValue` | @@ -307,25 +307,35 @@ are integers; metric and measurement fields are finite numbers. See [DATA_MODEL. and `android/app/src/main/java/com/noop/data/Entities.kt` for the local meanings and units. The wire registry, not automatic reflection over either database, determines what is sent. -The `dailyMetric` field list in protocol 1.0 is unchanged. Protocol 1.1 adds only the nullable -`data.sleepPerformance` number (0–100), sourced from the matching `metricSeries` row whose key is -`sleep_performance` and whose source/day match the enclosing `dailyMetric` row. It is not mapped from -`recovery`. When there is no matching score point, the field is JSON `null`. - -The value is source-local: it may be an imported Sleep Performance value or NOOP's locally computed -approximate Rest composite. The latter is not WHOOP's proprietary score. The exporter does not merge -values across device/source scopes; receivers can distinguish scopes from the batch `deviceId` and -should label the score honestly. This field is never a recovery score and is not recomputed or inferred -by the exporter. - -A receiver that selects protocol 1.0 receives the original field set. It does not receive -`sleepPerformance`, preserving compatibility with existing receivers. +The `dailyMetric` field list in protocol 1.0 is unchanged. Protocol 1.1 adds two nullable 0–100 values: + +- `data.sleepPerformance` is sourced from the matching `metricSeries` row whose key is + `sleep_performance` and whose source/day match the enclosing `dailyMetric` row. It is not mapped from + `recovery`. When there is no matching score point, the field is JSON `null`. +- `data.sleepConsistency` uses the matching `metricSeries` row whose key is `sleep_consistency` when the + latest exported daily day has a point, matching SleepModel's imported-series preference. Otherwise + Android reproduces the Sleep screen's existing bedtime-onset consistency for the same source: for each + local wake day, it uses the trailing up-to-14 sleep sessions ordered by detected `startTs`, with an + edited onset (`startTsAdjusted`) taking precedence. After wrapping after-midnight bedtimes past noon, + population SD of bedtime minutes maps to `100 * (1 - SD / 120)`, clamped to 0–100; fewer than three + sessions yields JSON `null`. A day with multiple sessions uses its last session's point. It is not + mapped from `recovery` or `sleepPerformance`. + +Both values are source-local: Sleep Performance may be an imported value or NOOP's locally computed +approximate Rest composite; sleep consistency may be imported or the NOOP bedtime-regularity calculation +above. The Rest composite is not WHOOP's proprietary score. The exporter does not merge values across +device/source scopes; receivers can distinguish scopes from the batch `deviceId` and should label the +scores honestly. Neither field is a recovery score. + +A receiver that selects protocol 1.0 receives the original field set. It does not receive either new +field, preserving compatibility with existing receivers. Newer tables such as `ppgHrSample`, `stepSample`, `sleepStateSample`, raw waveform / IMU tables, and any future schema additions are not silently exported. The general `metricSeries` table is not -a stream: v1.1 projects only the single `sleep_performance` value that matches each exported -`dailyMetric` source/day. Other metric-series keys remain local. Adding a stream or optional `data` -member requires a documented registry update and protocol minor version. +a stream: v1.1 projects only the matching `sleep_performance` and `sleep_consistency` values for each +exported `dailyMetric` source/day (the latter falls back to the documented session calculation when the +latest exported day has no imported consistency point). Other metric-series keys remain local. Adding a +stream or optional `data` member requires a documented registry update and protocol minor version. ## Acceptance, errors, and retry idempotency @@ -392,7 +402,7 @@ opened; senders never optimistically emit a version the receiver did not select. - A minor version may add an optional header/data member or a registry stream. Receivers supporting the same major must ignore unknown object members. They may reject an unknown stream without rejecting batches for supported streams. Specifically, v1.1 adds only the nullable - `dailyMetric.data.sleepPerformance` field described above. + `dailyMetric.data.sleepPerformance` and `dailyMetric.data.sleepConsistency` fields described above. - A sender must not emit a new stream or field while claiming an older minor version. Removing or changing the meaning/type of a field, or changing a stream key or delivery mode, requires a new major version. @@ -408,7 +418,8 @@ endpoint, request diagnostics, set cadence, or otherwise control NOOP. The contract is platform-neutral. NOOP on iOS/macOS uses GRDB/SQLite with the same natural keys and logical v1 streams, but every implementation uses explicit registry projections rather than reflection or `SELECT *`. A sender may continue to negotiate protocol 1.0. A sender that advertises 1.1 must -project the matching `sleep_performance` metric-series point into `dailyMetric.data.sleepPerformance` -using the same source and day as the daily row; a missing point emits `null`. Other metric-series keys -and platform-only columns stay absent. Scheduling and credential storage are platform concerns and do -not change NDJSON, acknowledgement, idempotency, or replacement semantics. +project `sleepPerformance` from the matching `sleep_performance` metric-series point and `sleepConsistency` +from either the matching `sleep_consistency` metric-series point or the documented same-source sleep-session +calculation. Missing values emit `null`. Other metric-series keys and platform-only columns stay absent. +Scheduling and credential storage are platform concerns and do not change NDJSON, acknowledgement, +idempotency, or replacement semantics.