oura: exclude unwritten hypnogram tail from layout

Packages/OuraProtocol/Sources/OuraProtocol/HypnogramAssembler.swift

@@ -8,8 +8,9 @@ import Foundation
 // timestamp therefore marks WHEN THE ANALYSIS WAS SAVED, not when the sleep happened; anchoring each
 // record at its envelope collapses an entire night onto a few seconds.
 //
-// THE RECONSTRUCTION: the burst's codes are one contiguous sequence at 30 s/code, ENDING at the burst's
-// envelope time. Laying N codes backward from the anchored burst end recovers the real window. The
+// THE RECONSTRUCTION: the burst's written codes and interior gaps form a sequence at 30 s/code,
+// ENDING at the anchored burst end. Trailing unwritten flash pad has no elapsed time. Laying the
+// remaining codes backward from the end recovers the real window. The
 // 30 s epoch is triple-confirmed: open_oura's sleepnet.md ("DEEP/LIGHT/REM/WAKE classification at
 // 30-second intervals"), the observed window math (1,196 codes over a ~23:35-09:00 night), and the
 // 0x49 summary's 600-minute window over those same 1,196 codes (~30.1 s/code).
@@ -42,7 +43,8 @@ public struct OuraHypnogramBurst: Equatable, Sendable {
         zip(records, records.dropFirst()).contains { $1.ringTimestamp < $0.ringTimestamp }
     }
 
-    /// Lay the burst's codes out backward from `endUnixSeconds` at `secondsPerCode`. Code j of N gets
+    /// Lay the burst's codes out backward from `endUnixSeconds` at `secondsPerCode`. Trailing unwritten
+    /// codes are excluded from N; interior unwritten runs retain their time slots. Code j of N gets
     /// `ts = end - (N - j) * secondsPerCode` — i.e. each ts marks the START of that code's interval and
     /// the final code's interval ends exactly at the burst end. Order: records in arrival order, codes
     /// by their in-record index (the sequence is the ground truth; the spacing is the documented 30 s
@@ -56,21 +58,16 @@ public struct OuraHypnogramBurst: Equatable, Sendable {
     /// full unclamped lay is returned) so a mis-paired window can never empty the night. `nil` = no clip.
     public func codesWithTimes(endUnixSeconds: Int, sleepStartUnixSeconds: Int? = nil, secondsPerCode: Int = 30)
         -> [(phase: OuraSleepPhase, ts: Int)] {
-        let n = totalCodes
+        let phases = records.flatMap(\.phases)
+        let n = (phases.lastIndex { !$0.unwritten } ?? -1) + 1
         var out: [(phase: OuraSleepPhase, ts: Int)] = []
         out.reserveCapacity(n)
-        var j = 0
-        for record in records {
-            for phase in record.phases {
-                out.append((phase, endUnixSeconds - (n - j) * secondsPerCode))
-                j += 1
-            }
+        for (j, phase) in phases.prefix(n).enumerated() {
+            out.append((phase, endUnixSeconds - (n - j) * secondsPerCode))
         }
-        // #1246: drop UNWRITTEN epochs (whole-record `0xFF` erased pages) from the reconstructed hypnogram
-        // — they are a GAP, not `awake`. Done AFTER the lay so `n`/`j` (and thus every WRITTEN code's ts)
-        // are computed over the full sequence: an unwritten run anywhere — head, middle, or tail — leaves
-        // the real codes around it correctly timed. An all-unwritten burst yields [] (an honest no-stage
-        // night), which the caller drops rather than persisting a blank session.
+        // #1246: interior UNWRITTEN epochs occupy their slots, then leave a gap rather than false awake
+        // stages. #2564: trailing pad is removed before layout so the final written code ends at the
+        // anchor. An all-unwritten burst yields [] and is not persisted as a blank session.
         let written = out.filter { !$0.phase.unwritten }
         guard let start = sleepStartUnixSeconds else { return written }
         let clipped = written.filter { $0.ts >= start }

Packages/OuraProtocol/Tests/OuraProtocolTests/HypnogramAssemblerTests.swift

@@ -23,7 +23,7 @@ final class HypnogramAssemblerTests: XCTestCase {
         XCTAssertEqual(laid.last!.ts + 30, 10_000)
     }
 
-    // MARK: - #1246 unwritten (0xFF) epochs excluded, real codes stay timed
+    // MARK: - Unwritten (0xFF) gaps keep time; trailing padding does not
 
     func testUnwrittenEpochsDroppedButRealCodesKeepFullSequenceTimes_middleGap() {
         // A burst with an UNWRITTEN run in the MIDDLE. codesWithTimes lays times over the FULL count (so
@@ -46,6 +46,40 @@ final class HypnogramAssemblerTests: XCTestCase {
         XCTAssertEqual(laid.suffix(4).map { $0.ts }, [9_880, 9_910, 9_940, 9_970])
     }
 
+    func testTrailingPadDoesNotShiftWrittenCodesOrInteriorGap() {
+        let real1 = phases([.deep, .light, .rem, .awake], rt: 1000)
+        let gap = (0..<4).map { OuraSleepPhase(ringTimestamp: 1001, index: $0, stage: .awake, unwritten: true) }
+        let real2 = phases([.light, .deep, .rem, .light], rt: 1002)
+        let pad = (0..<4).map { OuraSleepPhase(ringTimestamp: 1003, index: $0, stage: .awake, unwritten: true) }
+        let burst = OuraHypnogramBurst(records: [
+            OuraHypnogramRecord(ringTimestamp: 1000, phases: real1),
+            OuraHypnogramRecord(ringTimestamp: 1001, phases: gap),
+            OuraHypnogramRecord(ringTimestamp: 1002, phases: real2),
+            OuraHypnogramRecord(ringTimestamp: 1003, phases: pad),
+        ])
+        let laid = burst.codesWithTimes(endUnixSeconds: 10_000)
+        XCTAssertEqual(laid.map { $0.ts }, [9_640, 9_670, 9_700, 9_730, 9_880, 9_910, 9_940, 9_970])
+        XCTAssertEqual(laid.last!.ts + 30, 10_000)
+    }
+
+    func testLaterPassWithTailPadRemainsMoreCompleteAfterOnsetClip() {
+        let onset = 20_360
+        let earlier = OuraHypnogramBurst(records: [OuraHypnogramRecord(
+            ringTimestamp: 1000, phases: phases(Array(repeating: .light, count: 988), rt: 1000)
+        )])
+        let pad = (0..<40).map { OuraSleepPhase(ringTimestamp: 1002, index: $0, stage: .awake, unwritten: true) }
+        let later = OuraHypnogramBurst(records: [
+            OuraHypnogramRecord(ringTimestamp: 1001, phases: phases(Array(repeating: .light, count: 1000), rt: 1001)),
+            OuraHypnogramRecord(ringTimestamp: 1002, phases: pad),
+        ])
+        let first = earlier.codesWithTimes(endUnixSeconds: 50_000, sleepStartUnixSeconds: onset)
+        let final = later.codesWithTimes(endUnixSeconds: 50_360, sleepStartUnixSeconds: onset)
+        XCTAssertEqual(first.count, 988)
+        XCTAssertEqual(final.count, 1000)
+        XCTAssertEqual(final.first!.ts, onset)
+        XCTAssertEqual(final.last!.ts + 30, 50_360)
+    }
+
     func testAllUnwrittenBurstReconstructsEmpty() {
         // Every page erased → no stageable sleep. Returns [] so the caller drops it (no blank/awake night).
         let gap = (0..<4).map { OuraSleepPhase(ringTimestamp: 1000, index: $0, stage: .awake, unwritten: true) }

android/app/src/main/java/com/noop/oura/HypnogramAssembler.kt

@@ -9,8 +9,9 @@ package com.noop.oura
 // timestamp therefore marks WHEN THE ANALYSIS WAS SAVED, not when the sleep happened; anchoring each
 // record at its envelope collapses an entire night onto a few seconds.
 //
-// THE RECONSTRUCTION: the burst's codes are one contiguous sequence at 30 s/code, ENDING at the
-// anchored burst end. Laying N codes backward from that end recovers the real window. The 30 s epoch
+// THE RECONSTRUCTION: the burst's written codes and interior gaps form a sequence at 30 s/code,
+// ENDING at the anchored burst end. Trailing unwritten flash pad has no elapsed time. Laying the
+// remaining codes backward recovers the real window. The 30 s epoch
 // is triple-confirmed: open_oura's sleepnet.md ("DEEP/LIGHT/REM/WAKE classification at 30-second
 // intervals"), the observed window math (1,196 codes over a ~10 h night), and the 0x49 summary's
 // window minutes over those same codes. The burst end itself is the matching 0x49 window's TRUE
@@ -47,7 +48,8 @@ data class OuraHypnogramBurst(val records: List<OuraHypnogramRecord>) {
         get() = records.zipWithNext().any { (a, b) -> b.ringTimestamp < a.ringTimestamp }
 
     /**
-     * Lay the burst's codes out backward from [endUnixSeconds] at [secondsPerCode]. Code j of N gets
+     * Lay the burst's codes out backward from [endUnixSeconds] at [secondsPerCode]. Trailing unwritten
+     * codes are excluded from N; interior unwritten runs retain their time slots. Code j of N gets
      * `ts = end - (N - j) * secondsPerCode` — i.e. each ts marks the START of that code's interval
      * and the final code's interval ends exactly at the burst end. Order: records in arrival order,
      * codes by their in-record index (the sequence is the ground truth; the spacing is the documented
@@ -64,20 +66,15 @@ data class OuraHypnogramBurst(val records: List<OuraHypnogramRecord>) {
         sleepStartUnixSeconds: Long? = null,
         secondsPerCode: Long = 30,
     ): List<OuraHypnogramCode> {
-        val n = totalCodes
+        val phases = records.flatMap { it.phases }
+        val n = phases.indexOfLast { !it.unwritten } + 1
         val out = ArrayList<OuraHypnogramCode>(n)
-        var j = 0
-        for (record in records) {
-            for (phase in record.phases) {
-                out.add(OuraHypnogramCode(phase, endUnixSeconds - (n - j) * secondsPerCode))
-                j += 1
-            }
+        for (j in 0 until n) {
+            out.add(OuraHypnogramCode(phases[j], endUnixSeconds - (n - j) * secondsPerCode))
         }
-        // #1246: drop UNWRITTEN epochs (whole-record 0xFF erased pages) from the reconstructed hypnogram —
-        // they are a GAP, not AWAKE. Done AFTER the lay so n/j (and thus every WRITTEN code's ts) are
-        // computed over the full sequence: an unwritten run anywhere — head, middle, or tail — leaves the
-        // real codes around it correctly timed. An all-unwritten burst yields [] (an honest no-stage night),
-        // which the caller drops rather than persisting a blank session. Byte-parity twin of Swift.
+        // #1246: interior UNWRITTEN epochs occupy their slots, then leave a gap rather than false awake
+        // stages. #2564: trailing pad is removed before layout so the final written code ends at the
+        // anchor. An all-unwritten burst yields [] and is not persisted as a blank session.
         val written = out.filter { !it.phase.unwritten }
         if (sleepStartUnixSeconds == null) return written
         val clipped = written.filter { it.ts >= sleepStartUnixSeconds }

android/app/src/test/java/com/noop/oura/HypnogramAssemblerTest.kt

@@ -33,7 +33,7 @@ class HypnogramAssemblerTest {
         assertEquals(10_000L, laid.last().ts + 30)
     }
 
-    // MARK: - #1246 unwritten (0xFF) epochs excluded, real codes stay timed. Twin of Swift.
+    // MARK: - Unwritten (0xFF) gaps keep time; trailing padding does not. Twin of Swift.
 
     @Test
     fun unwrittenEpochsDroppedButRealCodesKeepFullSequenceTimes_middleGap() {
@@ -57,6 +57,42 @@ class HypnogramAssemblerTest {
         assertEquals(listOf(9_880L, 9_910L, 9_940L, 9_970L), laid.takeLast(4).map { it.ts })
     }
 
+    @Test
+    fun trailingPadDoesNotShiftWrittenCodesOrInteriorGap() {
+        val real1 = phases(listOf(OuraSleepStage.DEEP, OuraSleepStage.LIGHT, OuraSleepStage.REM, OuraSleepStage.AWAKE), 1000)
+        val gap = (0 until 4).map { OuraSleepPhase(ringTimestamp = 1001, index = it, stage = OuraSleepStage.AWAKE, unwritten = true) }
+        val real2 = phases(listOf(OuraSleepStage.LIGHT, OuraSleepStage.DEEP, OuraSleepStage.REM, OuraSleepStage.LIGHT), 1002)
+        val pad = (0 until 4).map { OuraSleepPhase(ringTimestamp = 1003, index = it, stage = OuraSleepStage.AWAKE, unwritten = true) }
+        val burst = OuraHypnogramBurst(listOf(
+            OuraHypnogramRecord(1000, real1),
+            OuraHypnogramRecord(1001, gap),
+            OuraHypnogramRecord(1002, real2),
+            OuraHypnogramRecord(1003, pad),
+        ))
+        val laid = burst.codesWithTimes(endUnixSeconds = 10_000)
+        assertEquals(listOf(9_640L, 9_670L, 9_700L, 9_730L, 9_880L, 9_910L, 9_940L, 9_970L), laid.map { it.ts })
+        assertEquals(10_000L, laid.last().ts + 30)
+    }
+
+    @Test
+    fun laterPassWithTailPadRemainsMoreCompleteAfterOnsetClip() {
+        val onset = 20_360L
+        val earlier = OuraHypnogramBurst(listOf(OuraHypnogramRecord(
+            1000, phases(List(988) { OuraSleepStage.LIGHT }, 1000),
+        )))
+        val pad = (0 until 40).map { OuraSleepPhase(ringTimestamp = 1002, index = it, stage = OuraSleepStage.AWAKE, unwritten = true) }
+        val later = OuraHypnogramBurst(listOf(
+            OuraHypnogramRecord(1001, phases(List(1000) { OuraSleepStage.LIGHT }, 1001)),
+            OuraHypnogramRecord(1002, pad),
+        ))
+        val first = earlier.codesWithTimes(endUnixSeconds = 50_000, sleepStartUnixSeconds = onset)
+        val final = later.codesWithTimes(endUnixSeconds = 50_360, sleepStartUnixSeconds = onset)
+        assertEquals(988, first.size)
+        assertEquals(1000, final.size)
+        assertEquals(onset, final.first().ts)
+        assertEquals(50_360L, final.last().ts + 30)
+    }
+
     @Test
     fun allUnwrittenBurstReconstructsEmpty() {
         // Every page erased → no stageable sleep. Returns [] so the caller drops it (no blank/awake night).

docs/OURA_PROTOCOL.md

@@ -873,9 +873,10 @@ arrive as a BURST of `0x4E`/`0x5A` (and `0x4B`) records finalized after wake —
 envelope ring-times (the WRITE moment, seconds apart), so arrival order is the code sequence, not a
 sortable time. NOOP:
 1. **Groups** consecutive phase records into one burst (envelope-gap grouping).
-2. **Reconstructs the time axis** by laying the codes BACKWARD from the burst end at the 30 s SleepNet
-   epoch: code *j* of *N* gets `ts = end − (N − j)·30 s` (each ts marks the start of its 30 s interval,
-   the last interval ending exactly at the burst end).
+2. **Reconstructs the time axis** by discarding trailing unwritten (`0xFF`) pad, then laying the
+   remaining codes BACKWARD from the burst end at the 30 s SleepNet epoch: code *j* of *N* gets
+   `ts = end − (N − j)·30 s` (each ts marks the start of its 30 s interval, the last written interval
+   ending exactly at the burst end). Unwritten runs between real codes keep their time slots as gaps.
 3. **Refines BOTH ends with the paired `0x49` window** (closest envelope ring-time within 10 min): the
    END anchors to `event − end_offset·60 s` (the true sleep-end, ahead of the write moment) and the
    START clamps to `event − start_offset·60 s` (the onset), clipping the few pre-window codes. Both use