diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 777bd7056..925416069 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1958,8 +1958,10 @@ const String kProtocolPin = 'f04931ba7a06d0a20dc9e5e8bd750e14fb0a9510'; /// Raw is kept this many days past derivation, then pruned (derived stays). const int rawRetentionDays = 3; -/// A day stays recomputable for this long after its wake, then FINALIZES (locks) -/// — more flash may still drain within this buffer (ARCHITECTURE_V2: ~48 h). +/// A day stays recomputable until the data edge is this far past the day's +/// LAST record (`endSec`, ≈ the end of the calendar day), then FINALIZES +/// (locks). So a full day locks ~72 h after it starts. More flash may still +/// drain within this buffer. const int _finalizationSec = 48 * 3600; /// How many trailing derived days feed readiness/composite baselines. @@ -2815,7 +2817,7 @@ class DerivationEngine { _diag['stage'] = 'notifications'; await _runNotifications(); } - // 5. Prune raw — never for a day still inside its raw window / un-derived. + // 5. Prune raw — never for a day still inside its raw window / not finalized. // Runs on EVERY derive, not just a full restage: `rawRetentionDays` is // the only cap on the 1 Hz substrate, and behind `scope.fullHistory` it // fired only on a manual "Re-analyze data", so an ordinary install grew @@ -4802,7 +4804,7 @@ class DerivationEngine { TelemetryService.instance.recordNonFatal(e, st, reason: 'day_blocks_failed'); } - // Finalize once the DATA EDGE has moved >48 h past the day's wake — i.e. we + // Finalize once the DATA EDGE has moved >48 h past the day's last record — we // have continuous drained data well beyond it, so no more flash can land for // this day. (Anchored on the last record ts, NOT the wall clock.) Imports // force-finalize: there is no stored raw to ever recompute them from, so @@ -5824,15 +5826,14 @@ class DerivationEngine { // ── raw pruning (raw-first invariant) ────────────────────────────────────── /// Longest the raw-first hold below may keep substrate past - /// [rawRetentionDays] for a day that still hasn't produced a complete result. + /// [rawRetentionDays] for a day that still hasn't finalized. /// /// The hold has to be bounded or it is not a hold, it is an off switch. A /// `partial` day is DELIBERATELY never finalized by age (see /// `_derivePreparedDay` — a headline-only row must keep its chance to be - /// filled in), and `dayResultIds` excludes both `partial` and `skipped`, so a - /// day whose second half fails every single time never becomes "derived" and - /// never becomes finalized either. One such day used to latch pruning off for - /// the whole install, forever, at ~12 MB/day. + /// filled in), and a transient skip never finalizes, so a day whose second + /// half fails every single time never becomes finalized. One such day used + /// to latch pruning off for the whole install, forever, at ~12 MB/day. static const int _maxRawHoldDays = 14; /// Cursor holding the highest `rec_ts` cutoff the raw prune has applied. @@ -5845,18 +5846,22 @@ class DerivationEngine { static int? rawPruneCutoffSec({ required int dataNowSec, required List rawDayIds, - required Set derivedDayIds, + required Set finalizedDayIds, }) { final cutoffSec = dataNowSec - rawRetentionDays * 86400; if (cutoffSec <= 0) return null; - final pending = rawDayIds.where((d) => !derivedDayIds.contains(d)).toList() - ..sort(); + // Pending = not FINALIZED, not merely "has a result": an unlocked day is + // still re-derived, and a re-derive from a cut night overwrites the good + // row and then locks it. + final pending = + rawDayIds.where((d) => !finalizedDayIds.contains(d)).toList()..sort(); if (pending.isEmpty) return _localDayStartOf(cutoffSec); - // Hold at the START of the oldest day still owed a result — its own rows - // survive, everything before it goes — floored so a permanently stuck day - // cannot hold the whole install (see [_maxRawHoldDays]). + // Hold at the start of the oldest pending day's derive window (the + // previous noon, where its night search begins, see [_targetDayWindow]), + // floored so a permanently stuck day cannot hold the whole install (see + // [_maxRawHoldDays]). final barrier = math.max( - _localDayLabelToSec(pending.first), + _localDayLabelToSec(pending.first) - kNocturnalSearchLookbackSec, dataNowSec - _maxRawHoldDays * 86400, ); return _localDayStartOf(barrier < cutoffSec ? barrier : cutoffSec); @@ -5878,8 +5883,8 @@ class DerivationEngine { /// backfill received in one sync must not be pruned just because it landed /// "now". /// - /// RAW-FIRST, DAY-SCOPED. A day in [rawDayIds] with no complete result at the - /// current algo version pulls the cutoff back to ITS OWN start instead of + /// RAW-FIRST, DAY-SCOPED. A day in [rawDayIds] not yet finalized at the + /// current algo version pulls the cutoff back to its derive window instead of /// aborting the whole prune — the old all-or-nothing guard meant a single /// stuck day kept every older day's substrate too. Bounded by /// [_maxRawHoldDays] so a permanently-stuck day cannot wedge it. @@ -5903,11 +5908,10 @@ class DerivationEngine { /// why a workout's average HR is written into the bundle (it was once /// re-derived lazily and vanished the moment its substrate aged out). Future _pruneOldDecoded(List rawDayIds, int dataNowSec) async { - final derivedIds = await LocalDb.dayResultIds(kAlgoVersion); final cutoffSec = rawPruneCutoffSec( dataNowSec: dataNowSec, rawDayIds: rawDayIds, - derivedDayIds: derivedIds, + finalizedDayIds: await LocalDb.finalizedDayIds(kAlgoVersion), ); if (cutoffSec == null) return; // Highest cutoff ever applied: a held-back pass can cut lower, but rows diff --git a/test/raw_prune_gating_test.dart b/test/raw_prune_gating_test.dart index 66471cae6..d742b4f5a 100644 --- a/test/raw_prune_gating_test.dart +++ b/test/raw_prune_gating_test.dart @@ -29,14 +29,14 @@ int _midnightOf(int sec) { void main() { group('rawPruneCutoffSec', () { - // A settled install: everything with raw is derived, so the plain + // A settled install: everything with raw is finalized, so the plain // retention window applies. - test('prunes at the retention edge when every raw day is derived', () { + test('prunes at the retention edge when every raw day is finalized', () { const dataNow = 1780000000; final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, rawDayIds: const ['2026-05-01', '2026-05-02', '2026-05-03'], - derivedDayIds: const {'2026-05-01', '2026-05-02', '2026-05-03'}, + finalizedDayIds: const {'2026-05-01', '2026-05-02', '2026-05-03'}, ); expect(cutoff, _midnightOf(dataNow - rawRetentionDays * 86400)); }); @@ -49,37 +49,56 @@ void main() { final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, rawDayIds: const ['2026-05-16', '2026-05-17', '2026-05-20'], - derivedDayIds: const {'2026-05-16', '2026-05-17', '2026-05-20'}, + finalizedDayIds: const {'2026-05-16', '2026-05-17', '2026-05-20'}, ); - // 05-17 09:47 is the raw retention edge; 05-17 survives whole. - expect(cutoff, _dayStart('2026-05-17')); + // The retention edge falls mid-morning; that day survives whole. + expect(cutoff, _midnightOf(dataNow - rawRetentionDays * 86400)); + expect(cutoff, isNot(dataNow - rawRetentionDays * 86400)); }); - test('an un-derived day holds the cutoff at ITS OWN start, not off', () { - // Data edge well past the un-derived day, so the plain cutoff would + test('an unfinalized day holds the cutoff, not off', () { + // Data edge well past the unfinalized day, so the plain cutoff would // otherwise delete it. final dataNow = _dayStart('2026-05-20') + 12 * 3600; final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, rawDayIds: const ['2026-05-12', '2026-05-13', '2026-05-19'], - derivedDayIds: const {'2026-05-12', '2026-05-19'}, + finalizedDayIds: const {'2026-05-12', '2026-05-19'}, ); - // Held at 05-13 — that day's own rows survive… - expect(cutoff, _dayStart('2026-05-13')); - // …but 05-12, which IS derived, is still reclaimed. The old guard - // returned early and kept it too. - expect(cutoff, greaterThan(_dayStart('2026-05-12'))); + // Held at 05-12 (05-13's night search starts 05-12 noon)… + expect(cutoff, _dayStart('2026-05-12')); + // …but older finalized days are still reclaimed. The old guard + // returned early and kept them too. + expect(cutoff, greaterThan(_dayStart('2026-05-11'))); + }); + + // Day D's night starts the evening before. A day with a result that has + // not LOCKED yet is still re-derived; cutting D-1's evening first made + // that re-derive score a half night, overwrite the good row, then lock. + test('keeps the evening before an unfinalized day with a result', () { + // Past the plain retention edge for 05-16 and 05-17. + final dataNow = _dayStart('2026-05-16') + + (rawRetentionDays + 1) * 86400 + + 21 * 3600; + final cutoff = DerivationEngine.rawPruneCutoffSec( + dataNowSec: dataNow, + rawDayIds: const ['2026-05-14', '2026-05-15', '2026-05-16'], + // 05-16 has a complete row but never got locked. + finalizedDayIds: const {'2026-05-14', '2026-05-15'}, + )!; + final eveningBefore = _dayStart('2026-05-16') - 4 * 3600; + expect(cutoff, lessThanOrEqualTo(eveningBefore)); + expect(cutoff, _dayStart('2026-05-15')); }); test('the hold is bounded — a permanently stuck day cannot wedge it', () { - // A day that never completes: `partial` rows are excluded from - // `dayResultIds` AND are never finalized by age, so this state is - // reachable and used to be permanent. + // A day that never completes: `partial` rows are never finalized by + // age, so this state is reachable and used to be permanent. final dataNow = _dayStart('2026-06-30'); final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, rawDayIds: const ['2026-01-05', '2026-06-29'], - derivedDayIds: const {'2026-06-29'}, + finalizedDayIds: const {'2026-06-29'}, )!; expect(cutoff, greaterThan(_dayStart('2026-01-05'))); // Never keeps more than the documented ceiling behind the data edge. @@ -91,9 +110,9 @@ void main() { final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, rawDayIds: const ['2026-05-19'], - derivedDayIds: const {}, + finalizedDayIds: const {}, ); - // The un-derived day is newer than the retention edge, so the edge wins. + // The unfinalized day is newer than the retention edge, so the edge wins. expect(cutoff, _midnightOf(dataNow - rawRetentionDays * 86400)); }); @@ -102,7 +121,7 @@ void main() { DerivationEngine.rawPruneCutoffSec( dataNowSec: 0, rawDayIds: const [], - derivedDayIds: const {}, + finalizedDayIds: const {}, ), isNull, ); @@ -198,5 +217,18 @@ void main() { expect(lines[i], contains('scope.rawDays'), reason: 'line ${i + 1}'); } }); + + test('the prune holds for FINALIZED days, not merely derived ones', () { + // A day with a complete but unfinalized result still re-derives; if a + // derived row were enough to release the hold, D-1's evening would be + // cut and D would re-derive from half a night, then lock. + final start = + lines.indexWhere((l) => l.contains('Future _pruneOldDecoded(')); + expect(start, isNot(-1), reason: 'the prune declaration vanished'); + final end = lines.indexWhere((l) => l.startsWith(' }'), start); + final body = lines.sublist(start, end).join('\n'); + expect(body, contains('LocalDb.finalizedDayIds(')); + expect(body, isNot(contains('dayResultIds'))); + }); }); }