diff --git a/lib/data/db.dart b/lib/data/db.dart index ed954c51b..87ee0f995 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -8776,11 +8776,21 @@ class LocalDb { /// THIS rather than writing its own `MAX(algo_version)` — the pattern had /// already reappeared at five readers and two views, and the ceiling has to /// be on all of them or it is on none of them. - static const String _servedDayJoin = - 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' - 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id) m ' + static const String _servedDayJoin = 'JOIN ($_servedVersions) m $_servedOn'; + static const String _servedVersions = + 'SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id'; + static const String _servedOn = 'ON r.day_id = m.day_id AND r.algo_version = m.v'; + /// [_servedDayJoin] for the newest N days only (bind N). A LIMIT on the + /// outer query comes after the join and its sort, so SQLite fetched and + /// sorted EVERY day's row, bundle and all, to return N: a 30-day read went + /// from ~60 ms to 250-600 ms between 30 and 730 days of history. Choosing + /// the days first, from the covering index, leaves N rows to fetch. + static const String _servedNewestDaysJoin = + 'JOIN ($_servedVersions ORDER BY day_id DESC LIMIT ?) m $_servedOn'; + /// The result row for one day_id at the highest version this build serves /// (see [_servedAlgoCeiling]), with a normalized `date` alias for callers. /// Null if absent. @@ -8943,8 +8953,8 @@ class LocalDb { // For each day_id pick MAX(algo_version), then join back for the full row. final rows = await db.rawQuery( 'SELECT r.* FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; @@ -8968,8 +8978,8 @@ class LocalDb { 'SELECT r.day_id, r.algo_version, r.computed_at, r.finalized, ' 'r.skipped, r.partial, r.rhr, r.rmssd, r.readiness ' 'FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart index f160691c5..93e5baaa5 100644 --- a/test/db_serve_version_and_reads_test.dart +++ b/test/db_serve_version_and_reads_test.dart @@ -46,8 +46,10 @@ void main() { test('the sibling pins match the SHAs kAlgoVersion was derived against', () { final deps = (loadYaml(File('pubspec.yaml').readAsStringSync()) - as Map)['dependencies'] as Map; - String ref(String pkg) => ((deps[pkg] as Map)['git'] as Map)['ref'] as String; + as Map)['dependencies'] + as Map; + String ref(String pkg) => + ((deps[pkg] as Map)['git'] as Map)['ref'] as String; const why = 'sibling repinned without visiting kAlgoVersion. bump it and say what ' @@ -62,8 +64,8 @@ void main() { // comments promise this test catches, and it did not until it read this // file. `resolved-ref` too: `ref` alone can name a moved branch. final locked = - (loadYaml(File('pubspec.lock').readAsStringSync()) - as Map)['packages'] as Map; + (loadYaml(File('pubspec.lock').readAsStringSync()) as Map)['packages'] + as Map; String lockRef(String pkg, String field) => ((locked[pkg] as Map)['description'] as Map)[field] as String; @@ -72,10 +74,16 @@ void main() { 'partial repin. Move all three together, or a build resolves a ' 'sibling nobody reasoned about'; for (final field in const ['ref', 'resolved-ref']) { - expect(lockRef('openstrap_analytics', field), kAnalyticsPin, - reason: whyLock); - expect(lockRef('openstrap_protocol', field), kProtocolPin, - reason: whyLock); + expect( + lockRef('openstrap_analytics', field), + kAnalyticsPin, + reason: whyLock, + ); + expect( + lockRef('openstrap_protocol', field), + kProtocolPin, + reason: whyLock, + ); } }); @@ -137,26 +145,30 @@ void main() { expect(await LocalDb.dayResultDayIdsDesc(), contains(day)); }); - test('the future row is kept, not deleted — re-upgrading gets it back', - () async { - const day = '2026-08-10'; - final db = await LocalDb.instance; - final all = await db.query( - 'day_result', - where: 'day_id = ?', - whereArgs: [day], - ); - expect(all, hasLength(2)); - }); + test( + 'the future row is kept, not deleted — re-upgrading gets it back', + () async { + const day = '2026-08-10'; + final db = await LocalDb.instance; + final all = await db.query( + 'day_result', + where: 'day_id = ?', + whereArgs: [day], + ); + expect(all, hasLength(2)); + }, + ); - test('a day that ONLY has a future row is served as absent, not wrong', - () async { - const day = '2026-08-11'; - await _put(day, kAlgoVersion + 3, readiness: 12); - expect(await LocalDb.dayResult(day), isNull); - expect(await LocalDb.availableDayIds(), isNot(contains(day))); - expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day))); - }); + test( + 'a day that ONLY has a future row is served as absent, not wrong', + () async { + const day = '2026-08-11'; + await _put(day, kAlgoVersion + 3, readiness: 12); + expect(await LocalDb.dayResult(day), isNull); + expect(await LocalDb.availableDayIds(), isNot(contains(day))); + expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day))); + }, + ); test('an OLDER version is still served — that is not the bug', () async { const day = '2026-08-09'; @@ -168,68 +180,157 @@ void main() { }); group('read-path hygiene', () { - test('the decoded counter index is gone and sessions.start_ts is indexed', - () async { - final db = await LocalDb.instance; - final idx = await db.rawQuery( - "SELECT name, tbl_name FROM sqlite_master WHERE type='index'", - ); - final names = idx.map((r) => r['name']).toSet(); - // Never on a read path: every decoded read orders by (rec_ts, counter), - // which a single-column counter index cannot serve. - expect(names, isNot(contains('idx_decoded_onehz_counter'))); - // The most-called uncovered query in the UI seam was a full scan + sort. - expect(names, contains('idx_sessions_start')); - }); + test( + 'the decoded counter index is gone and sessions.start_ts is indexed', + () async { + final db = await LocalDb.instance; + final idx = await db.rawQuery( + "SELECT name, tbl_name FROM sqlite_master WHERE type='index'", + ); + final names = idx.map((r) => r['name']).toSet(); + // Never on a read path: every decoded read orders by (rec_ts, counter), + // which a single-column counter index cannot serve. + expect(names, isNot(contains('idx_decoded_onehz_counter'))); + // The most-called uncovered query in the UI seam was a full scan + sort. + expect(names, contains('idx_sessions_start')); + }, + ); - test('decodedRecTsMaxByDay matches the strftime grouping it replaced', - () async { - final db = await LocalDb.instance; - await db.delete('decoded_onehz'); - // Three local days' worth of edges, plus a second row per day so the MAX - // actually has to choose. - final days = []; - var t = DateTime.now() - .subtract(const Duration(days: 3)) - .millisecondsSinceEpoch ~/ - 1000; - for (var i = 0; i < 3; i++) { - final label = dayLabelOf( - DateTime.fromMillisecondsSinceEpoch(t * 1000, isUtc: true).toLocal(), + test( + 'decodedRecTsMaxByDay matches the strftime grouping it replaced', + () async { + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + // Three local days' worth of edges, plus a second row per day so the MAX + // actually has to choose. + final days = []; + var t = + DateTime.now() + .subtract(const Duration(days: 3)) + .millisecondsSinceEpoch ~/ + 1000; + for (var i = 0; i < 3; i++) { + final label = dayLabelOf( + DateTime.fromMillisecondsSinceEpoch( + t * 1000, + isUtc: true, + ).toLocal(), + ); + days.add(label); + final start = localDayStartSec(label)!; + final end = localDayEndSec(label)!; + for (final ts in [start + 60, end - 90]) { + await db.insert('decoded_onehz', { + // v47 key — see _createDecodedStore. + 'ts_ms': ts * 1000, + 'rec_ts': ts, + 'counter': ts, + 'hr': 60, + }); + } + t = end + 60; + } + + final got = await LocalDb.decodedRecTsMaxByDay(); + // The old implementation, verbatim, as the oracle. + final oracle = await db.rawQuery( + "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, " + 'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d', ); - days.add(label); - final start = localDayStartSec(label)!; - final end = localDayEndSec(label)!; - for (final ts in [start + 60, end - 90]) { - await db.insert('decoded_onehz', { - // v47 key — see _createDecodedStore. - 'ts_ms': ts * 1000, - 'rec_ts': ts, - 'counter': ts, - 'hr': 60, - }); + final want = { + for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(), + }; + expect(got, want); + expect(got.keys.toSet(), days.toSet()); + }, + ); + + test( + 'decodedRecTsMaxByDay on an empty store is empty, not a 1970 day', + () async { + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); + }, + ); + + test('the newest-N reads stay flat as history grows', () async { + // recentDayResults joined EVERY day's full row (~100 KB of bundle) and + // sorted them all before its LIMIT, so reading the last 30 days took + // 57 ms at 30 days of history and 250-600 ms at 730 — on every + // refreshComputeFreshness, i.e. every derive and every Home load whose + // night is still building. + final db = await LocalDb.instance; + await db.delete('day_result'); + final bundle = '{"pad":"${'x' * 100000}"}'; + final today = DateTime.now(); + Future grow(int days) async { + final b = db.batch(); + for (var i = 0; i < days; i++) { + final day = dayLabelOf( + DateTime(today.year, today.month, today.day - i), + ); + for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) { + // Every third day stops at an older version; the rest also carry a + // row from a future build, which must never be served. + if (v == kAlgoVersion && i % 3 == 0) continue; + b.insert('day_result', { + 'day_id': day, + 'algo_version': v, + 'payload_json': bundle, + 'window_json': '{}', + 'computed_at': 0, + }); + } } - t = end + 60; + await b.commit(noResult: true); } - final got = await LocalDb.decodedRecTsMaxByDay(); - // The old implementation, verbatim, as the oracle. - final oracle = await db.rawQuery( - "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, " - 'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d', + Future medianMs(Future Function() f) async { + await f(); + final us = []; + for (var i = 0; i < 5; i++) { + final sw = Stopwatch()..start(); + await f(); + us.add(sw.elapsedMicroseconds); + } + us.sort(); + return us[2] ~/ 1000; + } + + // The query it replaced, verbatim, as the oracle for WHICH rows come back. + Future>> oracle(int limit) => db.rawQuery( + 'SELECT r.* FROM day_result r ' + 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m ' + 'ON r.day_id = m.day_id AND r.algo_version = m.v ' + 'ORDER BY r.day_id DESC LIMIT ?', + [limit], ); - final want = { - for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(), - }; - expect(got, want); - expect(got.keys.toSet(), days.toSet()); - }); + List key(List> rows) => [ + for (final r in rows) '${r['day_id']}@${r['algo_version']}', + ]; - test('decodedRecTsMaxByDay on an empty store is empty, not a 1970 day', - () async { - final db = await LocalDb.instance; - await db.delete('decoded_onehz'); - expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); + await grow(600); + expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30))); + expect( + key(await LocalDb.recentDayResultsMeta(30)), + key(await oracle(30)), + ); + + // Against the old query, timed back to back on the same machine, so a + // slow or loaded runner slows both sides alike. On 600 days the old one + // sorts ~60 MB of bundles to return 30; measured ~10x slower. + final now = await medianMs(() => LocalDb.recentDayResults(30)); + final before = await medianMs(() => oracle(30)); + expect( + now * 3, + lessThanOrEqualTo(before), + reason: + 'recentDayResults(30) took $now ms at 600 days of history, ' + 'the full-history sort it replaced $before ms', + ); + await db.delete('day_result'); }); }); }