From e84537c9e1f196f72a45f118e5c6757d16c2348e Mon Sep 17 00:00:00 2001 From: Alex Wagner Date: Thu, 22 Oct 2026 12:12:19 +0200 Subject: [PATCH 1/2] perf(db): read the newest N days without sorting all of history recentDayResults applied its LIMIT after joining and sorting every day's served row, payload_json (~100 KB) included, so the 30-day read behind refreshComputeFreshness went from ~60 ms at 30 days of history to 250-600 ms at 730. That runs on every derive and on every Home load while a night is still building: the growth soak (#609) measured getToday at 22 -> 303 ms. Pick the newest N served versions from the covering index first, then fetch only those rows. The served-version subquery is factored out of _servedDayJoin, which expands to the same SQL as before, so the ceiling stays in one place. recentDayResultsMeta takes the same join. Co-Authored-By: Claude Opus 5.5 --- lib/data/db.dart | 24 +++++--- test/db_serve_version_and_reads_test.dart | 69 +++++++++++++++++++++++ 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index ed954c51b..87ee0f995 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -8776,11 +8776,21 @@ class LocalDb { /// THIS rather than writing its own `MAX(algo_version)` — the pattern had /// already reappeared at five readers and two views, and the ceiling has to /// be on all of them or it is on none of them. - static const String _servedDayJoin = - 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' - 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id) m ' + static const String _servedDayJoin = 'JOIN ($_servedVersions) m $_servedOn'; + static const String _servedVersions = + 'SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id'; + static const String _servedOn = 'ON r.day_id = m.day_id AND r.algo_version = m.v'; + /// [_servedDayJoin] for the newest N days only (bind N). A LIMIT on the + /// outer query comes after the join and its sort, so SQLite fetched and + /// sorted EVERY day's row, bundle and all, to return N: a 30-day read went + /// from ~60 ms to 250-600 ms between 30 and 730 days of history. Choosing + /// the days first, from the covering index, leaves N rows to fetch. + static const String _servedNewestDaysJoin = + 'JOIN ($_servedVersions ORDER BY day_id DESC LIMIT ?) m $_servedOn'; + /// The result row for one day_id at the highest version this build serves /// (see [_servedAlgoCeiling]), with a normalized `date` alias for callers. /// Null if absent. @@ -8943,8 +8953,8 @@ class LocalDb { // For each day_id pick MAX(algo_version), then join back for the full row. final rows = await db.rawQuery( 'SELECT r.* FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; @@ -8968,8 +8978,8 @@ class LocalDb { 'SELECT r.day_id, r.algo_version, r.computed_at, r.finalized, ' 'r.skipped, r.partial, r.rhr, r.rmssd, r.readiness ' 'FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart index f160691c5..9f6eb5d13 100644 --- a/test/db_serve_version_and_reads_test.dart +++ b/test/db_serve_version_and_reads_test.dart @@ -231,5 +231,74 @@ void main() { await db.delete('decoded_onehz'); expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); }); + + test('the newest-N reads stay flat as history grows', () async { + // recentDayResults joined EVERY day's full row (~100 KB of bundle) and + // sorted them all before its LIMIT, so reading the last 30 days took + // 57 ms at 30 days of history and 250-600 ms at 730 — on every + // refreshComputeFreshness, i.e. every derive and every Home load whose + // night is still building. + final db = await LocalDb.instance; + await db.delete('day_result'); + final bundle = '{"pad":"${'x' * 100000}"}'; + final today = DateTime.now(); + var have = 0; + Future grow(int days) async { + final b = db.batch(); + for (var i = have; i < days; i++) { + final day = dayLabelOf(DateTime(today.year, today.month, today.day - i)); + for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) { + // Every third day stops at an older version; the rest also carry a + // row from a future build, which must never be served. + if (v == kAlgoVersion && i % 3 == 0) continue; + b.insert('day_result', { + 'day_id': day, + 'algo_version': v, + 'payload_json': bundle, + 'window_json': '{}', + 'computed_at': 0, + }); + } + } + await b.commit(noResult: true); + have = days; + } + + Future medianMs(Future Function() f) async { + await f(); + final us = []; + for (var i = 0; i < 5; i++) { + final sw = Stopwatch()..start(); + await f(); + us.add(sw.elapsedMicroseconds); + } + us.sort(); + return us[2] ~/ 1000; + } + + // The query it replaced, verbatim, as the oracle for WHICH rows come back. + Future>> oracle(int limit) => db.rawQuery( + 'SELECT r.* FROM day_result r ' + 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m ' + 'ON r.day_id = m.day_id AND r.algo_version = m.v ' + 'ORDER BY r.day_id DESC LIMIT ?', + [limit], + ); + List key(List> rows) => + [for (final r in rows) '${r['day_id']}@${r['algo_version']}']; + + await grow(30); + final short = await medianMs(() => LocalDb.recentDayResults(30)); + await grow(600); + final long = await medianMs(() => LocalDb.recentDayResults(30)); + + expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30))); + expect(key(await LocalDb.recentDayResultsMeta(30)), key(await oracle(30))); + expect(long, lessThanOrEqualTo(short * 3 + 30), + reason: 'recentDayResults(30): $short ms at 30 days of history, ' + '$long ms at 600'); + await db.delete('day_result'); + }); }); } From c7492adcb1f485040b6798cdb00969110beaf618 Mon Sep 17 00:00:00 2001 From: Alex Wagner Date: Thu, 22 Oct 2026 12:30:27 +0200 Subject: [PATCH 2/2] test: time the newest-N read against the query it replaced An absolute bound (600-day read <= 3x the 30-day read + 30 ms) can fail on a slow or loaded runner. Time the old full-history query back to back on the same 600 days instead, so load slows both sides alike: 3x faster or better (measured ~10x with the change, 1x without). Co-Authored-By: Claude Opus 5.5 --- test/db_serve_version_and_reads_test.dart | 246 ++++++++++++---------- 1 file changed, 139 insertions(+), 107 deletions(-) diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart index 9f6eb5d13..93e5baaa5 100644 --- a/test/db_serve_version_and_reads_test.dart +++ b/test/db_serve_version_and_reads_test.dart @@ -46,8 +46,10 @@ void main() { test('the sibling pins match the SHAs kAlgoVersion was derived against', () { final deps = (loadYaml(File('pubspec.yaml').readAsStringSync()) - as Map)['dependencies'] as Map; - String ref(String pkg) => ((deps[pkg] as Map)['git'] as Map)['ref'] as String; + as Map)['dependencies'] + as Map; + String ref(String pkg) => + ((deps[pkg] as Map)['git'] as Map)['ref'] as String; const why = 'sibling repinned without visiting kAlgoVersion. bump it and say what ' @@ -62,8 +64,8 @@ void main() { // comments promise this test catches, and it did not until it read this // file. `resolved-ref` too: `ref` alone can name a moved branch. final locked = - (loadYaml(File('pubspec.lock').readAsStringSync()) - as Map)['packages'] as Map; + (loadYaml(File('pubspec.lock').readAsStringSync()) as Map)['packages'] + as Map; String lockRef(String pkg, String field) => ((locked[pkg] as Map)['description'] as Map)[field] as String; @@ -72,10 +74,16 @@ void main() { 'partial repin. Move all three together, or a build resolves a ' 'sibling nobody reasoned about'; for (final field in const ['ref', 'resolved-ref']) { - expect(lockRef('openstrap_analytics', field), kAnalyticsPin, - reason: whyLock); - expect(lockRef('openstrap_protocol', field), kProtocolPin, - reason: whyLock); + expect( + lockRef('openstrap_analytics', field), + kAnalyticsPin, + reason: whyLock, + ); + expect( + lockRef('openstrap_protocol', field), + kProtocolPin, + reason: whyLock, + ); } }); @@ -137,26 +145,30 @@ void main() { expect(await LocalDb.dayResultDayIdsDesc(), contains(day)); }); - test('the future row is kept, not deleted — re-upgrading gets it back', - () async { - const day = '2026-08-10'; - final db = await LocalDb.instance; - final all = await db.query( - 'day_result', - where: 'day_id = ?', - whereArgs: [day], - ); - expect(all, hasLength(2)); - }); + test( + 'the future row is kept, not deleted — re-upgrading gets it back', + () async { + const day = '2026-08-10'; + final db = await LocalDb.instance; + final all = await db.query( + 'day_result', + where: 'day_id = ?', + whereArgs: [day], + ); + expect(all, hasLength(2)); + }, + ); - test('a day that ONLY has a future row is served as absent, not wrong', - () async { - const day = '2026-08-11'; - await _put(day, kAlgoVersion + 3, readiness: 12); - expect(await LocalDb.dayResult(day), isNull); - expect(await LocalDb.availableDayIds(), isNot(contains(day))); - expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day))); - }); + test( + 'a day that ONLY has a future row is served as absent, not wrong', + () async { + const day = '2026-08-11'; + await _put(day, kAlgoVersion + 3, readiness: 12); + expect(await LocalDb.dayResult(day), isNull); + expect(await LocalDb.availableDayIds(), isNot(contains(day))); + expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day))); + }, + ); test('an OLDER version is still served — that is not the bug', () async { const day = '2026-08-09'; @@ -168,69 +180,79 @@ void main() { }); group('read-path hygiene', () { - test('the decoded counter index is gone and sessions.start_ts is indexed', - () async { - final db = await LocalDb.instance; - final idx = await db.rawQuery( - "SELECT name, tbl_name FROM sqlite_master WHERE type='index'", - ); - final names = idx.map((r) => r['name']).toSet(); - // Never on a read path: every decoded read orders by (rec_ts, counter), - // which a single-column counter index cannot serve. - expect(names, isNot(contains('idx_decoded_onehz_counter'))); - // The most-called uncovered query in the UI seam was a full scan + sort. - expect(names, contains('idx_sessions_start')); - }); - - test('decodedRecTsMaxByDay matches the strftime grouping it replaced', - () async { - final db = await LocalDb.instance; - await db.delete('decoded_onehz'); - // Three local days' worth of edges, plus a second row per day so the MAX - // actually has to choose. - final days = []; - var t = DateTime.now() - .subtract(const Duration(days: 3)) - .millisecondsSinceEpoch ~/ - 1000; - for (var i = 0; i < 3; i++) { - final label = dayLabelOf( - DateTime.fromMillisecondsSinceEpoch(t * 1000, isUtc: true).toLocal(), + test( + 'the decoded counter index is gone and sessions.start_ts is indexed', + () async { + final db = await LocalDb.instance; + final idx = await db.rawQuery( + "SELECT name, tbl_name FROM sqlite_master WHERE type='index'", ); - days.add(label); - final start = localDayStartSec(label)!; - final end = localDayEndSec(label)!; - for (final ts in [start + 60, end - 90]) { - await db.insert('decoded_onehz', { - // v47 key — see _createDecodedStore. - 'ts_ms': ts * 1000, - 'rec_ts': ts, - 'counter': ts, - 'hr': 60, - }); + final names = idx.map((r) => r['name']).toSet(); + // Never on a read path: every decoded read orders by (rec_ts, counter), + // which a single-column counter index cannot serve. + expect(names, isNot(contains('idx_decoded_onehz_counter'))); + // The most-called uncovered query in the UI seam was a full scan + sort. + expect(names, contains('idx_sessions_start')); + }, + ); + + test( + 'decodedRecTsMaxByDay matches the strftime grouping it replaced', + () async { + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + // Three local days' worth of edges, plus a second row per day so the MAX + // actually has to choose. + final days = []; + var t = + DateTime.now() + .subtract(const Duration(days: 3)) + .millisecondsSinceEpoch ~/ + 1000; + for (var i = 0; i < 3; i++) { + final label = dayLabelOf( + DateTime.fromMillisecondsSinceEpoch( + t * 1000, + isUtc: true, + ).toLocal(), + ); + days.add(label); + final start = localDayStartSec(label)!; + final end = localDayEndSec(label)!; + for (final ts in [start + 60, end - 90]) { + await db.insert('decoded_onehz', { + // v47 key — see _createDecodedStore. + 'ts_ms': ts * 1000, + 'rec_ts': ts, + 'counter': ts, + 'hr': 60, + }); + } + t = end + 60; } - t = end + 60; - } - final got = await LocalDb.decodedRecTsMaxByDay(); - // The old implementation, verbatim, as the oracle. - final oracle = await db.rawQuery( - "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, " - 'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d', - ); - final want = { - for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(), - }; - expect(got, want); - expect(got.keys.toSet(), days.toSet()); - }); + final got = await LocalDb.decodedRecTsMaxByDay(); + // The old implementation, verbatim, as the oracle. + final oracle = await db.rawQuery( + "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, " + 'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d', + ); + final want = { + for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(), + }; + expect(got, want); + expect(got.keys.toSet(), days.toSet()); + }, + ); - test('decodedRecTsMaxByDay on an empty store is empty, not a 1970 day', - () async { - final db = await LocalDb.instance; - await db.delete('decoded_onehz'); - expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); - }); + test( + 'decodedRecTsMaxByDay on an empty store is empty, not a 1970 day', + () async { + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); + }, + ); test('the newest-N reads stay flat as history grows', () async { // recentDayResults joined EVERY day's full row (~100 KB of bundle) and @@ -242,11 +264,12 @@ void main() { await db.delete('day_result'); final bundle = '{"pad":"${'x' * 100000}"}'; final today = DateTime.now(); - var have = 0; Future grow(int days) async { final b = db.batch(); - for (var i = have; i < days; i++) { - final day = dayLabelOf(DateTime(today.year, today.month, today.day - i)); + for (var i = 0; i < days; i++) { + final day = dayLabelOf( + DateTime(today.year, today.month, today.day - i), + ); for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) { // Every third day stops at an older version; the rest also carry a // row from a future build, which must never be served. @@ -261,7 +284,6 @@ void main() { } } await b.commit(noResult: true); - have = days; } Future medianMs(Future Function() f) async { @@ -278,26 +300,36 @@ void main() { // The query it replaced, verbatim, as the oracle for WHICH rows come back. Future>> oracle(int limit) => db.rawQuery( - 'SELECT r.* FROM day_result r ' - 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' - 'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m ' - 'ON r.day_id = m.day_id AND r.algo_version = m.v ' - 'ORDER BY r.day_id DESC LIMIT ?', - [limit], - ); - List key(List> rows) => - [for (final r in rows) '${r['day_id']}@${r['algo_version']}']; + 'SELECT r.* FROM day_result r ' + 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m ' + 'ON r.day_id = m.day_id AND r.algo_version = m.v ' + 'ORDER BY r.day_id DESC LIMIT ?', + [limit], + ); + List key(List> rows) => [ + for (final r in rows) '${r['day_id']}@${r['algo_version']}', + ]; - await grow(30); - final short = await medianMs(() => LocalDb.recentDayResults(30)); await grow(600); - final long = await medianMs(() => LocalDb.recentDayResults(30)); - expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30))); - expect(key(await LocalDb.recentDayResultsMeta(30)), key(await oracle(30))); - expect(long, lessThanOrEqualTo(short * 3 + 30), - reason: 'recentDayResults(30): $short ms at 30 days of history, ' - '$long ms at 600'); + expect( + key(await LocalDb.recentDayResultsMeta(30)), + key(await oracle(30)), + ); + + // Against the old query, timed back to back on the same machine, so a + // slow or loaded runner slows both sides alike. On 600 days the old one + // sorts ~60 MB of bundles to return 30; measured ~10x slower. + final now = await medianMs(() => LocalDb.recentDayResults(30)); + final before = await medianMs(() => oracle(30)); + expect( + now * 3, + lessThanOrEqualTo(before), + reason: + 'recentDayResults(30) took $now ms at 600 days of history, ' + 'the full-history sort it replaced $before ms', + ); await db.delete('day_result'); }); });