Skip to content

perf(db): read the newest N days without sorting all of history #611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DropTabl wants to merge 2 commits into
base: main
Choose a base branch
from
Open

perf(db): read the newest N days without sorting all of history #611

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e84537c
perf(db): read the newest N days without sorting all of history
DropTabl Oct 22, 2026
c7492ad
test: time the newest-N read against the query it replaced
DropTabl Oct 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions lib/data/db.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8776,11 +8776,21 @@ class LocalDb {
/// THIS rather than writing its own `MAX(algo_version)` — the pattern had
/// already reappeared at five readers and two views, and the ceiling has to
/// be on all of them or it is on none of them.
static const String _servedDayJoin =
'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id) m '
static const String _servedDayJoin = 'JOIN ($_servedVersions) m $_servedOn';
static const String _servedVersions =
'SELECT day_id, MAX(algo_version) AS v FROM day_result '
'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id';
static const String _servedOn =
'ON r.day_id = m.day_id AND r.algo_version = m.v';

/// [_servedDayJoin] for the newest N days only (bind N). A LIMIT on the
/// outer query comes after the join and its sort, so SQLite fetched and
/// sorted EVERY day's row, bundle and all, to return N: a 30-day read went
/// from ~60 ms to 250-600 ms between 30 and 730 days of history. Choosing
/// the days first, from the covering index, leaves N rows to fetch.
static const String _servedNewestDaysJoin =
'JOIN ($_servedVersions ORDER BY day_id DESC LIMIT ?) m $_servedOn';

/// The result row for one day_id at the highest version this build serves
/// (see [_servedAlgoCeiling]), with a normalized `date` alias for callers.
/// Null if absent.
Expand Down Expand Up @@ -8943,8 +8953,8 @@ class LocalDb {
// For each day_id pick MAX(algo_version), then join back for the full row.
final rows = await db.rawQuery(
'SELECT r.* FROM day_result r '
'$_servedDayJoin '
'ORDER BY r.day_id DESC LIMIT ?',
'$_servedNewestDaysJoin '
'ORDER BY r.day_id DESC',
[limit],
);
return [for (final r in rows) _withDate(r)];
Expand All @@ -8968,8 +8978,8 @@ class LocalDb {
'SELECT r.day_id, r.algo_version, r.computed_at, r.finalized, '
'r.skipped, r.partial, r.rhr, r.rmssd, r.readiness '
'FROM day_result r '
'$_servedDayJoin '
'ORDER BY r.day_id DESC LIMIT ?',
'$_servedNewestDaysJoin '
'ORDER BY r.day_id DESC',
[limit],
);
return [for (final r in rows) _withDate(r)];
Expand Down
265 changes: 183 additions & 82 deletions test/db_serve_version_and_reads_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,10 @@ void main() {
test('the sibling pins match the SHAs kAlgoVersion was derived against', () {
final deps =
(loadYaml(File('pubspec.yaml').readAsStringSync())
as Map)['dependencies'] as Map;
String ref(String pkg) => ((deps[pkg] as Map)['git'] as Map)['ref'] as String;
as Map)['dependencies']
as Map;
String ref(String pkg) =>
((deps[pkg] as Map)['git'] as Map)['ref'] as String;

const why =
'sibling repinned without visiting kAlgoVersion. bump it and say what '
Expand All @@ -62,8 +64,8 @@ void main() {
// comments promise this test catches, and it did not until it read this
// file. `resolved-ref` too: `ref` alone can name a moved branch.
final locked =
(loadYaml(File('pubspec.lock').readAsStringSync())
as Map)['packages'] as Map;
(loadYaml(File('pubspec.lock').readAsStringSync()) as Map)['packages']
as Map;
String lockRef(String pkg, String field) =>
((locked[pkg] as Map)['description'] as Map)[field] as String;

Expand All @@ -72,10 +74,16 @@ void main() {
'partial repin. Move all three together, or a build resolves a '
'sibling nobody reasoned about';
for (final field in const ['ref', 'resolved-ref']) {
expect(lockRef('openstrap_analytics', field), kAnalyticsPin,
reason: whyLock);
expect(lockRef('openstrap_protocol', field), kProtocolPin,
reason: whyLock);
expect(
lockRef('openstrap_analytics', field),
kAnalyticsPin,
reason: whyLock,
);
expect(
lockRef('openstrap_protocol', field),
kProtocolPin,
reason: whyLock,
);
}
});

Expand Down Expand Up @@ -137,26 +145,30 @@ void main() {
expect(await LocalDb.dayResultDayIdsDesc(), contains(day));
});

test('the future row is kept, not deleted — re-upgrading gets it back',
() async {
const day = '2026-08-10';
final db = await LocalDb.instance;
final all = await db.query(
'day_result',
where: 'day_id = ?',
whereArgs: [day],
);
expect(all, hasLength(2));
});
test(
'the future row is kept, not deleted — re-upgrading gets it back',
() async {
const day = '2026-08-10';
final db = await LocalDb.instance;
final all = await db.query(
'day_result',
where: 'day_id = ?',
whereArgs: [day],
);
expect(all, hasLength(2));
},
);

test('a day that ONLY has a future row is served as absent, not wrong',
() async {
const day = '2026-08-11';
await _put(day, kAlgoVersion + 3, readiness: 12);
expect(await LocalDb.dayResult(day), isNull);
expect(await LocalDb.availableDayIds(), isNot(contains(day)));
expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
});
test(
'a day that ONLY has a future row is served as absent, not wrong',
() async {
const day = '2026-08-11';
await _put(day, kAlgoVersion + 3, readiness: 12);
expect(await LocalDb.dayResult(day), isNull);
expect(await LocalDb.availableDayIds(), isNot(contains(day)));
expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
},
);

test('an OLDER version is still served — that is not the bug', () async {
const day = '2026-08-09';
Expand All @@ -168,68 +180,157 @@ void main() {
});

group('read-path hygiene', () {
test('the decoded counter index is gone and sessions.start_ts is indexed',
() async {
final db = await LocalDb.instance;
final idx = await db.rawQuery(
"SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
);
final names = idx.map((r) => r['name']).toSet();
// Never on a read path: every decoded read orders by (rec_ts, counter),
// which a single-column counter index cannot serve.
expect(names, isNot(contains('idx_decoded_onehz_counter')));
// The most-called uncovered query in the UI seam was a full scan + sort.
expect(names, contains('idx_sessions_start'));
});
test(
'the decoded counter index is gone and sessions.start_ts is indexed',
() async {
final db = await LocalDb.instance;
final idx = await db.rawQuery(
"SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
);
final names = idx.map((r) => r['name']).toSet();
// Never on a read path: every decoded read orders by (rec_ts, counter),
// which a single-column counter index cannot serve.
expect(names, isNot(contains('idx_decoded_onehz_counter')));
// The most-called uncovered query in the UI seam was a full scan + sort.
expect(names, contains('idx_sessions_start'));
},
);

test('decodedRecTsMaxByDay matches the strftime grouping it replaced',
() async {
final db = await LocalDb.instance;
await db.delete('decoded_onehz');
// Three local days' worth of edges, plus a second row per day so the MAX
// actually has to choose.
final days = <String>[];
var t = DateTime.now()
.subtract(const Duration(days: 3))
.millisecondsSinceEpoch ~/
1000;
for (var i = 0; i < 3; i++) {
final label = dayLabelOf(
DateTime.fromMillisecondsSinceEpoch(t * 1000, isUtc: true).toLocal(),
test(
'decodedRecTsMaxByDay matches the strftime grouping it replaced',
() async {
final db = await LocalDb.instance;
await db.delete('decoded_onehz');
// Three local days' worth of edges, plus a second row per day so the MAX
// actually has to choose.
final days = <String>[];
var t =
DateTime.now()
.subtract(const Duration(days: 3))
.millisecondsSinceEpoch ~/
1000;
for (var i = 0; i < 3; i++) {
final label = dayLabelOf(
DateTime.fromMillisecondsSinceEpoch(
t * 1000,
isUtc: true,
).toLocal(),
);
days.add(label);
final start = localDayStartSec(label)!;
final end = localDayEndSec(label)!;
for (final ts in [start + 60, end - 90]) {
await db.insert('decoded_onehz', {
// v47 key — see _createDecodedStore.
'ts_ms': ts * 1000,
'rec_ts': ts,
'counter': ts,
'hr': 60,
});
}
t = end + 60;
}

final got = await LocalDb.decodedRecTsMaxByDay();
// The old implementation, verbatim, as the oracle.
final oracle = await db.rawQuery(
"SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
);
days.add(label);
final start = localDayStartSec(label)!;
final end = localDayEndSec(label)!;
for (final ts in [start + 60, end - 90]) {
await db.insert('decoded_onehz', {
// v47 key — see _createDecodedStore.
'ts_ms': ts * 1000,
'rec_ts': ts,
'counter': ts,
'hr': 60,
});
final want = {
for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
};
expect(got, want);
expect(got.keys.toSet(), days.toSet());
},
);

test(
'decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
() async {
final db = await LocalDb.instance;
await db.delete('decoded_onehz');
expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
},
);

test('the newest-N reads stay flat as history grows', () async {
// recentDayResults joined EVERY day's full row (~100 KB of bundle) and
// sorted them all before its LIMIT, so reading the last 30 days took
// 57 ms at 30 days of history and 250-600 ms at 730 — on every
// refreshComputeFreshness, i.e. every derive and every Home load whose
// night is still building.
final db = await LocalDb.instance;
await db.delete('day_result');
final bundle = '{"pad":"${'x' * 100000}"}';
final today = DateTime.now();
Future<void> grow(int days) async {
final b = db.batch();
for (var i = 0; i < days; i++) {
final day = dayLabelOf(
DateTime(today.year, today.month, today.day - i),
);
for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) {
// Every third day stops at an older version; the rest also carry a
// row from a future build, which must never be served.
if (v == kAlgoVersion && i % 3 == 0) continue;
b.insert('day_result', {
'day_id': day,
'algo_version': v,
'payload_json': bundle,
'window_json': '{}',
'computed_at': 0,
});
}
}
t = end + 60;
await b.commit(noResult: true);
}

final got = await LocalDb.decodedRecTsMaxByDay();
// The old implementation, verbatim, as the oracle.
final oracle = await db.rawQuery(
"SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
Future<int> medianMs(Future<Object?> Function() f) async {
await f();
final us = <int>[];
for (var i = 0; i < 5; i++) {
final sw = Stopwatch()..start();
await f();
us.add(sw.elapsedMicroseconds);
}
us.sort();
return us[2] ~/ 1000;
}

// The query it replaced, verbatim, as the oracle for WHICH rows come back.
Future<List<Map<String, Object?>>> oracle(int limit) => db.rawQuery(
'SELECT r.* FROM day_result r '
'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m '
'ON r.day_id = m.day_id AND r.algo_version = m.v '
'ORDER BY r.day_id DESC LIMIT ?',
[limit],
);
final want = {
for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
};
expect(got, want);
expect(got.keys.toSet(), days.toSet());
});
List<String> key(List<Map<String, Object?>> rows) => [
for (final r in rows) '${r['day_id']}@${r['algo_version']}',
];

test('decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
() async {
final db = await LocalDb.instance;
await db.delete('decoded_onehz');
expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
await grow(600);
expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30)));
expect(
key(await LocalDb.recentDayResultsMeta(30)),
key(await oracle(30)),
);

// Against the old query, timed back to back on the same machine, so a
// slow or loaded runner slows both sides alike. On 600 days the old one
// sorts ~60 MB of bundles to return 30; measured ~10x slower.
final now = await medianMs(() => LocalDb.recentDayResults(30));
final before = await medianMs(() => oracle(30));
expect(
now * 3,
lessThanOrEqualTo(before),
reason:
'recentDayResults(30) took $now ms at 600 days of history, '
'the full-history sort it replaced $before ms',
);
await db.delete('day_result');
});
});
}