diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..24eb8ee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Install pnpm + uses: pnpm/action-setup@f40ffcd9367d9f12939873eb1018b921a783ffaa # v4 + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + cache: pnpm + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Install Chromium + run: pnpm exec playwright install --with-deps chromium + - name: Verify formatting + run: pnpm format:check + - name: Type-check + run: pnpm check + - name: Build package + run: pnpm build + - name: Validate benchmark publisher + run: pnpm test:benchmark-publisher + - name: Run browser tests + run: pnpm test:browser diff --git a/.gitignore b/.gitignore index 014f474..640ccb6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ node_modules dist playwright-report test-results +benchmark-results .DS_Store *.tgz diff --git a/.prettierignore b/.prettierignore index 7fbd026..450452e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,3 +4,4 @@ playwright-report test-results coverage *.tgz +src/vendor/WriteAhead.js diff --git a/README.md b/README.md index 3219cc7..b125bbc 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ It provides one shared, named, in-memory SQLite runtime that can be structured-c Originally developed for internal use at Beeldi; this standalone package contains only the generic VFS. +[Reproducible PowerSync MemoryVFS comparison benchmark](https://github.com/whygee-dev/memory-write-ahead-vfs/blob/v0.2.0/benchmarks/memory-vfs-comparison/README.md) + ## Motivation wa-sqlite's in-memory VFS examples store their bytes in one JavaScript context: a database created in one worker is invisible to every other worker. The common workarounds are to proxy every statement through a single owner worker over `postMessage` (serializing each query and result, and funneling all work through one thread) or to switch to persistent storage such as OPFS or IndexedDB (paying real file I/O and exclusive access-handle coordination for data that never needed to outlive the page). @@ -32,7 +34,7 @@ On engines without growable `SharedArrayBuffer`, `allowFixedCapacityFallback: tr - Cross-origin isolation for pages and module workers that use the runtime: `Cross-Origin-Opener-Policy: same-origin` and `Cross-Origin-Embedder-Policy: require-corp`. - Dedicated workers for blocking lock contention. Opening on the main thread is useful for simple flows, but a contended file lock on the main thread throws instead of blocking. -- wa-sqlite `>=1.7.0`. +- wa-sqlite `>=1.7.0 <1.8.0`. This VFS is ephemeral. It does not persist data across page reloads, tab closes, crashes, or process restarts. If growable `SharedArrayBuffer` is unavailable, `allowFixedCapacityFallback: true` uses fixed buffers allocated at each file's max capacity. @@ -128,6 +130,36 @@ Your development and production servers must send the COOP/COEP headers above fo File locks are cooperative and heartbeat-based. During long writes and truncates the VFS refreshes the lock heartbeat; if a holder makes no progress for longer than `fileLockStaleMs`, a waiting worker may recover the lock. Tune `fileLockStaleMs` above your worst expected GC pause, tab throttling delay, and write workload duration. +When an application deliberately terminates a worker, it can recover that worker's runtime handles immediately instead of waiting for the stale-heartbeat timeout. Have the worker report `vfs.getRuntimeOwnerId()` to its parent, terminate the worker, then call `recoverTerminatedMemoryWriteAheadRuntimeOwner(runtime, ownerId)` from the parent. If the worker owned an attached-database transaction, the VFS quarantines that database instead of releasing its lock: its rollback journal was local to the terminated context, so exposing the shared pages would be unsafe. Reset the runtime and reload the database from a trusted source before using it again. + +## Attached databases + +Register each attached database in the shared runtime before executing `ATTACH DATABASE`. While the owning context remains live, atomic transactions spanning the primary and attached databases are coordinated through SQLite's super-journal path. Transactions opened directly on an attached database are serialized across connections because attached databases use SQLite's rollback-journal protocol rather than the primary database's write-ahead path. + +```ts +import { ensureMemoryWriteAheadDatabaseFiles } from 'memory-write-ahead-vfs'; + +ensureMemoryWriteAheadDatabaseFiles(runtime, '/analytics.sqlite'); +``` + +If a separate JavaScript context creates the attached files, transfer them back to the runtime owner and register them before another connection opens that database: + +```ts +import { ensureMemoryWriteAheadDatabaseFiles, registerMemoryWriteAheadSharedFiles } from 'memory-write-ahead-vfs'; + +// Worker that first opens the attached database: +const attachedFiles = ensureMemoryWriteAheadDatabaseFiles(workerRuntime, '/analytics.sqlite'); +postMessage({ files: [attachedFiles.database, ...attachedFiles.writeAhead] }); + +// Runtime-owning parent after receiving that structured clone: +registerMemoryWriteAheadSharedFiles({ + runtimeDbFilename: parentRuntime.dbFilename, + files: event.data.files, +}); +``` + +Structured cloning shares the underlying `SharedArrayBuffer`s; it does not copy the database bytes. + ## Capacity options ```ts @@ -152,7 +184,9 @@ import { getMemoryWriteAheadRuntimeDiagnostics } from 'memory-write-ahead-vfs'; const diagnostics = getMemoryWriteAheadRuntimeDiagnostics('/app.sqlite'); ``` -Diagnostics include logical file sizes, segment counts, reclaimed capacity, lock state, open handle counts, and stale-lock recovery counters. +Diagnostics include logical file sizes, segment counts, reclaimed capacity, lock state, open handle counts, stale-lock recovery counters, and attached databases quarantined after an abandoned transaction. + +Version 0.2.0 also records logical-size, active-capacity, and allocated-capacity high-water marks; write and truncate counts; bytes written; lock wait counts and time; and checkpoint counts, duration, and write-ahead pages. Pass `logger` to `MemoryWriteAheadVFS.create(..., { logger })` for structured debug/error output. `vfs.lastError` keeps the last VFS-layer exception for SQLite's `xGetLastError`, and `vfs.log` is a wa-sqlite trace sink toggled by @@ -169,3 +203,5 @@ pnpm test:browser ``` The browser integration tests run through Playwright against a Vite server with COOP/COEP headers enabled. + +The package includes a narrowly modified copy of wa-sqlite 1.7.0's `WriteAhead` helper so SQLite deferred transactions can safely upgrade a current read snapshot to a write snapshot. See [`THIRD_PARTY_NOTICES`](https://github.com/whygee-dev/memory-write-ahead-vfs/blob/v0.2.0/THIRD_PARTY_NOTICES) for provenance and license terms. diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES new file mode 100644 index 0000000..e1eaf46 --- /dev/null +++ b/THIRD_PARTY_NOTICES @@ -0,0 +1,25 @@ +This package includes a modified copy of the WriteAhead helper from +@journeyapps/wa-sqlite 1.7.0: +https://github.com/powersync-ja/wa-sqlite/blob/v1.7.0/src/examples/WriteAhead.js + +MIT License + +Copyright (c) 2023 Roy T. Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmarks/memory-vfs-comparison/README.md b/benchmarks/memory-vfs-comparison/README.md new file mode 100644 index 0000000..f67c4f2 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/README.md @@ -0,0 +1,131 @@ +# Memory VFS implementation-stack benchmark + +This benchmark compares PowerSync's public `InMemoryVFS` configuration with Memory Write-Ahead VFS connection pools. It measures the complete JavaScript stack used by each arm—adapter, worker transport, SQLite WASM, VFS, and pool routing—not an isolated VFS function call. + +Latency is p50 / p95 in milliseconds. Lower is better. + +## Primary comparison: workers enabled for both implementations + +| Configuration | Sequential reads | Concurrent reads | Mixed reads | Mixed write completion | Pool ready | +| ------------------------------------------ | ---------------: | ---------------: | ------------: | ---------------------: | -------------: | +| PowerSync MemoryVFS — 1 connection | 31.97 / 34.69 | 31.70 / 34.78 | 31.84 / 36.35 | 32.27 / 36.80 | 36.20 / 45.56 | +| Memory Write-Ahead VFS — 1 connection | 31.71 / 35.76 | 31.43 / 34.05 | 31.47 / 33.25 | 31.89 / 33.76 | 42.39 / 49.74 | +| Memory Write-Ahead VFS — 2-connection pool | 31.64 / 34.69 | 18.34 / 19.85 | 19.30 / 22.86 | 13.77 / 16.36 | 74.39 / 89.62 | +| Memory Write-Ahead VFS — 3-connection pool | 31.91 / 34.32 | 16.17 / 18.88 | 16.69 / 19.68 | 7.99 / 9.51 | 80.16 / 96.01 | +| Memory Write-Ahead VFS — 4-connection pool | 31.49 / 35.02 | 15.09 / 17.69 | 15.93 / 18.02 | 7.06 / 8.55 | 87.64 / 104.61 | + +Compared with the PowerSync MemoryVFS arm, the 3-connection Memory Write-Ahead VFS pool recorded: + +- 49.0% lower median concurrent-read latency. +- 47.6% lower median mixed-read latency. +- 75.2% lower median mixed-workload write-completion latency. +- 0.2% lower median sequential latency. +- 43.96 ms more median pool-ready latency because three workers and SQLite instances are opened instead of one. + +The 3-connection pool means one writer plus two additional readers. For throughput-focused desktop use, it is the startup/concurrency trade-off suggested by this run: the fourth connection improved median concurrent reads by 6.7% and mixed reads by 4.6%, while adding another worker and 7.48 ms of median pool-ready latency. This is not a memory-balanced conclusion; the aggregate cache ceilings are disclosed below. + +[Full result](./results/both-workers.json) + +### Connection and cache topology + +Every SQLite connection receives the same configured 32 MiB page-cache ceiling. A pool therefore has a higher aggregate configured ceiling: + +| Configuration | Total connections | Additional readers | Aggregate configured cache ceiling | +| ------------------------------------------ | ----------------: | -----------------: | ---------------------------------: | +| PowerSync MemoryVFS — 1 connection | 1 | 0 | 32 MiB | +| Memory Write-Ahead VFS — 1 connection | 1 | 0 | 32 MiB | +| Memory Write-Ahead VFS — 2-connection pool | 2 | 1 | 64 MiB | +| Memory Write-Ahead VFS — 3-connection pool | 3 | 2 | 96 MiB | +| Memory Write-Ahead VFS — 4-connection pool | 4 | 3 | 128 MiB | + +SQLite's cache setting is a ceiling, not preallocated resident memory, and this harness does not measure RSS. No fixed-total-cache control was run. The results compare the implementations as configured per connection and must not be presented as equal-memory results. + +## Application configuration: PowerSync on the main thread, MWA in workers + +This asymmetric configuration represents an application that uses `useWebWorker: false` for PowerSync MemoryVFS and dedicated workers for Memory Write-Ahead VFS. It is useful as an application-level measurement, but it is not the primary like-for-like worker comparison. + +| Configuration | Sequential reads | Concurrent reads | Mixed reads | Mixed write completion | Pool ready | +| ------------------------------------------ | ---------------: | ---------------: | ------------: | ---------------------: | -------------: | +| PowerSync MemoryVFS — 1 connection | 27.88 / 31.06 | 28.70 / 31.64 | 28.36 / 31.27 | 28.63 / 31.50 | 6.78 / 10.80 | +| Memory Write-Ahead VFS — 1 connection | 29.75 / 32.57 | 29.18 / 32.68 | 29.04 / 32.03 | 29.47 / 32.64 | 29.69 / 40.02 | +| Memory Write-Ahead VFS — 2-connection pool | 30.59 / 38.89 | 17.88 / 20.68 | 18.53 / 20.57 | 13.31 / 14.94 | 59.05 / 73.05 | +| Memory Write-Ahead VFS — 3-connection pool | 30.45 / 34.81 | 15.33 / 17.19 | 15.88 / 20.35 | 7.74 / 9.42 | 62.50 / 130.55 | +| Memory Write-Ahead VFS — 4-connection pool | 30.28 / 32.93 | 14.72 / 15.74 | 15.45 / 16.89 | 6.94 / 7.77 | 67.68 / 81.82 | + +[Full result](./results/application-workers.json) + +## Control: workers disabled for both implementations + +| Configuration | Sequential reads | Concurrent reads | Mixed reads | Mixed write completion | Pool ready | +| ------------------------------------------ | ---------------: | ---------------: | ------------: | ---------------------: | ------------: | +| PowerSync MemoryVFS — 1 connection | 27.06 / 30.20 | 27.22 / 30.09 | 27.18 / 29.46 | 27.40 / 29.79 | 5.32 / 7.55 | +| Memory Write-Ahead VFS — 1 connection | 27.03 / 28.59 | 26.96 / 29.40 | 27.12 / 28.67 | 27.37 / 28.92 | 5.28 / 8.20 | +| Memory Write-Ahead VFS — 2-connection pool | 27.03 / 29.64 | 27.28 / 30.49 | 27.30 / 29.94 | 11.75 / 13.13 | 10.69 / 14.18 | +| Memory Write-Ahead VFS — 3-connection pool | 27.30 / 29.57 | 27.38 / 29.14 | 27.71 / 30.71 | 20.63 / 22.53 | 14.58 / 18.87 | +| Memory Write-Ahead VFS — 4-connection pool | 27.36 / 29.21 | 27.43 / 29.09 | 27.74 / 29.86 | 13.41 / 14.57 | 18.51 / 26.20 | + +These pools still contain multiple SQLite connections, but every connection runs on the same JavaScript thread. The read workload therefore gains no CPU parallelism; this control should not be presented as evidence of concurrent-reader acceleration. + +[Full result](./results/both-main-thread.json) + +## Reproduce + +Requirements: Node.js 20+, pnpm 10.27.0, and Playwright Chromium. + +```bash +pnpm install --frozen-lockfile +pnpm exec playwright install chromium +``` + +Workers enabled for both implementations: + +```bash +pnpm benchmark +``` + +Validate and sanitize that raw result for publication: + +```bash +pnpm benchmark:publish-result \ + benchmark-results/memory-vfs-worker-worker.json \ + benchmarks/memory-vfs-comparison/results/both-workers.json \ + true true +``` + +PowerSync on the main thread and Memory Write-Ahead VFS in workers: + +```bash +BENCHMARK_POWERSYNC_USE_WEB_WORKER=false \ +BENCHMARK_MWA_USE_WEB_WORKER=true \ +BENCHMARK_OUTPUT=benchmark-results/application-workers.json \ +pnpm benchmark +``` + +Workers disabled for both implementations: + +```bash +BENCHMARK_POWERSYNC_USE_WEB_WORKER=false \ +BENCHMARK_MWA_USE_WEB_WORKER=false \ +BENCHMARK_OUTPUT=benchmark-results/both-main-thread.json \ +pnpm benchmark +``` + +`useWebWorker` is PowerSync's public `flags.useWebWorker` option for the PowerSync arm. `BENCHMARK_MWA_USE_WEB_WORKER` controls the equivalent benchmark harness choice for the Memory Write-Ahead VFS arm; it is not a package API. + +## Methodology + +- Pinned runtime versions: `@powersync/web` 1.39.0, `@powersync/common` 1.57.2, `@journeyapps/wa-sqlite` 1.7.0, `@playwright/test` 1.60.0, Vite 8.0.16, and `memory-write-ahead-vfs` 0.2.0. +- Same Chromium page, deterministic schema and data, eight SQL read queries, 32 MiB SQLite cache per connection, and identical SQL for every arm. Aggregate configured cache ceilings consequently range from 32 MiB to 128 MiB as shown above. +- Dataset: 30 collections, 6 groups, 12 runs, 30,000 records, and 3,000 labels, plus deterministic join rows and indexes. +- MWA reads use the least-active connection among the writer and additional readers. Writes stay pinned to the writer. Each connection is serialized independently. +- Ten workload passes per arm. Each pass excludes two warmups and records 10 sequential trials, 12 concurrent-read trials, and 8 mixed read/write trials. +- Published percentiles use the raw 100 sequential, 120 concurrent, 80 mixed-read, and 80 mixed-write samples—not percentiles of pass averages. +- Thirty startup passes per arm. Arm order rotates and reverses between passes to reduce order and thermal bias. +- Every arm must pass dataset row counts, full-result SHA-256 equivalence across arms, per-trial checksums, exact affected-row counts, and final write-value verification. MWA pools must also prove that mutations routed through an additional reader are rejected and leave data unchanged. +- The committed result was recorded in cross-origin-isolated Playwright Chromium 148 with `navigator.hardwareConcurrency = 18`. + +## Interpretation limits + +This is one synthetic workload on one machine and browser build. It does not establish universal application performance, equal-memory performance, memory consumption, energy use, or behavior on lower-end devices. Worker scheduling and p95 latency vary between runs. Use the harness with production-shaped queries and device classes before choosing a pool size. + +The benchmark pool is reference harness code, not an exported connection-pool API. PowerSync MemoryVFS is tested through its public `WASQLiteOpenFactory` with `WASQLiteVFS.InMemoryVfs`; Memory Write-Ahead VFS is tested through its public VFS API and the reference pool in this directory. diff --git a/benchmarks/memory-vfs-comparison/benchmark.spec.ts b/benchmarks/memory-vfs-comparison/benchmark.spec.ts new file mode 100644 index 0000000..3305bd3 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/benchmark.spec.ts @@ -0,0 +1,85 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, join, parse, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { expect, test } from '@playwright/test'; +import memoryWriteAheadPackage from '../../package.json' with { type: 'json' }; + +type BrowserRuntime = typeof import('./runtime.js'); + +const readBoolean = (name: string, fallback: boolean) => { + const value = process.env[name]; + if (value === undefined) { + return fallback; + } + if (value === 'true') { + return true; + } + if (value === 'false') { + return false; + } + throw new Error(`${name} must be true or false; received ${value}`); +}; + +const readInstalledPackageVersion = (packageName: string) => { + let directory = dirname(fileURLToPath(import.meta.resolve(packageName))); + const root = parse(directory).root; + + while (directory !== root) { + const packagePath = join(directory, 'package.json'); + if (existsSync(packagePath)) { + const packageJson = JSON.parse(readFileSync(packagePath, 'utf8')) as { + name?: string; + version?: string; + }; + if (packageJson.name === packageName && packageJson.version) { + return packageJson.version; + } + } + directory = dirname(directory); + } + + throw new Error(`Unable to resolve installed version for ${packageName}`); +}; + +test('compares PowerSync MemoryVFS with Memory Write-Ahead VFS connection pools', async ({ page, baseURL }) => { + test.setTimeout(30 * 60_000); + await page.goto(new URL('/test-host.html', baseURL ?? 'http://127.0.0.1:5174').toString()); + + const flags = { + memoryWriteAheadUseWebWorker: readBoolean('BENCHMARK_MWA_USE_WEB_WORKER', true), + powerSyncUseWebWorker: readBoolean('BENCHMARK_POWERSYNC_USE_WEB_WORKER', true), + }; + const versions = { + memoryWriteAheadVfs: memoryWriteAheadPackage.version, + playwright: readInstalledPackageVersion('@playwright/test'), + powerSyncCommon: readInstalledPackageVersion('@powersync/common'), + powerSyncWeb: readInstalledPackageVersion('@powersync/web'), + vite: readInstalledPackageVersion('vite'), + waSqlite: readInstalledPackageVersion('@journeyapps/wa-sqlite'), + }; + const result = await page.evaluate( + async ({ benchmarkFlags, benchmarkVersions }) => { + const modulePath = '/benchmarks/memory-vfs-comparison/runtime.ts'; + const runtime = (await import(modulePath)) as BrowserRuntime; + return runtime.runPublicMemoryVfsBenchmark(benchmarkFlags, benchmarkVersions); + }, + { benchmarkFlags: flags, benchmarkVersions: versions } + ); + + expect(result.arms).toHaveLength(5); + expect(result.environment.crossOriginIsolated).toBe(true); + expect(result.arms.every((arm) => arm.sequential.samples === 100)).toBe(true); + expect(result.arms.every((arm) => arm.burst.samples === 120)).toBe(true); + expect(result.arms.every((arm) => arm.mixedRead.samples === 80)).toBe(true); + expect(result.arms.every((arm) => arm.mixedWrite.samples === 80)).toBe(true); + + const outputPath = resolve( + process.env.BENCHMARK_OUTPUT ?? + `benchmark-results/memory-vfs-${flags.powerSyncUseWebWorker ? 'worker' : 'main'}-${ + flags.memoryWriteAheadUseWebWorker ? 'worker' : 'main' + }.json` + ); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`, 'utf8'); +}); diff --git a/benchmarks/memory-vfs-comparison/mwaConnection.ts b/benchmarks/memory-vfs-comparison/mwaConnection.ts new file mode 100644 index 0000000..37545c4 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/mwaConnection.ts @@ -0,0 +1,296 @@ +import * as SQLite from '@journeyapps/wa-sqlite'; +import waSqliteModuleFactory from '@journeyapps/wa-sqlite/dist/wa-sqlite.mjs'; +import { + DBAdapterDefaultMixin, + type DBAdapterListener, + type DBLockOptions, + type LockContext, + type QueryResult, +} from '@powersync/common'; +import * as Comlink from 'comlink'; + +import { MEMORY_WRITE_AHEAD_VFS, MemoryWriteAheadVFS, type MemoryWriteAheadSharedRuntime } from '../../src/index.js'; + +type SqlValue = bigint | number | string | Uint8Array | number[] | null; + +export type BenchmarkSqlResult = { + columns: string[]; + insertId: number; + rows: SqlValue[][]; + rowsAffected: number; +}; + +export type BenchmarkWorkerApi = { + close(): Promise; + execute(sql: string): Promise; + init(options: BenchmarkConnectionOptions): Promise; +}; + +type BenchmarkConnectionOptions = { + cacheSizeKiB: number; + dbFilename: string; + initializePragmas?: boolean; + runtime: MemoryWriteAheadSharedRuntime; +}; + +type BenchmarkConnection = { + close(): Promise; + execute(sql: string): Promise; +}; + +type NamedConnection = { + active: number; + connection: BenchmarkConnection; + label: string; + tail: Promise; +}; + +const rowsToObjects = (result: BenchmarkSqlResult) => + result.rows.map((row) => Object.fromEntries(result.columns.map((column, index) => [column, row[index]]))); + +const createLockContext = (connection: BenchmarkConnection): LockContext => { + const execute = async (query: string, params?: unknown[]): Promise => { + if (params && params.length > 0) { + throw new Error('The public benchmark uses literal deterministic SQL and does not accept bindings'); + } + + const result = await connection.execute(query); + const rows = rowsToObjects(result); + return { + insertId: result.insertId, + rowsAffected: result.rowsAffected, + ...(result.columns.length > 0 + ? { + rows: { + _array: rows, + length: rows.length, + item: (index: number) => rows[index], + }, + } + : {}), + }; + }; + + return { + connectionType: 'writer', + execute, + executeBatch: async (query: string, params: unknown[][] = []) => { + const results = await params.reduce>( + (previous, bindings) => + previous.then(async (entries) => { + entries.push(await execute(query, bindings)); + return entries; + }), + Promise.resolve([]) + ); + const insertId = results.at(-1)?.insertId; + return { + rowsAffected: results.reduce((sum, result) => sum + result.rowsAffected, 0), + ...(insertId === undefined ? {} : { insertId }), + }; + }, + executeRaw: async (query: string, params?: unknown[]) => { + if (params && params.length > 0) { + throw new Error('The public benchmark uses literal deterministic SQL and does not accept bindings'); + } + return (await connection.execute(query)).rows; + }, + get: async (query: string, params?: unknown[]) => { + const rows = (await execute(query, params)).rows?._array ?? []; + if (rows.length !== 1) { + throw new Error(`Expected exactly one row, received ${rows.length}`); + } + return rows[0] as T; + }, + getAll: async (query: string, params?: unknown[]) => + ((await execute(query, params)).rows?._array ?? []) as T[], + getOptional: async (query: string, params?: unknown[]) => + (((await execute(query, params)).rows?._array ?? [])[0] as T | undefined) ?? null, + }; +}; + +export const createLocalBenchmarkConnection = async ({ + cacheSizeKiB, + dbFilename, + initializePragmas = true, + runtime, +}: BenchmarkConnectionOptions): Promise => { + const module = await waSqliteModuleFactory(); + const sqlite3 = SQLite.Factory(module); + const vfs = await MemoryWriteAheadVFS.create(MEMORY_WRITE_AHEAD_VFS, module, runtime); + sqlite3.vfs_register(vfs as unknown as SQLiteVFS, true); + const db = await sqlite3.open_v2(dbFilename, undefined, MEMORY_WRITE_AHEAD_VFS); + let closed = false; + + const execute = async (sql: string): Promise => { + const rows: SqlValue[][] = []; + let columns: string[] = []; + await sqlite3.exec(db, sql, (row: SqlValue[], columnNames: string[]) => { + if (columnNames.length > 0) { + columns = columnNames; + rows.push(row); + } + }); + return { + columns, + insertId: Number(sqlite3.last_insert_id(db)), + rows, + rowsAffected: sqlite3.changes(db), + }; + }; + + if (initializePragmas) { + await execute('PRAGMA temp_store = memory'); + await execute(`PRAGMA cache_size = -${cacheSizeKiB}`); + } + + return { + async close() { + if (closed) { + return; + } + closed = true; + await sqlite3.close(db); + await vfs.close(); + }, + execute, + ownerId: vfs.getRuntimeOwnerId() ?? 0, + }; +}; + +const createWorkerBenchmarkConnection = async ( + options: BenchmarkConnectionOptions, + workerUrl: string, + label: string +): Promise => { + const worker = new Worker(workerUrl, { name: `mwa-benchmark-${label}`, type: 'module' }); + const remote = Comlink.wrap(worker); + + try { + await remote.init(options); + } catch (error) { + worker.terminate(); + remote[Comlink.releaseProxy](); + throw error; + } + + return { + async close() { + try { + await remote.close(); + } finally { + remote[Comlink.releaseProxy](); + worker.terminate(); + } + }, + execute: (sql) => remote.execute(sql), + }; +}; + +const runOnConnection = async (connection: NamedConnection, operation: () => Promise) => { + connection.active += 1; + const previous = connection.tail; + let release: () => void = () => undefined; + connection.tail = new Promise((resolve) => { + release = resolve; + }); + + try { + await previous; + return await operation(); + } finally { + connection.active -= 1; + release(); + } +}; + +type PoolOptions = BenchmarkConnectionOptions & { + additionalReaders: number; + useWebWorker: boolean; + workerUrl: string; +}; + +class BenchmarkMemoryWriteAheadConnectionPool { + readonly name: string; + readonly #readers: NamedConnection[] = []; + readonly #readersReady: Promise; + readonly #writer: Promise; + + constructor(private readonly options: PoolOptions) { + this.name = options.dbFilename; + this.#writer = this.#openConnection('writer'); + this.#readersReady = this.#writer.then(async () => { + const readers = await Promise.all( + Array.from({ length: options.additionalReaders }, (_, index) => + this.#openConnection(`reader-${index + 1}`) + ) + ); + this.#readers.push(...readers); + }); + } + + async #openConnection(label: string): Promise { + const connectionOptions = { + ...this.options, + initializePragmas: label === 'writer', + }; + const connection = this.options.useWebWorker + ? await createWorkerBenchmarkConnection(connectionOptions, this.options.workerUrl, label) + : await createLocalBenchmarkConnection(connectionOptions); + return { active: 0, connection, label, tail: Promise.resolve() }; + } + + async init() { + await this.waitForAdditionalReaderPool(); + } + + async waitForAdditionalReaderPool() { + await this.#readersReady; + } + + async close() { + await this.#readersReady.catch(() => undefined); + const writer = await this.#writer; + await Promise.all([writer.connection.close(), ...this.#readers.map((reader) => reader.connection.close())]); + } + + async readLock(fn: (tx: LockContext) => Promise, _options?: DBLockOptions): Promise { + const writer = await this.#writer; + const candidates = [...this.#readers, writer]; + const selected = candidates.reduce((current, candidate) => + candidate.active < current.active ? candidate : current + ); + return runOnConnection(selected, () => fn(createLockContext(selected.connection))); + } + + async writeLock(fn: (tx: LockContext) => Promise, _options?: DBLockOptions): Promise { + const writer = await this.#writer; + return runOnConnection(writer, () => fn(createLockContext(writer.connection))); + } + + async refreshSchema() { + await this.#readersReady; + const writer = await this.#writer; + await Promise.all( + [writer, ...this.#readers].map((connection) => + runOnConnection(connection, async () => { + await connection.connection.execute('PRAGMA schema_version'); + await connection.connection.execute('PRAGMA temp_store = memory'); + await connection.connection.execute(`PRAGMA cache_size = -${this.options.cacheSizeKiB}`); + if (connection.label !== 'writer') { + await connection.connection.execute('PRAGMA query_only = ON'); + } + }) + ) + ); + } + + registerListener(_listener: Partial) { + return () => undefined; + } +} + +const BenchmarkMemoryWriteAheadAdapter = DBAdapterDefaultMixin(BenchmarkMemoryWriteAheadConnectionPool); + +export const createMemoryWriteAheadBenchmarkDatabase = (options: PoolOptions) => + new BenchmarkMemoryWriteAheadAdapter(options); diff --git a/benchmarks/memory-vfs-comparison/mwaWorker.ts b/benchmarks/memory-vfs-comparison/mwaWorker.ts new file mode 100644 index 0000000..f0e3dd3 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/mwaWorker.ts @@ -0,0 +1,27 @@ +import * as Comlink from 'comlink'; + +import { createLocalBenchmarkConnection, type BenchmarkSqlResult, type BenchmarkWorkerApi } from './mwaConnection.js'; + +let connection: Awaited> | null = null; + +const api: BenchmarkWorkerApi = { + async close() { + await connection?.close(); + connection = null; + }, + async execute(sql: string): Promise { + if (!connection) { + throw new Error('Memory Write-Ahead VFS benchmark worker is not initialized'); + } + return connection.execute(sql); + }, + async init(options) { + if (connection) { + throw new Error('Memory Write-Ahead VFS benchmark worker is already initialized'); + } + connection = await createLocalBenchmarkConnection(options); + return connection.ownerId; + }, +}; + +Comlink.expose(api); diff --git a/benchmarks/memory-vfs-comparison/results/application-workers.json b/benchmarks/memory-vfs-comparison/results/application-workers.json new file mode 100644 index 0000000..dd7cdb2 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/results/application-workers.json @@ -0,0 +1,406 @@ +{ + "resultFormat": "memory-vfs-implementation-stack-v1", + "schemaVersion": 1, + "runDate": "2026-07-14", + "comparisonScope": "implementation-stack", + "environment": { + "browser": "Chromium 148.0.7778.96", + "crossOriginIsolated": true, + "hardwareConcurrency": 18 + }, + "harness": { + "memoryWriteAheadUseWebWorker": true, + "powerSyncUseWebWorker": false, + "webServer": "vite-development" + }, + "runtimeVersions": { + "memoryWriteAheadVfs": "0.2.0", + "playwright": "1.60.0", + "powerSyncCommon": "1.57.2", + "powerSyncWeb": "1.39.0", + "vite": "8.0.16", + "waSqlite": "1.7.0" + }, + "configuration": { + "burstTrials": 12, + "cacheSizeKiB": 32768, + "correctnessValidation": [ + "dataset-row-counts", + "cross-arm-full-result-sha256", + "per-trial-checksum", + "write-row-count", + "final-write-values", + "additional-reader-query-only" + ], + "dataset": { + "collections": 30, + "runs": 12, + "recordsPerCollection": 1000, + "labelsPerCollection": 100, + "groups": 6 + }, + "mixedTrials": 8, + "openPasses": 30, + "queriesPerTrial": 8, + "sequentialTrials": 10, + "warmupTrials": 2, + "workloadPasses": 10 + }, + "arms": [ + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 32.41, + "mean": 28.72, + "min": 26.05, + "p50": 28.7, + "p95": 31.64, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 27.26, + "mean": 8.26, + "min": 3.82, + "p50": 5.84, + "p95": 15.91, + "samples": 30 + }, + "firstWrite": { + "max": 11.05, + "mean": 6.19, + "min": 3.26, + "p50": 5.89, + "p95": 9.92, + "samples": 30 + }, + "mixedRead": { + "max": 32.68, + "mean": 28.45, + "min": 26.15, + "p50": 28.36, + "p95": 31.27, + "samples": 80 + }, + "mixedWrite": { + "max": 32.93, + "mean": 28.71, + "min": 26.31, + "p50": 28.63, + "p95": 31.5, + "samples": 80 + }, + "name": "powersync-memory-vfs-1", + "poolReady": { + "max": 10.9, + "mean": 6.93, + "min": 3.41, + "p50": 6.78, + "p95": 10.8, + "samples": 30 + }, + "seed": { + "max": 82.66, + "mean": 72.63, + "min": 55.72, + "p50": 71.79, + "p95": 82.66, + "samples": 10 + }, + "sequential": { + "max": 32.25, + "mean": 28.67, + "min": 26.16, + "p50": 27.88, + "p95": 31.06, + "samples": 100 + }, + "vfs": "InMemoryVFS" + }, + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 34.64, + "mean": 29.13, + "min": 26.27, + "p50": 29.18, + "p95": 32.68, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 45.64, + "mean": 33.66, + "min": 27.61, + "p50": 32.04, + "p95": 42.38, + "samples": 30 + }, + "firstWrite": { + "max": 44.03, + "mean": 32.78, + "min": 28.35, + "p50": 30.74, + "p95": 43.77, + "samples": 30 + }, + "mixedRead": { + "max": 34.71, + "mean": 29.02, + "min": 25.72, + "p50": 29.04, + "p95": 32.03, + "samples": 80 + }, + "mixedWrite": { + "max": 35.23, + "mean": 29.4, + "min": 25.99, + "p50": 29.47, + "p95": 32.64, + "samples": 80 + }, + "name": "memory-write-ahead-1", + "poolReady": { + "max": 41.22, + "mean": 31.45, + "min": 27.99, + "p50": 29.69, + "p95": 40.02, + "samples": 30 + }, + "seed": { + "max": 102.55, + "mean": 87.2, + "min": 73.84, + "p50": 84.11, + "p95": 102.55, + "samples": 10 + }, + "sequential": { + "max": 34.59, + "mean": 29.52, + "min": 26.13, + "p50": 29.75, + "p95": 32.57, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 1, + "aggregateCacheSizeKiB": 65536, + "burst": { + "max": 23.86, + "mean": 17.9, + "min": 15.32, + "p50": 17.88, + "p95": 20.68, + "samples": 120 + }, + "connectionCount": 2, + "firstRead": { + "max": 44.94, + "mean": 32.23, + "min": 28.75, + "p50": 31.02, + "p95": 41.01, + "samples": 30 + }, + "firstWrite": { + "max": 46.87, + "mean": 32.43, + "min": 27.95, + "p50": 31.02, + "p95": 43.36, + "samples": 30 + }, + "mixedRead": { + "max": 23.62, + "mean": 18.42, + "min": 15.98, + "p50": 18.53, + "p95": 20.57, + "samples": 80 + }, + "mixedWrite": { + "max": 16.25, + "mean": 13.31, + "min": 11.39, + "p50": 13.31, + "p95": 14.94, + "samples": 80 + }, + "name": "memory-write-ahead-2", + "poolReady": { + "max": 84.66, + "mean": 61.2, + "min": 55.64, + "p50": 59.05, + "p95": 73.05, + "samples": 30 + }, + "seed": { + "max": 101.09, + "mean": 85.49, + "min": 74.08, + "p50": 84.7, + "p95": 101.09, + "samples": 10 + }, + "sequential": { + "max": 43.04, + "mean": 30.99, + "min": 26.7, + "p50": 30.59, + "p95": 38.89, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 2, + "aggregateCacheSizeKiB": 98304, + "burst": { + "max": 18.38, + "mean": 15.18, + "min": 13.46, + "p50": 15.33, + "p95": 17.19, + "samples": 120 + }, + "connectionCount": 3, + "firstRead": { + "max": 75.37, + "mean": 35.42, + "min": 27.5, + "p50": 33.31, + "p95": 44.06, + "samples": 30 + }, + "firstWrite": { + "max": 48.82, + "mean": 35.89, + "min": 30.95, + "p50": 33.41, + "p95": 48.18, + "samples": 30 + }, + "mixedRead": { + "max": 23.97, + "mean": 15.94, + "min": 13.8, + "p50": 15.88, + "p95": 20.35, + "samples": 80 + }, + "mixedWrite": { + "max": 10.73, + "mean": 7.84, + "min": 6.7, + "p50": 7.74, + "p95": 9.42, + "samples": 80 + }, + "name": "memory-write-ahead-3", + "poolReady": { + "max": 182.24, + "mean": 70.3, + "min": 58.47, + "p50": 62.5, + "p95": 130.55, + "samples": 30 + }, + "seed": { + "max": 132.66, + "mean": 94.08, + "min": 74.2, + "p50": 83.58, + "p95": 132.66, + "samples": 10 + }, + "sequential": { + "max": 36.83, + "mean": 30.4, + "min": 26.82, + "p50": 30.45, + "p95": 34.81, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 3, + "aggregateCacheSizeKiB": 131072, + "burst": { + "max": 16.14, + "mean": 14.42, + "min": 12.73, + "p50": 14.72, + "p95": 15.74, + "samples": 120 + }, + "connectionCount": 4, + "firstRead": { + "max": 68.16, + "mean": 34.76, + "min": 30.15, + "p50": 33.19, + "p95": 44.14, + "samples": 30 + }, + "firstWrite": { + "max": 67.92, + "mean": 39.82, + "min": 32.21, + "p50": 37.32, + "p95": 59.38, + "samples": 30 + }, + "mixedRead": { + "max": 17.87, + "mean": 15.37, + "min": 13.02, + "p50": 15.45, + "p95": 16.89, + "samples": 80 + }, + "mixedWrite": { + "max": 8.48, + "mean": 6.92, + "min": 5.82, + "p50": 6.94, + "p95": 7.77, + "samples": 80 + }, + "name": "memory-write-ahead-4", + "poolReady": { + "max": 101.65, + "mean": 70.21, + "min": 61.12, + "p50": 67.68, + "p95": 81.82, + "samples": 30 + }, + "seed": { + "max": 105.02, + "mean": 87.9, + "min": 76.97, + "p50": 84.64, + "p95": 105.02, + "samples": 10 + }, + "sequential": { + "max": 33.56, + "mean": 30.07, + "min": 26.82, + "p50": 30.28, + "p95": 32.93, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + } + ] +} diff --git a/benchmarks/memory-vfs-comparison/results/both-main-thread.json b/benchmarks/memory-vfs-comparison/results/both-main-thread.json new file mode 100644 index 0000000..36e233e --- /dev/null +++ b/benchmarks/memory-vfs-comparison/results/both-main-thread.json @@ -0,0 +1,406 @@ +{ + "resultFormat": "memory-vfs-implementation-stack-v1", + "schemaVersion": 1, + "runDate": "2026-07-14", + "comparisonScope": "implementation-stack", + "environment": { + "browser": "Chromium 148.0.7778.96", + "crossOriginIsolated": true, + "hardwareConcurrency": 18 + }, + "harness": { + "memoryWriteAheadUseWebWorker": false, + "powerSyncUseWebWorker": false, + "webServer": "vite-development" + }, + "runtimeVersions": { + "memoryWriteAheadVfs": "0.2.0", + "playwright": "1.60.0", + "powerSyncCommon": "1.57.2", + "powerSyncWeb": "1.39.0", + "vite": "8.0.16", + "waSqlite": "1.7.0" + }, + "configuration": { + "burstTrials": 12, + "cacheSizeKiB": 32768, + "correctnessValidation": [ + "dataset-row-counts", + "cross-arm-full-result-sha256", + "per-trial-checksum", + "write-row-count", + "final-write-values", + "additional-reader-query-only" + ], + "dataset": { + "collections": 30, + "runs": 12, + "recordsPerCollection": 1000, + "labelsPerCollection": 100, + "groups": 6 + }, + "mixedTrials": 8, + "openPasses": 30, + "queriesPerTrial": 8, + "sequentialTrials": 10, + "warmupTrials": 2, + "workloadPasses": 10 + }, + "arms": [ + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 32.22, + "mean": 27.74, + "min": 25.99, + "p50": 27.22, + "p95": 30.09, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 24.24, + "mean": 6.09, + "min": 3.25, + "p50": 5.38, + "p95": 9.3, + "samples": 30 + }, + "firstWrite": { + "max": 14.64, + "mean": 6, + "min": 3.39, + "p50": 5.56, + "p95": 14.04, + "samples": 30 + }, + "mixedRead": { + "max": 30.55, + "mean": 27.59, + "min": 26.27, + "p50": 27.18, + "p95": 29.46, + "samples": 80 + }, + "mixedWrite": { + "max": 30.8, + "mean": 27.82, + "min": 26.45, + "p50": 27.4, + "p95": 29.79, + "samples": 80 + }, + "name": "powersync-memory-vfs-1", + "poolReady": { + "max": 7.85, + "mean": 5.35, + "min": 3.29, + "p50": 5.32, + "p95": 7.55, + "samples": 30 + }, + "seed": { + "max": 81.64, + "mean": 58.59, + "min": 54.86, + "p50": 55.23, + "p95": 81.64, + "samples": 10 + }, + "sequential": { + "max": 31.57, + "mean": 27.69, + "min": 26.19, + "p50": 27.06, + "p95": 30.2, + "samples": 100 + }, + "vfs": "InMemoryVFS" + }, + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 31.84, + "mean": 27.44, + "min": 25.99, + "p50": 26.96, + "p95": 29.4, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 13.95, + "mean": 5.94, + "min": 3.38, + "p50": 5.6, + "p95": 10.28, + "samples": 30 + }, + "firstWrite": { + "max": 13.41, + "mean": 6.64, + "min": 3.66, + "p50": 6.63, + "p95": 11.85, + "samples": 30 + }, + "mixedRead": { + "max": 28.91, + "mean": 27.35, + "min": 26.27, + "p50": 27.12, + "p95": 28.67, + "samples": 80 + }, + "mixedWrite": { + "max": 29.37, + "mean": 27.6, + "min": 26.45, + "p50": 27.37, + "p95": 28.92, + "samples": 80 + }, + "name": "memory-write-ahead-1", + "poolReady": { + "max": 8.68, + "mean": 5.43, + "min": 3.15, + "p50": 5.28, + "p95": 8.2, + "samples": 30 + }, + "seed": { + "max": 71.85, + "mean": 66.42, + "min": 64.16, + "p50": 64.82, + "p95": 71.85, + "samples": 10 + }, + "sequential": { + "max": 29.75, + "mean": 27.22, + "min": 26.13, + "p50": 27.03, + "p95": 28.59, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 1, + "aggregateCacheSizeKiB": 65536, + "burst": { + "max": 32.39, + "mean": 27.74, + "min": 26.24, + "p50": 27.28, + "p95": 30.49, + "samples": 120 + }, + "connectionCount": 2, + "firstRead": { + "max": 15.63, + "mean": 6.43, + "min": 3.65, + "p50": 5.89, + "p95": 13.85, + "samples": 30 + }, + "firstWrite": { + "max": 9.19, + "mean": 5.79, + "min": 3.82, + "p50": 5.61, + "p95": 8.16, + "samples": 30 + }, + "mixedRead": { + "max": 30.93, + "mean": 27.68, + "min": 26.41, + "p50": 27.3, + "p95": 29.94, + "samples": 80 + }, + "mixedWrite": { + "max": 13.94, + "mean": 11.95, + "min": 11.3, + "p50": 11.75, + "p95": 13.13, + "samples": 80 + }, + "name": "memory-write-ahead-2", + "poolReady": { + "max": 15.93, + "mean": 10.77, + "min": 6.71, + "p50": 10.69, + "p95": 14.18, + "samples": 30 + }, + "seed": { + "max": 71.59, + "mean": 67.18, + "min": 64.07, + "p50": 65.83, + "p95": 71.59, + "samples": 10 + }, + "sequential": { + "max": 30.26, + "mean": 27.46, + "min": 26.17, + "p50": 27.03, + "p95": 29.64, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 2, + "aggregateCacheSizeKiB": 98304, + "burst": { + "max": 30.32, + "mean": 27.57, + "min": 26.4, + "p50": 27.38, + "p95": 29.14, + "samples": 120 + }, + "connectionCount": 3, + "firstRead": { + "max": 14.21, + "mean": 6.32, + "min": 3.68, + "p50": 6.45, + "p95": 11.26, + "samples": 30 + }, + "firstWrite": { + "max": 8.82, + "mean": 5.96, + "min": 3.97, + "p50": 6.12, + "p95": 8.28, + "samples": 30 + }, + "mixedRead": { + "max": 32.64, + "mean": 28.19, + "min": 26.52, + "p50": 27.71, + "p95": 30.71, + "samples": 80 + }, + "mixedWrite": { + "max": 24.76, + "mean": 20.94, + "min": 19.74, + "p50": 20.63, + "p95": 22.53, + "samples": 80 + }, + "name": "memory-write-ahead-3", + "poolReady": { + "max": 19.09, + "mean": 13.94, + "min": 8.38, + "p50": 14.58, + "p95": 18.87, + "samples": 30 + }, + "seed": { + "max": 73.72, + "mean": 66.75, + "min": 63.55, + "p50": 64.93, + "p95": 73.72, + "samples": 10 + }, + "sequential": { + "max": 30.89, + "mean": 27.66, + "min": 26.22, + "p50": 27.3, + "p95": 29.57, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 3, + "aggregateCacheSizeKiB": 131072, + "burst": { + "max": 31.78, + "mean": 27.76, + "min": 26.42, + "p50": 27.43, + "p95": 29.09, + "samples": 120 + }, + "connectionCount": 4, + "firstRead": { + "max": 20.09, + "mean": 6.59, + "min": 3.74, + "p50": 5.93, + "p95": 11.8, + "samples": 30 + }, + "firstWrite": { + "max": 12.21, + "mean": 5.34, + "min": 3.75, + "p50": 4.55, + "p95": 7.81, + "samples": 30 + }, + "mixedRead": { + "max": 31.11, + "mean": 28.08, + "min": 26.7, + "p50": 27.74, + "p95": 29.86, + "samples": 80 + }, + "mixedWrite": { + "max": 15.01, + "mean": 13.61, + "min": 12.81, + "p50": 13.41, + "p95": 14.57, + "samples": 80 + }, + "name": "memory-write-ahead-4", + "poolReady": { + "max": 26.43, + "mean": 18.58, + "min": 12.36, + "p50": 18.51, + "p95": 26.2, + "samples": 30 + }, + "seed": { + "max": 72.53, + "mean": 67.78, + "min": 63.89, + "p50": 67.32, + "p95": 72.53, + "samples": 10 + }, + "sequential": { + "max": 30.51, + "mean": 27.64, + "min": 26.38, + "p50": 27.36, + "p95": 29.21, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + } + ] +} diff --git a/benchmarks/memory-vfs-comparison/results/both-workers.json b/benchmarks/memory-vfs-comparison/results/both-workers.json new file mode 100644 index 0000000..86d2519 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/results/both-workers.json @@ -0,0 +1,406 @@ +{ + "resultFormat": "memory-vfs-implementation-stack-v1", + "schemaVersion": 1, + "runDate": "2026-07-14", + "comparisonScope": "implementation-stack", + "environment": { + "browser": "Chromium 148.0.7778.96", + "crossOriginIsolated": true, + "hardwareConcurrency": 18 + }, + "harness": { + "memoryWriteAheadUseWebWorker": true, + "powerSyncUseWebWorker": true, + "webServer": "vite-development" + }, + "runtimeVersions": { + "memoryWriteAheadVfs": "0.2.0", + "playwright": "1.60.0", + "powerSyncCommon": "1.57.2", + "powerSyncWeb": "1.39.0", + "vite": "8.0.16", + "waSqlite": "1.7.0" + }, + "configuration": { + "burstTrials": 12, + "cacheSizeKiB": 32768, + "correctnessValidation": [ + "dataset-row-counts", + "cross-arm-full-result-sha256", + "per-trial-checksum", + "write-row-count", + "final-write-values", + "additional-reader-query-only" + ], + "dataset": { + "collections": 30, + "runs": 12, + "recordsPerCollection": 1000, + "labelsPerCollection": 100, + "groups": 6 + }, + "mixedTrials": 8, + "openPasses": 30, + "queriesPerTrial": 8, + "sequentialTrials": 10, + "warmupTrials": 2, + "workloadPasses": 10 + }, + "arms": [ + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 38.19, + "mean": 32.03, + "min": 29.55, + "p50": 31.7, + "p95": 34.78, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 76.43, + "mean": 42.39, + "min": 36.18, + "p50": 39.68, + "p95": 57.71, + "samples": 30 + }, + "firstWrite": { + "max": 54.27, + "mean": 43.74, + "min": 39.3, + "p50": 42.52, + "p95": 50.14, + "samples": 30 + }, + "mixedRead": { + "max": 41.32, + "mean": 32.11, + "min": 29.15, + "p50": 31.84, + "p95": 36.35, + "samples": 80 + }, + "mixedWrite": { + "max": 41.74, + "mean": 32.57, + "min": 29.64, + "p50": 32.27, + "p95": 36.8, + "samples": 80 + }, + "name": "powersync-memory-vfs-1", + "poolReady": { + "max": 46.81, + "mean": 37.51, + "min": 33.38, + "p50": 36.2, + "p95": 45.56, + "samples": 30 + }, + "seed": { + "max": 96.8, + "mean": 90.1, + "min": 86.11, + "p50": 88.41, + "p95": 96.8, + "samples": 10 + }, + "sequential": { + "max": 40.7, + "mean": 32.38, + "min": 29.81, + "p50": 31.97, + "p95": 34.69, + "samples": 100 + }, + "vfs": "InMemoryVFS" + }, + { + "additionalReaderCount": 0, + "aggregateCacheSizeKiB": 32768, + "burst": { + "max": 38.47, + "mean": 31.59, + "min": 28.76, + "p50": 31.43, + "p95": 34.05, + "samples": 120 + }, + "connectionCount": 1, + "firstRead": { + "max": 55.59, + "mean": 44.45, + "min": 39.46, + "p50": 44.06, + "p95": 54.13, + "samples": 30 + }, + "firstWrite": { + "max": 52.92, + "mean": 45.61, + "min": 42.82, + "p50": 44.8, + "p95": 51.34, + "samples": 30 + }, + "mixedRead": { + "max": 34.53, + "mean": 31.47, + "min": 29.03, + "p50": 31.47, + "p95": 33.25, + "samples": 80 + }, + "mixedWrite": { + "max": 35.09, + "mean": 31.92, + "min": 29.56, + "p50": 31.89, + "p95": 33.76, + "samples": 80 + }, + "name": "memory-write-ahead-1", + "poolReady": { + "max": 52.58, + "mean": 43.2, + "min": 39.46, + "p50": 42.39, + "p95": 49.74, + "samples": 30 + }, + "seed": { + "max": 133.64, + "mean": 111.34, + "min": 102.76, + "p50": 107.09, + "p95": 133.64, + "samples": 10 + }, + "sequential": { + "max": 36.66, + "mean": 31.99, + "min": 29.57, + "p50": 31.71, + "p95": 35.76, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 1, + "aggregateCacheSizeKiB": 65536, + "burst": { + "max": 23.86, + "mean": 18.53, + "min": 17.26, + "p50": 18.34, + "p95": 19.85, + "samples": 120 + }, + "connectionCount": 2, + "firstRead": { + "max": 65.41, + "mean": 45.31, + "min": 39.61, + "p50": 43.59, + "p95": 53.15, + "samples": 30 + }, + "firstWrite": { + "max": 55.38, + "mean": 46.09, + "min": 41.71, + "p50": 44.99, + "p95": 52.58, + "samples": 30 + }, + "mixedRead": { + "max": 23.85, + "mean": 19.58, + "min": 18.01, + "p50": 19.3, + "p95": 22.86, + "samples": 80 + }, + "mixedWrite": { + "max": 17.88, + "mean": 14.11, + "min": 12.66, + "p50": 13.77, + "p95": 16.36, + "samples": 80 + }, + "name": "memory-write-ahead-2", + "poolReady": { + "max": 94.03, + "mean": 77, + "min": 71.1, + "p50": 74.39, + "p95": 89.62, + "samples": 30 + }, + "seed": { + "max": 112.65, + "mean": 106.51, + "min": 97.83, + "p50": 107.87, + "p95": 112.65, + "samples": 10 + }, + "sequential": { + "max": 43.56, + "mean": 32.01, + "min": 29.91, + "p50": 31.64, + "p95": 34.69, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 2, + "aggregateCacheSizeKiB": 98304, + "burst": { + "max": 21.82, + "mean": 16.47, + "min": 14.97, + "p50": 16.17, + "p95": 18.88, + "samples": 120 + }, + "connectionCount": 3, + "firstRead": { + "max": 64.98, + "mean": 46.41, + "min": 39.71, + "p50": 44.24, + "p95": 64.41, + "samples": 30 + }, + "firstWrite": { + "max": 63.13, + "mean": 51.07, + "min": 44.95, + "p50": 49.91, + "p95": 61.93, + "samples": 30 + }, + "mixedRead": { + "max": 21.2, + "mean": 17.02, + "min": 15.44, + "p50": 16.69, + "p95": 19.68, + "samples": 80 + }, + "mixedWrite": { + "max": 10.26, + "mean": 8.26, + "min": 7.26, + "p50": 7.99, + "p95": 9.51, + "samples": 80 + }, + "name": "memory-write-ahead-3", + "poolReady": { + "max": 101.04, + "mean": 83.03, + "min": 75.06, + "p50": 80.16, + "p95": 96.01, + "samples": 30 + }, + "seed": { + "max": 113.31, + "mean": 107.7, + "min": 102.64, + "p50": 106.63, + "p95": 113.31, + "samples": 10 + }, + "sequential": { + "max": 38.03, + "mean": 32.06, + "min": 30.01, + "p50": 31.91, + "p95": 34.32, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + }, + { + "additionalReaderCount": 3, + "aggregateCacheSizeKiB": 131072, + "burst": { + "max": 18.87, + "mean": 15.36, + "min": 14, + "p50": 15.09, + "p95": 17.69, + "samples": 120 + }, + "connectionCount": 4, + "firstRead": { + "max": 56.03, + "mean": 45.32, + "min": 40.89, + "p50": 44.63, + "p95": 49.59, + "samples": 30 + }, + "firstWrite": { + "max": 62.36, + "mean": 52.61, + "min": 43.5, + "p50": 52.78, + "p95": 61.03, + "samples": 30 + }, + "mixedRead": { + "max": 19.61, + "mean": 16.14, + "min": 14.14, + "p50": 15.93, + "p95": 18.02, + "samples": 80 + }, + "mixedWrite": { + "max": 9.16, + "mean": 7.29, + "min": 6.39, + "p50": 7.06, + "p95": 8.55, + "samples": 80 + }, + "name": "memory-write-ahead-4", + "poolReady": { + "max": 106.63, + "mean": 86.88, + "min": 74.96, + "p50": 87.64, + "p95": 104.61, + "samples": 30 + }, + "seed": { + "max": 114.5, + "mean": 105.94, + "min": 99.24, + "p50": 105.65, + "p95": 114.5, + "samples": 10 + }, + "sequential": { + "max": 38.02, + "mean": 31.72, + "min": 29.15, + "p50": 31.49, + "p95": 35.02, + "samples": 100 + }, + "vfs": "MemoryWriteAheadVFS" + } + ] +} diff --git a/benchmarks/memory-vfs-comparison/runner.ts b/benchmarks/memory-vfs-comparison/runner.ts new file mode 100644 index 0000000..474f02a --- /dev/null +++ b/benchmarks/memory-vfs-comparison/runner.ts @@ -0,0 +1,867 @@ +import { type DBAdapter, type LockContext } from '@powersync/common'; + +type BenchmarkArmName = + | 'powersync-memory-vfs-1' + | 'memory-write-ahead-1' + | 'memory-write-ahead-2' + | 'memory-write-ahead-3' + | 'memory-write-ahead-4'; + +type BenchmarkArm = { + additionalReaders: number; + connectionCount: number; + name: BenchmarkArmName; + vfs: 'InMemoryVFS' | 'MemoryWriteAheadVFS'; +}; + +type DatabaseOptions = { + additionalReaders: number; + dbFilename: string; + workerUrl: string; +}; + +export type MemoryDatabaseStackBenchmarkRuntime = { + cacheSizeKiB: number; + createInMemoryDatabase: (options: { dbFilename: string }) => DBAdapter; + createMemoryWriteAheadDatabase: (options: DatabaseOptions) => DBAdapter; + harness: { + powerSyncUseWebWorker: boolean; + memoryWriteAheadUseWebWorker: boolean; + webServer: 'vite-development'; + }; + onProgress: (message: string) => void; + resetMemoryWriteAheadRuntime: () => void; + versions: { + memoryWriteAheadVfs: string; + playwright: string; + powerSyncCommon: string; + powerSyncWeb: string; + vite: string; + waSqlite: string; + }; + workerUrl: string; +}; + +type BenchmarkSamples = { + burstMs: number[]; + firstReadMs: number[]; + firstWriteMs: number[]; + mixedReadMs: number[]; + mixedWriteMs: number[]; + poolReadyMs: number[]; + seedMs: number[]; + sequentialMs: number[]; +}; + +type BenchmarkSummary = { + max: number; + mean: number; + min: number; + p50: number; + p95: number; + samples: number; +}; + +type QueryResultBaseline = { + armName: BenchmarkArmName; + fingerprints: readonly string[]; +}; + +type QueryResultBaselineState = { + current?: QueryResultBaseline; +}; + +export type MemoryDatabaseStackBenchmarkArmResult = { + additionalReaderCount: number; + aggregateCacheSizeKiB: number; + burst: BenchmarkSummary; + firstRead: BenchmarkSummary; + firstWrite: BenchmarkSummary; + mixedRead: BenchmarkSummary; + mixedWrite: BenchmarkSummary; + name: BenchmarkArmName; + connectionCount: number; + poolReady: BenchmarkSummary; + seed: BenchmarkSummary; + sequential: BenchmarkSummary; + vfs: BenchmarkArm['vfs']; +}; + +export type MemoryDatabaseStackComparisonBenchmarkResult = { + arms: MemoryDatabaseStackBenchmarkArmResult[]; + comparisonScope: 'implementation-stack'; + configuration: { + burstTrials: number; + cacheSizeKiB: number; + correctnessValidation: readonly [ + 'dataset-row-counts', + 'cross-arm-full-result-sha256', + 'per-trial-checksum', + 'write-row-count', + 'final-write-values', + 'additional-reader-query-only', + ]; + dataset: typeof DATASET; + mixedTrials: number; + openPasses: number; + queriesPerTrial: number; + sequentialTrials: number; + warmupTrials: number; + workloadPasses: number; + }; + environment: { + crossOriginIsolated: boolean; + hardwareConcurrency: number; + userAgent: string; + }; + generatedAt: string; + harness: MemoryDatabaseStackBenchmarkRuntime['harness']; + runtimeVersions: MemoryDatabaseStackBenchmarkRuntime['versions']; + schemaVersion: 1; +}; + +const DATASET = { + collections: 30, + runs: 12, + recordsPerCollection: 1_000, + labelsPerCollection: 100, + groups: 6, +} as const; +const WARMUP_TRIALS = 2; +const SEQUENTIAL_TRIALS = 10; +const BURST_TRIALS = 12; +const MIXED_TRIALS = 8; +const WORKLOAD_PASSES = 10; +const OPEN_PASSES = 30; + +const ARMS: readonly BenchmarkArm[] = [ + { additionalReaders: 0, connectionCount: 1, name: 'powersync-memory-vfs-1', vfs: 'InMemoryVFS' }, + { additionalReaders: 0, connectionCount: 1, name: 'memory-write-ahead-1', vfs: 'MemoryWriteAheadVFS' }, + { additionalReaders: 1, connectionCount: 2, name: 'memory-write-ahead-2', vfs: 'MemoryWriteAheadVFS' }, + { additionalReaders: 2, connectionCount: 3, name: 'memory-write-ahead-3', vfs: 'MemoryWriteAheadVFS' }, + { additionalReaders: 3, connectionCount: 4, name: 'memory-write-ahead-4', vfs: 'MemoryWriteAheadVFS' }, +]; + +const TARGET_COLLECTION_IDS = Array.from( + { length: 12 }, + (_, index) => `collection-${String(index + 1).padStart(3, '0')}` +); +const TARGET_RUN_IDS = Array.from({ length: 8 }, (_, index) => `run-${String(index + 1).padStart(3, '0')}`); +const inList = (values: readonly string[]) => values.map((value) => `'${value}'`).join(', '); + +const READ_QUERIES = [ + ` + SELECT l.id, l.collectionId, l."order", l.name + FROM label l + WHERE l.accountId = 'account-01' + AND l.nodeLevel = 3 + AND l.collectionId IN (${inList(TARGET_COLLECTION_IDS)}) + AND l.deletedAt IS NULL + ORDER BY l."order" ASC + LIMIT 500 + `, + ` + SELECT c.id, c.groupId + FROM run c + WHERE c.accountId = 'account-01' + AND c.deletedAt IS NULL + AND EXISTS ( + SELECT 1 + FROM collection_to_group bw + WHERE bw.B = c.groupId + AND bw.A IN (${inList(TARGET_COLLECTION_IDS)}) + ) + ORDER BY c.createdAt ASC + `, + ` + SELECT e.collectionId, COUNT(*) AS recordCount + FROM record e + WHERE e.collectionId IN (${inList(TARGET_COLLECTION_IDS)}) + AND e.deletedAt IS NULL + GROUP BY e.collectionId + ORDER BY e.collectionId + `, + ` + SELECT e.status, e.category, COUNT(*) AS recordCount + FROM record e + WHERE e.collectionId IN (${inList(TARGET_COLLECTION_IDS)}) + AND e.deletedAt IS NULL + AND e.archivedAt IS NULL + GROUP BY e.status, e.category + ORDER BY e.status, e.category + `, + ` + WITH run_scoped_collections AS ( + SELECT c.id AS runId, bw.A AS collectionId + FROM run c + INNER JOIN collection_to_group bw ON bw.B = c.groupId + WHERE c.id IN (${inList(TARGET_RUN_IDS)}) + AND bw.A IN (${inList(TARGET_COLLECTION_IDS)}) + ) + SELECT + csb.runId, + csb.collectionId, + COUNT(DISTINCT e.id) AS totalRecords, + COUNT(DISTINCT CASE WHEN ce.state = 'DONE' THEN ce.recordId END) AS doneRecords, + COUNT(DISTINCT CASE WHEN ce.state IN ('DONE', 'ONGOING') THEN ce.recordId END) AS ongoingOrDoneRecords, + MAX(CASE WHEN cb.id IS NOT NULL THEN 1 ELSE 0 END) AS collectionCompleted + FROM run_scoped_collections csb + LEFT JOIN record e ON e.collectionId = csb.collectionId AND e.deletedAt IS NULL + LEFT JOIN run_record ce + ON ce.recordId = e.id + AND ce.runId = csb.runId + AND ce.deletedAt IS NULL + LEFT JOIN run_collection cb + ON cb.collectionId = csb.collectionId + AND cb.runId = csb.runId + GROUP BY csb.runId, csb.collectionId + ORDER BY csb.runId, csb.collectionId + `, + ` + SELECT ce.runId, ce.state, COUNT(*) AS recordCount + FROM run_record ce + WHERE ce.runId IN (${inList(TARGET_RUN_IDS)}) + AND ce.deletedAt IS NULL + GROUP BY ce.runId, ce.state + ORDER BY ce.runId, ce.state + `, + ` + SELECT bw.B AS groupId, COUNT(DISTINCT e.id) AS recordCount + FROM collection_to_group bw + INNER JOIN record e ON e.collectionId = bw.A + WHERE bw.A IN (${inList(TARGET_COLLECTION_IDS)}) + AND e.deletedAt IS NULL + GROUP BY bw.B + ORDER BY bw.B + `, + ` + SELECT + COUNT(*) AS activeRecordCount, + SUM(CASE WHEN e.imported = 1 THEN 1 ELSE 0 END) AS importedRecordCount, + SUM(CASE WHEN e.status = 'UNTRACEABLE' THEN 1 ELSE 0 END) AS untraceableRecordCount + FROM record e + WHERE e.collectionId IN (${inList(TARGET_COLLECTION_IDS)}) + AND e.deletedAt IS NULL + AND e.archivedAt IS NULL + `, +] as const; + +const createEmptySamples = (): BenchmarkSamples => ({ + burstMs: [], + firstReadMs: [], + firstWriteMs: [], + mixedReadMs: [], + mixedWriteMs: [], + poolReadyMs: [], + seedMs: [], + sequentialMs: [], +}); + +const round = (value: number) => Math.round(value * 100) / 100; +const summarize = (samples: readonly number[]): BenchmarkSummary => { + const sorted = [...samples].sort((left, right) => left - right); + const percentile = (value: number) => + sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1)] ?? 0; + + return { + max: round(sorted.at(-1) ?? 0), + mean: round(sorted.reduce((sum, value) => sum + value, 0) / Math.max(1, sorted.length)), + min: round(sorted[0] ?? 0), + p50: round(percentile(0.5)), + p95: round(percentile(0.95)), + samples: sorted.length, + }; +}; + +const executeStatements = async (tx: LockContext, statements: readonly string[]) => { + await statements.reduce>( + (previous, statement) => previous.then(async () => void (await tx.execute(statement))), + Promise.resolve() + ); +}; + +const resetRuntimeForArm = (arm: BenchmarkArm, runtime: MemoryDatabaseStackBenchmarkRuntime) => { + if (arm.vfs === 'MemoryWriteAheadVFS') { + runtime.resetMemoryWriteAheadRuntime(); + } +}; + +const armFilenameCode = (arm: BenchmarkArm) => (arm.vfs === 'InMemoryVFS' ? 'im' : `mw${arm.additionalReaders}`); + +const openDatabase = ( + arm: BenchmarkArm, + pass: number, + purpose: 'first-read' | 'first-write' | 'pool-ready', + runtime: MemoryDatabaseStackBenchmarkRuntime +) => { + const dbFilename = `ms-${armFilenameCode(arm)}-${purpose}-${pass}-${crypto.randomUUID().slice(0, 8)}.db`; + + return arm.vfs === 'InMemoryVFS' + ? runtime.createInMemoryDatabase({ dbFilename }) + : runtime.createMemoryWriteAheadDatabase({ + additionalReaders: arm.additionalReaders, + dbFilename, + workerUrl: runtime.workerUrl, + }); +}; + +const closeDatabase = async (arm: BenchmarkArm, database: DBAdapter, runtime: MemoryDatabaseStackBenchmarkRuntime) => { + await Promise.resolve(database.close()).catch(() => undefined); + resetRuntimeForArm(arm, runtime); +}; + +const measureFirstOperation = async ( + arm: BenchmarkArm, + pass: number, + operation: 'first-read' | 'first-write', + runtime: MemoryDatabaseStackBenchmarkRuntime +) => { + const startedAt = performance.now(); + const database = openDatabase(arm, pass, operation, runtime); + + try { + if (operation === 'first-read') { + await database.readLock((tx) => tx.execute('SELECT 1')); + } else { + await database.writeLock((tx) => + tx.execute('CREATE TABLE benchmark_open_probe (id INTEGER PRIMARY KEY, value TEXT)') + ); + } + + return performance.now() - startedAt; + } finally { + await closeDatabase(arm, database, runtime); + } +}; + +const createPoolReadyDatabase = async ( + arm: BenchmarkArm, + pass: number, + runtime: MemoryDatabaseStackBenchmarkRuntime +) => { + const startedAt = performance.now(); + const database = openDatabase(arm, pass, 'pool-ready', runtime); + + try { + const waitForAdditionalReaderPool = Reflect.get(database, 'waitForAdditionalReaderPool'); + const init = Reflect.get(database, 'init'); + + if (typeof waitForAdditionalReaderPool === 'function') { + await waitForAdditionalReaderPool.call(database); + } else if (typeof init === 'function') { + await init.call(database); + } else { + throw new Error(`${arm.name} adapter does not expose a readiness method`); + } + + return { + database, + poolReadyMs: performance.now() - startedAt, + }; + } catch (error) { + await closeDatabase(arm, database, runtime); + throw error; + } +}; + +const createDatabase = async (arm: BenchmarkArm, pass: number, runtime: MemoryDatabaseStackBenchmarkRuntime) => { + try { + const firstReadMs = await measureFirstOperation(arm, pass, 'first-read', runtime); + const firstWriteMs = await measureFirstOperation(arm, pass, 'first-write', runtime); + const { database, poolReadyMs } = await createPoolReadyDatabase(arm, pass, runtime); + + return { + database, + firstReadMs, + firstWriteMs, + poolReadyMs, + }; + } catch (error) { + const code = typeof error === 'object' && error !== null && 'code' in error ? String(error.code) : 'unknown'; + runtime.onProgress(`${arm.name} pass ${pass}: startup probe failed with SQLite code ${code}`); + throw error; + } +}; + +const seedDatabase = async (database: DBAdapter) => { + const totalRecords = DATASET.collections * DATASET.recordsPerCollection; + const totalLabels = DATASET.collections * DATASET.labelsPerCollection; + + await database.writeLock((tx) => + executeStatements(tx, [ + 'CREATE TABLE collection (id TEXT PRIMARY KEY, accountId TEXT NOT NULL, sourceId TEXT NOT NULL, deletedAt INTEGER)', + 'CREATE TABLE label (id TEXT PRIMARY KEY, accountId TEXT NOT NULL, nodeLevel INTEGER NOT NULL, collectionId TEXT NOT NULL, parentId TEXT, "order" INTEGER NOT NULL, deletedAt INTEGER, name TEXT NOT NULL)', + 'CREATE TABLE run (id TEXT PRIMARY KEY, accountId TEXT NOT NULL, groupId TEXT NOT NULL, createdAt INTEGER NOT NULL, deletedAt INTEGER)', + 'CREATE TABLE collection_to_group (id TEXT PRIMARY KEY, A TEXT NOT NULL, B TEXT NOT NULL)', + 'CREATE TABLE record (id TEXT PRIMARY KEY, collectionId TEXT NOT NULL, deletedAt INTEGER, archivedAt INTEGER, imported INTEGER NOT NULL, status TEXT NOT NULL, category TEXT NOT NULL)', + 'CREATE TABLE run_record (id TEXT PRIMARY KEY, runId TEXT NOT NULL, recordId TEXT NOT NULL, state TEXT NOT NULL, deletedAt INTEGER)', + 'CREATE TABLE run_collection (id TEXT PRIMARY KEY, runId TEXT NOT NULL, collectionId TEXT NOT NULL)', + ` + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < ${DATASET.collections}) + INSERT INTO collection (id, accountId, sourceId, deletedAt) + SELECT printf('collection-%03d', n), 'account-01', printf('source-collection-%03d', n), NULL FROM seq + `, + ` + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < ${DATASET.collections}) + INSERT INTO collection_to_group (id, A, B) + SELECT + printf('collection-%03d.group-%02d', n, ((n - 1) % ${DATASET.groups}) + 1), + printf('collection-%03d', n), + printf('group-%02d', ((n - 1) % ${DATASET.groups}) + 1) + FROM seq + `, + ` + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < ${DATASET.runs}) + INSERT INTO run (id, accountId, groupId, createdAt, deletedAt) + SELECT + printf('run-%03d', n), + 'account-01', + printf('group-%02d', ((n - 1) % ${DATASET.groups}) + 1), + n, + NULL + FROM seq + `, + ` + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < ${totalLabels}) + INSERT INTO label (id, accountId, nodeLevel, collectionId, parentId, "order", deletedAt, name) + SELECT + printf('label-%06d', n), + 'account-01', + CASE WHEN n % 5 = 0 THEN 4 ELSE 3 END, + printf('collection-%03d', ((n - 1) / ${DATASET.labelsPerCollection}) + 1), + NULL, + ((n - 1) % ${DATASET.labelsPerCollection}) + 1, + CASE WHEN n % 97 = 0 THEN 1 ELSE NULL END, + printf('Label %06d', n) + FROM seq + `, + ` + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < ${totalRecords}) + INSERT INTO record (id, collectionId, deletedAt, archivedAt, imported, status, category) + SELECT + printf('record-%06d', n), + printf('collection-%03d', ((n - 1) / ${DATASET.recordsPerCollection}) + 1), + CASE WHEN n % 151 = 0 THEN 1 ELSE NULL END, + CASE WHEN n % 37 = 0 THEN 1 ELSE NULL END, + CASE WHEN n % 3 = 0 THEN 1 ELSE 0 END, + CASE n % 5 WHEN 0 THEN 'DEPOSIT' WHEN 1 THEN 'UNTRACEABLE' ELSE 'ACTIVE' END, + CASE n % 4 WHEN 0 THEN 'CONFIRMED' WHEN 1 THEN 'EXCLUDED' ELSE 'PENDING' END + FROM seq + `, + ` + INSERT INTO run_record (id, runId, recordId, state, deletedAt) + SELECT + c.id || '.' || e.id, + c.id, + e.id, + CASE ((CAST(substr(e.id, 11) AS INTEGER) + c.createdAt) % 3) + WHEN 0 THEN 'DONE' + WHEN 1 THEN 'ONGOING' + ELSE 'TODO' + END, + NULL + FROM run c + INNER JOIN collection_to_group bw ON bw.B = c.groupId + INNER JOIN record e ON e.collectionId = bw.A + WHERE (CAST(substr(e.id, 11) AS INTEGER) % 2) = (c.createdAt % 2) + `, + ` + INSERT INTO run_collection (id, runId, collectionId) + SELECT c.id || '.' || bw.A, c.id, bw.A + FROM run c + INNER JOIN collection_to_group bw ON bw.B = c.groupId + WHERE ((CAST(substr(bw.A, 10) AS INTEGER) + c.createdAt) % 5) = 0 + `, + 'CREATE INDEX idx_label_account_node_deleted_order ON label(accountId, nodeLevel, deletedAt, "order")', + 'CREATE INDEX idx_label_collection ON label(collectionId)', + 'CREATE INDEX idx_run_account_deleted_created ON run(accountId, deletedAt, createdAt)', + 'CREATE INDEX idx_btw_group_collection ON collection_to_group(B, A)', + 'CREATE INDEX idx_record_collection_deleted ON record(collectionId, deletedAt)', + 'CREATE INDEX idx_run_record_run_record_deleted ON run_record(runId, recordId, deletedAt, state)', + 'CREATE INDEX idx_run_collection_run_collection ON run_collection(runId, collectionId)', + ]) + ); +}; + +const queryChecksum = (rows: readonly Record[]) => + rows.reduce( + (sum, row) => + sum + + Object.values(row).reduce( + (rowSum, value) => rowSum + (typeof value === 'number' ? value : String(value ?? '').length), + 0 + ), + 0 + ); + +const serializeQueryResultValue = (value: unknown): readonly unknown[] => { + if (value === null) { + return ['null']; + } + if (value instanceof Uint8Array) { + return ['blob', ...value]; + } + if (typeof value === 'bigint') { + return ['bigint', value.toString()]; + } + if (typeof value === 'number' || typeof value === 'string') { + return [typeof value, value]; + } + + throw new Error(`Unsupported benchmark query result value: ${Object.prototype.toString.call(value)}`); +}; + +const fingerprintQueryRows = async (rows: readonly Record[]) => { + const canonicalRows = rows.map((row) => + JSON.stringify( + Object.entries(row) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([column, value]) => [column, ...serializeQueryResultValue(value)]) + ) + ); + const digest = new Uint8Array( + await crypto.subtle.digest('SHA-256', new TextEncoder().encode(JSON.stringify(canonicalRows))) + ); + + return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join(''); +}; + +const fingerprintReadQueryResults = (database: DBAdapter) => + READ_QUERIES.reduce>( + (previous, sql) => + previous.then(async (fingerprints) => { + const result = await database.readLock((tx) => tx.execute(sql)); + const rows = (result.rows?._array ?? []) as Record[]; + fingerprints.push(await fingerprintQueryRows(rows)); + return fingerprints; + }), + Promise.resolve([]) + ); + +const validateCrossArmQueryResults = async ( + arm: BenchmarkArm, + database: DBAdapter, + baselineState: QueryResultBaselineState +) => { + const fingerprints = await fingerprintReadQueryResults(database); + const baseline = baselineState.current; + + if (!baseline) { + baselineState.current = { armName: arm.name, fingerprints }; + return; + } + + const mismatchedQueries = fingerprints.flatMap((fingerprint, index) => + fingerprint === baseline.fingerprints[index] ? [] : [index + 1] + ); + if (mismatchedQueries.length > 0) { + throw new Error( + `${arm.name} query results differ from ${baseline.armName} for queries ${mismatchedQueries.join(', ')}` + ); + } +}; + +const executeRead = (database: DBAdapter, sql: string) => + database.readLock(async (tx) => { + const result = await tx.execute(sql); + const rows = (result.rows?._array ?? []) as Record[]; + return rows.length + queryChecksum(rows); + }); + +const runSequentialTrial = async (database: DBAdapter) => { + const startedAt = performance.now(); + const checksums = await READ_QUERIES.reduce>( + (previous, sql) => + previous.then(async (values) => { + values.push(await executeRead(database, sql)); + return values; + }), + Promise.resolve([]) + ); + + return { + checksum: checksums.reduce((sum, value) => sum + value, 0), + elapsedMs: performance.now() - startedAt, + }; +}; + +const runBurstTrial = async (database: DBAdapter) => { + const startedAt = performance.now(); + const checksums = await Promise.all(READ_QUERIES.map((sql) => executeRead(database, sql))); + + return { + checksum: checksums.reduce((sum, value) => sum + value, 0), + elapsedMs: performance.now() - startedAt, + }; +}; + +const runWriteTrial = async (database: DBAdapter, iteration: number) => { + const startedAt = performance.now(); + + const result = await database.writeLock((tx) => + tx.execute(` + UPDATE label + SET parentId = CASE parentId + WHEN 'benchmark-parent-a' THEN 'benchmark-parent-b' + ELSE 'benchmark-parent-a' + END + WHERE id IN ( + SELECT id + FROM label + WHERE collectionId = 'collection-${String((iteration % DATASET.collections) + 1).padStart(3, '0')}' + ORDER BY id + LIMIT 40 + ) + `) + ); + + if (result.rowsAffected !== 40) { + throw new Error(`Benchmark write updated ${result.rowsAffected} rows instead of 40`); + } + + return performance.now() - startedAt; +}; + +const verifyDataset = async (database: DBAdapter) => { + const result = await database.readLock((tx) => + tx.execute(` + SELECT + (SELECT COUNT(*) FROM collection) AS collections, + (SELECT COUNT(*) FROM collection_to_group) AS collectionGroups, + (SELECT COUNT(*) FROM run) AS runs, + (SELECT COUNT(*) FROM record) AS records, + (SELECT COUNT(*) FROM label) AS labels, + (SELECT COUNT(*) FROM run_record) AS runRecords, + (SELECT COUNT(*) FROM run_collection) AS runCollections + `) + ); + const row = (result.rows?._array?.[0] ?? {}) as Record; + + if ( + Number(row.collections) !== DATASET.collections || + Number(row.collectionGroups) !== DATASET.collections || + Number(row.runs) !== DATASET.runs || + Number(row.records) !== DATASET.collections * DATASET.recordsPerCollection || + Number(row.labels) !== DATASET.collections * DATASET.labelsPerCollection || + Number(row.runRecords) !== 30_000 || + Number(row.runCollections) !== 10 + ) { + throw new Error(`Benchmark dataset verification failed: ${JSON.stringify(row)}`); + } +}; + +const verifyMixedWrites = async (database: DBAdapter) => { + const result = await database.writeLock((tx) => + tx.execute(` + SELECT COUNT(*) AS updatedLabels + FROM label + WHERE parentId = 'benchmark-parent-a' + AND collectionId IN (${inList( + Array.from({ length: MIXED_TRIALS }, (_, index) => `collection-${String(index + 1).padStart(3, '0')}`) + )}) + `) + ); + const row = (result.rows?._array?.[0] ?? {}) as Record; + if (Number(row.updatedLabels) !== MIXED_TRIALS * 40) { + throw new Error(`Benchmark final write verification failed: ${JSON.stringify(row)}`); + } +}; + +const verifyAdditionalReadersAreQueryOnly = async (arm: BenchmarkArm, database: DBAdapter) => { + if (arm.vfs !== 'MemoryWriteAheadVFS' || arm.additionalReaders === 0) { + return; + } + + let rejected = false; + try { + await database.readLock((tx) => + tx.execute("UPDATE label SET parentId = 'unexpected-reader-write' WHERE id = 'label-000001'") + ); + } catch (_error) { + rejected = true; + } + + if (!rejected) { + throw new Error(`${arm.name} allowed a mutation through an additional reader`); + } + + const result = await database.writeLock((tx) => tx.execute("SELECT parentId FROM label WHERE id = 'label-000001'")); + const row = (result.rows?._array?.[0] ?? {}) as Record; + if (row.parentId !== null) { + throw new Error(`${arm.name} changed data through an additional reader`); + } +}; + +const runArmPass = async ( + arm: BenchmarkArm, + pass: number, + samples: BenchmarkSamples, + runtime: MemoryDatabaseStackBenchmarkRuntime, + queryResultBaselineState: QueryResultBaselineState +) => { + runtime.onProgress(`${arm.name} pass ${pass}: measuring adapter startup`); + const { database, firstReadMs, firstWriteMs, poolReadyMs } = await createDatabase(arm, pass, runtime); + samples.firstReadMs.push(firstReadMs); + samples.firstWriteMs.push(firstWriteMs); + samples.poolReadyMs.push(poolReadyMs); + + try { + runtime.onProgress(`${arm.name} pass ${pass}: seeding`); + const seedStartedAt = performance.now(); + await seedDatabase(database); + samples.seedMs.push(performance.now() - seedStartedAt); + await database.refreshSchema(); + await verifyDataset(database); + await verifyAdditionalReadersAreQueryOnly(arm, database); + runtime.onProgress(`${arm.name} pass ${pass}: validating cross-arm query results`); + await validateCrossArmQueryResults(arm, database, queryResultBaselineState); + + runtime.onProgress(`${arm.name} pass ${pass}: warming up`); + const warmupChecksums = await Promise.all( + Array.from({ length: WARMUP_TRIALS }, async () => (await runBurstTrial(database)).checksum) + ); + const expectedChecksum = warmupChecksums[0]; + + if (warmupChecksums.some((checksum) => checksum !== expectedChecksum)) { + throw new Error(`${arm.name} produced inconsistent warmup checksums`); + } + + runtime.onProgress(`${arm.name} pass ${pass}: sequential reads`); + const sequentialPassMs: number[] = []; + for (let trial = 0; trial < SEQUENTIAL_TRIALS; trial += 1) { + const result = await runSequentialTrial(database); + if (result.checksum !== expectedChecksum) { + throw new Error(`${arm.name} produced an inconsistent sequential checksum`); + } + sequentialPassMs.push(result.elapsedMs); + } + samples.sequentialMs.push(...sequentialPassMs); + + runtime.onProgress(`${arm.name} pass ${pass}: burst reads`); + const burstPassMs: number[] = []; + for (let trial = 0; trial < BURST_TRIALS; trial += 1) { + const result = await runBurstTrial(database); + if (result.checksum !== expectedChecksum) { + throw new Error(`${arm.name} produced an inconsistent burst checksum`); + } + burstPassMs.push(result.elapsedMs); + } + samples.burstMs.push(...burstPassMs); + + runtime.onProgress(`${arm.name} pass ${pass}: mixed reads and writes`); + const mixedReadPassMs: number[] = []; + const mixedWritePassMs: number[] = []; + for (let trial = 0; trial < MIXED_TRIALS; trial += 1) { + const [readResult, writeElapsedMs] = await Promise.all([ + runBurstTrial(database), + runWriteTrial(database, trial), + ]); + if (readResult.checksum !== expectedChecksum) { + throw new Error(`${arm.name} produced an inconsistent mixed-workload checksum`); + } + mixedReadPassMs.push(readResult.elapsedMs); + mixedWritePassMs.push(writeElapsedMs); + } + samples.mixedReadMs.push(...mixedReadPassMs); + samples.mixedWriteMs.push(...mixedWritePassMs); + await verifyMixedWrites(database); + } finally { + runtime.onProgress(`${arm.name} pass ${pass}: closing`); + await database.close(); + resetRuntimeForArm(arm, runtime); + runtime.onProgress(`${arm.name} pass ${pass}: complete`); + } +}; + +const runStartupOnlyPass = async ( + arm: BenchmarkArm, + pass: number, + samples: BenchmarkSamples, + runtime: MemoryDatabaseStackBenchmarkRuntime +) => { + runtime.onProgress(`${arm.name} pass ${pass}: measuring adapter startup`); + const { database, firstReadMs, firstWriteMs, poolReadyMs } = await createDatabase(arm, pass, runtime); + samples.firstReadMs.push(firstReadMs); + samples.firstWriteMs.push(firstWriteMs); + samples.poolReadyMs.push(poolReadyMs); + await database.close(); + resetRuntimeForArm(arm, runtime); + runtime.onProgress(`${arm.name} pass ${pass}: startup measurements complete`); +}; + +export const runMemoryDatabaseStackComparisonBenchmark = async ( + runtime: MemoryDatabaseStackBenchmarkRuntime +): Promise => { + const samplesByArm = new Map( + ARMS.map((arm) => [arm.name, createEmptySamples()]) + ); + const queryResultBaselineState: QueryResultBaselineState = {}; + + for (let pass = 0; pass < OPEN_PASSES; pass += 1) { + const offset = pass % ARMS.length; + const rotatedArms = [...ARMS.slice(offset), ...ARMS.slice(0, offset)]; + const passArms = Math.floor(pass / ARMS.length) % 2 === 0 ? rotatedArms : rotatedArms.reverse(); + + for (const arm of passArms) { + const samples = samplesByArm.get(arm.name); + if (!samples) { + throw new Error(`Missing benchmark sample collector for ${arm.name}`); + } + if (pass < WORKLOAD_PASSES) { + await runArmPass(arm, pass + 1, samples, runtime, queryResultBaselineState); + } else { + await runStartupOnlyPass(arm, pass + 1, samples, runtime); + } + } + } + + return { + arms: ARMS.map((arm) => { + const samples = samplesByArm.get(arm.name) ?? createEmptySamples(); + return { + additionalReaderCount: arm.additionalReaders, + aggregateCacheSizeKiB: runtime.cacheSizeKiB * arm.connectionCount, + burst: summarize(samples.burstMs), + connectionCount: arm.connectionCount, + firstRead: summarize(samples.firstReadMs), + firstWrite: summarize(samples.firstWriteMs), + mixedRead: summarize(samples.mixedReadMs), + mixedWrite: summarize(samples.mixedWriteMs), + name: arm.name, + poolReady: summarize(samples.poolReadyMs), + seed: summarize(samples.seedMs), + sequential: summarize(samples.sequentialMs), + vfs: arm.vfs, + }; + }), + comparisonScope: 'implementation-stack', + configuration: { + burstTrials: BURST_TRIALS, + cacheSizeKiB: runtime.cacheSizeKiB, + correctnessValidation: [ + 'dataset-row-counts', + 'cross-arm-full-result-sha256', + 'per-trial-checksum', + 'write-row-count', + 'final-write-values', + 'additional-reader-query-only', + ], + dataset: DATASET, + mixedTrials: MIXED_TRIALS, + openPasses: OPEN_PASSES, + queriesPerTrial: READ_QUERIES.length, + sequentialTrials: SEQUENTIAL_TRIALS, + warmupTrials: WARMUP_TRIALS, + workloadPasses: WORKLOAD_PASSES, + }, + environment: { + crossOriginIsolated, + hardwareConcurrency: navigator.hardwareConcurrency, + userAgent: navigator.userAgent, + }, + generatedAt: new Date().toISOString(), + harness: runtime.harness, + runtimeVersions: runtime.versions, + schemaVersion: 1, + }; +}; diff --git a/benchmarks/memory-vfs-comparison/runtime.ts b/benchmarks/memory-vfs-comparison/runtime.ts new file mode 100644 index 0000000..ea8b273 --- /dev/null +++ b/benchmarks/memory-vfs-comparison/runtime.ts @@ -0,0 +1,52 @@ +import { TemporaryStorageOption, WASQLiteOpenFactory, WASQLiteVFS } from '@powersync/web'; + +import { createMemoryWriteAheadSharedRuntime, resetMemoryWriteAheadSharedRuntime } from '../../src/index.js'; +import { createMemoryWriteAheadBenchmarkDatabase } from './mwaConnection.js'; +import { runMemoryDatabaseStackComparisonBenchmark, type MemoryDatabaseStackBenchmarkRuntime } from './runner.js'; + +const CACHE_SIZE_KIB = 32 * 1024; + +type BenchmarkFlags = { + memoryWriteAheadUseWebWorker: boolean; + powerSyncUseWebWorker: boolean; +}; + +type BenchmarkVersions = MemoryDatabaseStackBenchmarkRuntime['versions']; + +export const runPublicMemoryVfsBenchmark = (flags: BenchmarkFlags, versions: BenchmarkVersions) => { + const workerUrl = new URL('./mwaWorker.ts', import.meta.url).toString(); + + return runMemoryDatabaseStackComparisonBenchmark({ + cacheSizeKiB: CACHE_SIZE_KIB, + createInMemoryDatabase: ({ dbFilename }) => + new WASQLiteOpenFactory({ + cacheSizeKb: CACHE_SIZE_KIB, + dbFilename, + debugMode: false, + flags: { + enableMultiTabs: false, + useWebWorker: flags.powerSyncUseWebWorker, + }, + temporaryStorage: TemporaryStorageOption.MEMORY, + vfs: WASQLiteVFS.InMemoryVfs, + }).openDB(), + createMemoryWriteAheadDatabase: ({ additionalReaders, dbFilename }) => + createMemoryWriteAheadBenchmarkDatabase({ + additionalReaders, + cacheSizeKiB: CACHE_SIZE_KIB, + dbFilename, + runtime: createMemoryWriteAheadSharedRuntime(dbFilename), + useWebWorker: flags.memoryWriteAheadUseWebWorker, + workerUrl, + }), + harness: { + memoryWriteAheadUseWebWorker: flags.memoryWriteAheadUseWebWorker, + powerSyncUseWebWorker: flags.powerSyncUseWebWorker, + webServer: 'vite-development', + }, + onProgress: (message) => console.info(`[memory-vfs-benchmark] ${message}`), + resetMemoryWriteAheadRuntime: () => resetMemoryWriteAheadSharedRuntime(), + versions, + workerUrl, + }); +}; diff --git a/package.json b/package.json index a39fdc6..3e531fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "memory-write-ahead-vfs", - "version": "0.1.2", + "version": "0.2.0", "description": "SharedArrayBuffer-backed in-memory write-ahead VFS for wa-sqlite.", "type": "module", "license": "MIT", @@ -13,7 +13,8 @@ "files": [ "dist", "README.md", - "LICENSE" + "LICENSE", + "THIRD_PARTY_NOTICES" ], "exports": { ".": { @@ -25,14 +26,17 @@ "types": "./dist/index.d.ts", "scripts": { "build": "tsc -p tsconfig.build.json && node scripts/prepare-dist.mjs", + "benchmark": "playwright test --config playwright.benchmark.config.ts", + "benchmark:publish-result": "node scripts/publish-benchmark-result.mjs", "check": "tsc -p tsconfig.json --noEmit", "format": "prettier --write .", "format:check": "prettier --check .", "test": "playwright test", "test:browser": "playwright test", + "test:benchmark-publisher": "node --test scripts/publish-benchmark-result.test.mjs", "test:stress": "playwright test --repeat-each=20 --workers=1", "prepack": "pnpm build", - "prepublishOnly": "pnpm format:check && pnpm check && pnpm test:browser && pnpm build" + "prepublishOnly": "pnpm format:check && pnpm check && pnpm test:benchmark-publisher && pnpm test:browser && pnpm build" }, "keywords": [ "sqlite", @@ -43,12 +47,15 @@ "browser" ], "peerDependencies": { - "@journeyapps/wa-sqlite": ">=1.7.0" + "@journeyapps/wa-sqlite": ">=1.7.0 <1.8.0" }, "devDependencies": { "@journeyapps/wa-sqlite": "^1.7.0", "@playwright/test": "^1.51.1", + "@powersync/common": "1.57.2", + "@powersync/web": "1.39.0", "@types/node": "^20.19.0", + "comlink": "4.4.2", "prettier": "^3.8.4", "typescript": "^5.9.3", "vite": "^8.0.1" diff --git a/playwright.benchmark.config.ts b/playwright.benchmark.config.ts new file mode 100644 index 0000000..b310d1d --- /dev/null +++ b/playwright.benchmark.config.ts @@ -0,0 +1,28 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './benchmarks/memory-vfs-comparison', + testMatch: 'benchmark.spec.ts', + timeout: 30 * 60_000, + expect: { + timeout: 5_000, + }, + fullyParallel: false, + workers: 1, + use: { + baseURL: 'http://127.0.0.1:5174', + trace: 'retain-on-failure', + }, + webServer: { + command: 'pnpm exec vite --host 127.0.0.1 --port 5174', + url: 'http://127.0.0.1:5174/test-host.html', + reuseExistingServer: !process.env.CI, + timeout: 20_000, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 250f441..2584e74 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,9 +13,18 @@ importers: '@playwright/test': specifier: ^1.51.1 version: 1.60.0 + '@powersync/common': + specifier: 1.57.2 + version: 1.57.2 + '@powersync/web': + specifier: 1.39.0 + version: 1.39.0(@journeyapps/wa-sqlite@1.7.0)(@powersync/common@1.57.2) '@types/node': specifier: ^20.19.0 version: 20.19.42 + comlink: + specifier: 4.4.2 + version: 4.4.2 prettier: specifier: ^3.8.4 version: 3.8.4 @@ -60,6 +69,18 @@ packages: engines: { node: '>=18' } hasBin: true + '@powersync/common@1.57.2': + resolution: + { integrity: sha512-cOsmvBFBK0yvpwjaI46Lm3xCMRaTaydQQ4RolED3Upj6/kawGZHhkh4Guve3nmOSQ/0sGCuiFOcl4Gv9Mp6dOA== } + + '@powersync/web@1.39.0': + resolution: + { integrity: sha512-hl0V8wC4T44cLKG89/ePB97DsVboyVFkT5/WT9SxhIVQNs+zCaTg800UKQscNchUJyVjNHQwTdycjYsKBpqnOQ== } + hasBin: true + peerDependencies: + '@journeyapps/wa-sqlite': ^1.5.0 + '@powersync/common': ^1.57.2 + '@rolldown/binding-android-arm64@1.0.3': resolution: { integrity: sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw== } @@ -176,6 +197,15 @@ packages: resolution: { integrity: sha512-5L7SUaFC1RyDraj2yRhyBzHTobyXHmohD100CChNtyPyleoq37Mqab5Gn8XEKI04dfN/oqPdpHk38MgcQWHbZg== } + comlink@4.4.2: + resolution: + { integrity: sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g== } + + commander@12.1.0: + resolution: + { integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA== } + engines: { node: '>=18' } + detect-libc@2.1.2: resolution: { integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== } @@ -203,6 +233,10 @@ packages: engines: { node: ^8.16.0 || ^10.6.0 || >=11.0.0 } os: [darwin] + js-logger@1.6.1: + resolution: + { integrity: sha512-yTgMCPXVjhmg28CuUH8CKjU+cIKL/G+zTu4Fn4lQxs8mRFH/03QTNvEFngcxfg/gRDiQAOoyCKmMTOm9ayOzXA== } + lightningcss-android-arm64@1.32.0: resolution: { integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg== } @@ -429,6 +463,17 @@ snapshots: dependencies: playwright: 1.60.0 + '@powersync/common@1.57.2': + dependencies: + js-logger: 1.6.1 + + '@powersync/web@1.39.0(@journeyapps/wa-sqlite@1.7.0)(@powersync/common@1.57.2)': + dependencies: + '@journeyapps/wa-sqlite': 1.7.0 + '@powersync/common': 1.57.2 + comlink: 4.4.2 + commander: 12.1.0 + '@rolldown/binding-android-arm64@1.0.3': optional: true @@ -489,6 +534,10 @@ snapshots: dependencies: undici-types: 6.21.0 + comlink@4.4.2: {} + + commander@12.1.0: {} + detect-libc@2.1.2: {} fdir@6.5.0(picomatch@4.0.4): @@ -501,6 +550,8 @@ snapshots: fsevents@2.3.3: optional: true + js-logger@1.6.1: {} + lightningcss-android-arm64@1.32.0: optional: true diff --git a/scripts/prepare-dist.mjs b/scripts/prepare-dist.mjs index 2a3e21b..6bd0574 100644 --- a/scripts/prepare-dist.mjs +++ b/scripts/prepare-dist.mjs @@ -1,6 +1,8 @@ -import { copyFileSync, readFileSync, writeFileSync } from 'node:fs'; +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; copyFileSync('src/wa-sqlite-private.d.ts', 'dist/wa-sqlite-private.d.ts'); +mkdirSync('dist/vendor', { recursive: true }); +copyFileSync('src/vendor/WriteAhead.js', 'dist/vendor/WriteAhead.js'); const indexDeclarationPath = 'dist/index.d.ts'; const reference = '/// \n'; diff --git a/scripts/publish-benchmark-result.mjs b/scripts/publish-benchmark-result.mjs new file mode 100644 index 0000000..84be559 --- /dev/null +++ b/scripts/publish-benchmark-result.mjs @@ -0,0 +1,292 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join, parse, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { isDeepStrictEqual } from 'node:util'; + +const SCRIPT_PATH = fileURLToPath(import.meta.url); +const REPOSITORY_ROOT = resolve(dirname(SCRIPT_PATH), '..'); +const ROOT_PACKAGE = JSON.parse(readFileSync(join(REPOSITORY_ROOT, 'package.json'), 'utf8')); +const CORRECTNESS_VALIDATION = [ + 'dataset-row-counts', + 'cross-arm-full-result-sha256', + 'per-trial-checksum', + 'write-row-count', + 'final-write-values', + 'additional-reader-query-only', +]; +const DATASET = { + collections: 30, + runs: 12, + recordsPerCollection: 1_000, + labelsPerCollection: 100, + groups: 6, +}; +const EXPECTED_ARMS = [ + { + additionalReaderCount: 0, + connectionCount: 1, + name: 'powersync-memory-vfs-1', + vfs: 'InMemoryVFS', + }, + { + additionalReaderCount: 0, + connectionCount: 1, + name: 'memory-write-ahead-1', + vfs: 'MemoryWriteAheadVFS', + }, + { + additionalReaderCount: 1, + connectionCount: 2, + name: 'memory-write-ahead-2', + vfs: 'MemoryWriteAheadVFS', + }, + { + additionalReaderCount: 2, + connectionCount: 3, + name: 'memory-write-ahead-3', + vfs: 'MemoryWriteAheadVFS', + }, + { + additionalReaderCount: 3, + connectionCount: 4, + name: 'memory-write-ahead-4', + vfs: 'MemoryWriteAheadVFS', + }, +]; +const EXPECTED_CONFIGURATION = { + burstTrials: 12, + cacheSizeKiB: 32 * 1_024, + correctnessValidation: CORRECTNESS_VALIDATION, + dataset: DATASET, + mixedTrials: 8, + openPasses: 30, + queriesPerTrial: 8, + sequentialTrials: 10, + warmupTrials: 2, + workloadPasses: 10, +}; +const SUMMARY_SAMPLES = { + burst: 120, + firstRead: 30, + firstWrite: 30, + mixedRead: 80, + mixedWrite: 80, + poolReady: 30, + seed: 10, + sequential: 100, +}; + +const assertCondition = (condition, message) => { + if (!condition) { + throw new Error(`Invalid benchmark result: ${message}`); + } +}; + +const assertExactKeys = (value, expectedKeys, label) => { + assertCondition(value && typeof value === 'object' && !Array.isArray(value), `${label} must be an object`); + const actualKeys = Object.keys(value).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + assertCondition( + isDeepStrictEqual(actualKeys, sortedExpectedKeys), + `${label} keys must be ${sortedExpectedKeys.join(', ')}; received ${actualKeys.join(', ')}` + ); +}; + +const readInstalledPackageVersion = (packageName) => { + let directory = dirname(fileURLToPath(import.meta.resolve(packageName))); + const root = parse(directory).root; + + while (directory !== root) { + const packagePath = join(directory, 'package.json'); + if (existsSync(packagePath)) { + const packageJson = JSON.parse(readFileSync(packagePath, 'utf8')); + if (packageJson.name === packageName && typeof packageJson.version === 'string') { + return packageJson.version; + } + } + directory = dirname(directory); + } + + throw new Error(`Unable to resolve installed version for ${packageName}`); +}; + +export const getExpectedRuntimeVersions = () => ({ + memoryWriteAheadVfs: ROOT_PACKAGE.version, + playwright: readInstalledPackageVersion('@playwright/test'), + powerSyncCommon: readInstalledPackageVersion('@powersync/common'), + powerSyncWeb: readInstalledPackageVersion('@powersync/web'), + vite: readInstalledPackageVersion('vite'), + waSqlite: readInstalledPackageVersion('@journeyapps/wa-sqlite'), +}); + +const validateSummary = (summary, expectedSamples, label) => { + assertExactKeys(summary, ['max', 'mean', 'min', 'p50', 'p95', 'samples'], label); + const values = ['max', 'mean', 'min', 'p50', 'p95'].map((key) => summary[key]); + assertCondition( + values.every((value) => Number.isFinite(value) && value >= 0), + `${label} must be finite` + ); + assertCondition(Number.isInteger(summary.samples), `${label}.samples must be an integer`); + assertCondition(summary.samples === expectedSamples, `${label}.samples must be ${expectedSamples}`); + assertCondition( + summary.min <= summary.p50 && summary.p50 <= summary.p95 && summary.p95 <= summary.max, + `${label} percentiles must satisfy min <= p50 <= p95 <= max` + ); + assertCondition( + summary.min <= summary.mean && summary.mean <= summary.max, + `${label}.mean must be between min and max` + ); +}; + +export const sanitizeBenchmarkResult = (input, expectedFlags) => { + assertExactKeys( + input, + [ + 'arms', + 'comparisonScope', + 'configuration', + 'environment', + 'generatedAt', + 'harness', + 'runtimeVersions', + 'schemaVersion', + ], + 'root' + ); + assertCondition(input.schemaVersion === 1, 'schemaVersion must be 1'); + assertCondition(input.comparisonScope === 'implementation-stack', 'comparisonScope must be implementation-stack'); + assertCondition( + typeof input.generatedAt === 'string' && new Date(input.generatedAt).toISOString() === input.generatedAt, + 'generatedAt must be an ISO timestamp' + ); + + assertExactKeys(input.environment, ['crossOriginIsolated', 'hardwareConcurrency', 'userAgent'], 'environment'); + const chromeVersion = /Chrome\/([^ ]+)/.exec(input.environment.userAgent)?.[1]; + assertCondition(Boolean(chromeVersion), 'userAgent must contain a recognizable Chromium version'); + assertCondition(input.environment.crossOriginIsolated === true, 'crossOriginIsolated must be true'); + assertCondition( + Number.isInteger(input.environment.hardwareConcurrency) && input.environment.hardwareConcurrency > 0, + 'hardwareConcurrency must be a positive integer' + ); + + const expectedHarness = { + memoryWriteAheadUseWebWorker: expectedFlags.memoryWriteAheadUseWebWorker, + powerSyncUseWebWorker: expectedFlags.powerSyncUseWebWorker, + webServer: 'vite-development', + }; + assertCondition(isDeepStrictEqual(input.harness, expectedHarness), 'harness flags do not match the requested run'); + assertCondition( + isDeepStrictEqual(input.runtimeVersions, getExpectedRuntimeVersions()), + 'runtimeVersions do not match installed dependencies' + ); + assertCondition( + isDeepStrictEqual(input.configuration, EXPECTED_CONFIGURATION), + 'configuration does not match the published harness contract' + ); + + assertCondition( + Array.isArray(input.arms) && input.arms.length === EXPECTED_ARMS.length, + 'arms must contain 5 entries' + ); + const armNames = input.arms.map((arm) => arm?.name); + assertCondition(new Set(armNames).size === EXPECTED_ARMS.length, 'arm names must be unique'); + input.arms.forEach((arm, index) => { + assertExactKeys( + arm, + [ + 'additionalReaderCount', + 'aggregateCacheSizeKiB', + 'burst', + 'connectionCount', + 'firstRead', + 'firstWrite', + 'mixedRead', + 'mixedWrite', + 'name', + 'poolReady', + 'seed', + 'sequential', + 'vfs', + ], + `arms[${index}]` + ); + const expectedArm = EXPECTED_ARMS[index]; + assertCondition( + expectedArm && + arm.name === expectedArm.name && + arm.vfs === expectedArm.vfs && + arm.connectionCount === expectedArm.connectionCount && + arm.additionalReaderCount === expectedArm.additionalReaderCount, + `arms[${index}] topology does not match the harness` + ); + assertCondition( + arm.aggregateCacheSizeKiB === EXPECTED_CONFIGURATION.cacheSizeKiB * arm.connectionCount, + `arms[${index}].aggregateCacheSizeKiB does not match its connection count` + ); + Object.entries(SUMMARY_SAMPLES).forEach(([name, samples]) => { + validateSummary(arm[name], samples, `arms[${index}].${name}`); + }); + }); + + return { + resultFormat: 'memory-vfs-implementation-stack-v1', + schemaVersion: input.schemaVersion, + runDate: input.generatedAt.slice(0, 10), + comparisonScope: input.comparisonScope, + environment: { + browser: `Chromium ${chromeVersion}`, + crossOriginIsolated: input.environment.crossOriginIsolated, + hardwareConcurrency: input.environment.hardwareConcurrency, + }, + harness: input.harness, + runtimeVersions: input.runtimeVersions, + configuration: input.configuration, + arms: input.arms, + }; +}; + +const serializePublicResult = (result) => { + const serialized = `${JSON.stringify(result, null, 2)}\n`; + const forbiddenPatterns = [/\/Users\//i, /\\Users\\/i, /\.codex/i, /beeldi/i, /generatedAt/i, /userAgent/i]; + + for (const pattern of forbiddenPatterns) { + if (pattern.test(serialized)) { + throw new Error(`Refusing to publish benchmark result matching ${pattern}`); + } + } + return serialized; +}; + +export const publishBenchmarkResultFile = async (inputPath, outputPath, expectedFlags) => { + const input = JSON.parse(await readFile(resolve(inputPath), 'utf8')); + const serialized = serializePublicResult(sanitizeBenchmarkResult(input, expectedFlags)); + const resolvedOutputPath = resolve(outputPath); + await mkdir(dirname(resolvedOutputPath), { recursive: true }); + await writeFile(resolvedOutputPath, serialized, 'utf8'); +}; + +const parseBoolean = (value, label) => { + if (value === 'true') { + return true; + } + if (value === 'false') { + return false; + } + throw new Error(`${label} must be true or false; received ${value}`); +}; + +if (process.argv[1] && resolve(process.argv[1]) === SCRIPT_PATH) { + const [inputArgument, outputArgument, powerSyncWorkerArgument, memoryWriteAheadWorkerArgument] = + process.argv.slice(2); + if (!inputArgument || !outputArgument || !powerSyncWorkerArgument || !memoryWriteAheadWorkerArgument) { + throw new Error( + 'Usage: pnpm benchmark:publish-result ' + ); + } + + await publishBenchmarkResultFile(inputArgument, outputArgument, { + memoryWriteAheadUseWebWorker: parseBoolean(memoryWriteAheadWorkerArgument, 'mwa-worker'), + powerSyncUseWebWorker: parseBoolean(powerSyncWorkerArgument, 'powersync-worker'), + }); +} diff --git a/scripts/publish-benchmark-result.test.mjs b/scripts/publish-benchmark-result.test.mjs new file mode 100644 index 0000000..f3b48d6 --- /dev/null +++ b/scripts/publish-benchmark-result.test.mjs @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; +import { getExpectedRuntimeVersions, sanitizeBenchmarkResult } from './publish-benchmark-result.mjs'; + +const createValidRawResult = async () => { + const publicResult = JSON.parse( + await readFile( + new URL('../benchmarks/memory-vfs-comparison/results/both-workers.json', import.meta.url), + 'utf8' + ) + ); + const { resultFormat: _resultFormat, runDate, ...rawResult } = publicResult; + rawResult.generatedAt = `${runDate}T12:00:00.000Z`; + rawResult.environment = { + crossOriginIsolated: publicResult.environment.crossOriginIsolated, + hardwareConcurrency: publicResult.environment.hardwareConcurrency, + userAgent: `Mozilla/5.0 Chrome/${publicResult.environment.browser.replace('Chromium ', '')} Safari/537.36`, + }; + rawResult.runtimeVersions = getExpectedRuntimeVersions(); + rawResult.configuration.correctnessValidation = [ + ...rawResult.configuration.correctnessValidation.filter((name) => name !== 'additional-reader-query-only'), + 'additional-reader-query-only', + ]; + rawResult.arms.forEach((arm) => { + arm.aggregateCacheSizeKiB = rawResult.configuration.cacheSizeKiB * arm.connectionCount; + }); + return rawResult; +}; + +const WORKER_FLAGS = { + memoryWriteAheadUseWebWorker: true, + powerSyncUseWebWorker: true, +}; + +test('sanitizes a complete benchmark result', async () => { + const result = sanitizeBenchmarkResult(await createValidRawResult(), WORKER_FLAGS); + + assert.equal(result.resultFormat, 'memory-vfs-implementation-stack-v1'); + assert.equal(result.environment.browser.startsWith('Chromium '), true); + assert.equal('userAgent' in result.environment, false); + assert.equal('generatedAt' in result, false); +}); + +test('rejects stale versions, malformed summaries, topology drift, and wrong flags', async (context) => { + await context.test('stale dependency version', async () => { + const input = await createValidRawResult(); + input.runtimeVersions.powerSyncWeb = '0.0.0'; + assert.throws(() => sanitizeBenchmarkResult(input, WORKER_FLAGS), /runtimeVersions/); + }); + + await context.test('wrong sample count', async () => { + const input = await createValidRawResult(); + input.arms[0].burst.samples -= 1; + assert.throws(() => sanitizeBenchmarkResult(input, WORKER_FLAGS), /samples must be 120/); + }); + + await context.test('non-finite latency', async () => { + const input = await createValidRawResult(); + input.arms[0].sequential.p50 = Number.NaN; + assert.throws(() => sanitizeBenchmarkResult(input, WORKER_FLAGS), /must be finite/); + }); + + await context.test('aggregate cache mismatch', async () => { + const input = await createValidRawResult(); + input.arms[2].aggregateCacheSizeKiB = 32_768; + assert.throws(() => sanitizeBenchmarkResult(input, WORKER_FLAGS), /aggregateCacheSizeKiB/); + }); + + await context.test('wrong worker flags', async () => { + const input = await createValidRawResult(); + assert.throws( + () => + sanitizeBenchmarkResult(input, { + memoryWriteAheadUseWebWorker: true, + powerSyncUseWebWorker: false, + }), + /harness flags/ + ); + }); +}); + +test('published README tables match committed result artifacts', async () => { + const readme = await readFile(new URL('../benchmarks/memory-vfs-comparison/README.md', import.meta.url), 'utf8'); + const labels = { + 'powersync-memory-vfs-1': 'PowerSync MemoryVFS — 1 connection', + 'memory-write-ahead-1': 'Memory Write-Ahead VFS — 1 connection', + 'memory-write-ahead-2': 'Memory Write-Ahead VFS — 2-connection pool', + 'memory-write-ahead-3': 'Memory Write-Ahead VFS — 3-connection pool', + 'memory-write-ahead-4': 'Memory Write-Ahead VFS — 4-connection pool', + }; + + for (const resultName of ['both-workers', 'application-workers', 'both-main-thread']) { + const result = JSON.parse( + await readFile( + new URL(`../benchmarks/memory-vfs-comparison/results/${resultName}.json`, import.meta.url), + 'utf8' + ) + ); + for (const arm of result.arms) { + const cells = ['sequential', 'burst', 'mixedRead', 'mixedWrite', 'poolReady'] + .map((metric) => `${arm[metric].p50.toFixed(2)} / ${arm[metric].p95.toFixed(2)}`) + .join('\\s*\\|\\s*'); + const label = labels[arm.name].replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + assert.match(readme, new RegExp(`\\|\\s*${label}\\s*\\|\\s*${cells}\\s*\\|`)); + } + } +}); diff --git a/src/MemoryWriteAheadVFS.ts b/src/MemoryWriteAheadVFS.ts index aa3dc8d..8d9c344 100644 --- a/src/MemoryWriteAheadVFS.ts +++ b/src/MemoryWriteAheadVFS.ts @@ -1,8 +1,8 @@ import { FacadeVFS } from '@journeyapps/wa-sqlite/src/FacadeVFS.js'; import * as VFS from '@journeyapps/wa-sqlite/src/VFS.js'; import { LazyLock } from '@journeyapps/wa-sqlite/src/examples/LazyLock.js'; -import { WriteAhead } from '@journeyapps/wa-sqlite/src/examples/WriteAhead.js'; import { type MemoryWriteAheadLogger, resolveMemoryWriteAheadLogger } from './logger.js'; +import { WriteAhead } from './vendor/WriteAhead.js'; import { MEMORY_WRITE_AHEAD_FILE_META, @@ -13,8 +13,11 @@ import { getMemoryWriteAheadFileSizeBytes, getMemoryWriteAheadLockAgeMs, getMemoryWriteAheadLockClockMs, + getMemoryWriteAheadRuntimeOwnerId, normalizeMemoryWriteAheadPathname, readMemoryWriteAheadFile, + recordMemoryWriteAheadCheckpoint, + recordMemoryWriteAheadFileLockWait, registerMemoryWriteAheadRuntime, releaseMemoryWriteAheadRuntimeHandle, resetMemoryWriteAheadSharedRuntime, @@ -40,6 +43,9 @@ type MemoryWriteAheadAccessHandle = { type MemoryWriteAheadOpenFile = { accessHandle?: MemoryWriteAheadAccessHandle; + attachedTransactionLockHeld?: boolean; + attachedTransactionLockOwnerId?: number; + attachedTransactionLockRuntimeOwnerId?: number; flags: number; lockingMode?: 'exclusive' | 'normal'; lockState?: number; @@ -49,6 +55,7 @@ type MemoryWriteAheadOpenFile = { readLock?: LazyLock; retryResult?: Error | Record | null; synchronous?: 0 | 1 | 2 | 3; + sharedFile?: MemoryWriteAheadSharedFile; timeout?: number; useLazyLock?: 'none' | 'read' | 'readwrite' | 'write'; waHandles?: [MemoryWriteAheadAccessHandle, MemoryWriteAheadAccessHandle]; @@ -67,6 +74,12 @@ export type MemoryWriteAheadVFSOptions = { const FILE_LOCK_WAIT_TIMEOUT_MS = 50; const FILE_LOCK_UNLOCKED = 0; +class MemoryWriteAheadBusyError extends Error { + override readonly name = 'MemoryWriteAheadBusyError'; +} + +const getWaitNowMs = () => (typeof performance !== 'undefined' ? performance.now() : Date.now()); + const sanitizeTraceValue = (value: unknown): unknown => { if (typeof value !== 'string') { return value; @@ -119,7 +132,7 @@ const createFileLockOwnerId = () => { return Math.floor(Math.random() * 0x3fffffff) + 1; }; -// Exported from this module for browser regression tests; the package root does not expose these protocol internals. +// Exported from this module for browser regression tests; the library index does not expose these protocol internals. // File lock word: 0 means unlocked, +ownerId means held, -ownerId means a recovery claim owns cleanup. // The heartbeat belongs to the positive lock owner or the in-flight acquirer that just observed an unlocked word. // Contenders must never refresh the heartbeat while another owner or recovery claim is visible. @@ -212,7 +225,10 @@ export const acquireMemoryWriteAheadFileLock = ( staleMs: number, logger?: MemoryWriteAheadLogger ) => { - while (true) { + let waitCount = 0; + let waitMs = 0; + + for (;;) { if (Atomics.load(view, MEMORY_WRITE_AHEAD_FILE_META.lock) === FILE_LOCK_UNLOCKED) { Atomics.store(view, MEMORY_WRITE_AHEAD_FILE_META.lockHeartbeatMs, getMemoryWriteAheadLockClockMs()); @@ -221,12 +237,17 @@ export const acquireMemoryWriteAheadFileLock = ( FILE_LOCK_UNLOCKED ) { Atomics.store(view, MEMORY_WRITE_AHEAD_FILE_META.lockOwner, ownerId); - return; + return { waitCount, waitMs }; } } recoverAbandonedMemoryWriteAheadFileLock(view, staleMs, logger); - waitForUnlock(view); + if (Atomics.load(view, MEMORY_WRITE_AHEAD_FILE_META.lock) !== FILE_LOCK_UNLOCKED) { + const waitStartMs = getWaitNowMs(); + waitForUnlock(view); + waitCount += 1; + waitMs += getWaitNowMs() - waitStartMs; + } } }; @@ -276,7 +297,13 @@ class SABMemoryAccessHandle implements MemoryWriteAheadAccessHandle { } #withFileLock(operation: (refreshHeartbeat: () => void) => T) { - acquireMemoryWriteAheadFileLock(this.#metaView, this.#lockOwnerId, this.#staleMs, this.#logger); + const acquisition = acquireMemoryWriteAheadFileLock( + this.#metaView, + this.#lockOwnerId, + this.#staleMs, + this.#logger + ); + recordMemoryWriteAheadFileLockWait(this.file, acquisition.waitCount, acquisition.waitMs); try { return operation(() => refreshMemoryWriteAheadFileLockHeartbeat(this.#metaView, this.#lockOwnerId)); } finally { @@ -303,10 +330,12 @@ class SABMemoryAccessHandle implements MemoryWriteAheadAccessHandle { } class LocalMemoryAccessHandle implements MemoryWriteAheadAccessHandle { + #closed = false; + constructor(private readonly file: MemoryWriteAheadLocalFile) {} close() { - return; + this.#closed = true; } flush() { @@ -389,6 +418,10 @@ export class MemoryWriteAheadVFS extends FacadeVFS { resetMemoryWriteAheadSharedRuntime(dbFilename); } + getRuntimeOwnerId() { + return getMemoryWriteAheadRuntimeOwnerId(this.sharedRuntime); + } + #getModule() { return ( this as unknown as { @@ -410,7 +443,9 @@ export class MemoryWriteAheadVFS extends FacadeVFS { const module = this.#getModule(); const normalizedName = normalizeMemoryWriteAheadPathname(zName); - if (flags & VFS.SQLITE_OPEN_MAIN_DB) { + // The runtime's primary database owns the WriteAhead state and must not be shared between + // SQLite connections through one VFS instance; reopening after jClose is supported. + if (flags & VFS.SQLITE_OPEN_MAIN_DB && normalizedName === this.sharedRuntime.dbFilename) { const existingFile = this.mapPathToFile.get(normalizedName); // retryResult === null on an unopened existing file means the first open's #retryOpen is // still in flight; wa-sqlite only retries after that op settles, so another open arriving @@ -468,8 +503,39 @@ export class MemoryWriteAheadVFS extends FacadeVFS { this.mapPathToFile.set(normalizedName, file); shouldDeletePathOnError = true; - if (flags & (VFS.SQLITE_OPEN_WAL | VFS.SQLITE_OPEN_SUPER_JOURNAL)) { - throw new Error('WAL and super-journal files are managed internally'); + if (flags & VFS.SQLITE_OPEN_MAIN_DB) { + if (!file.accessHandle) { + const sharedFile = getMemoryWriteAheadFile(this.sharedRuntime, normalizedName); + if (!sharedFile) { + throw new Error( + `Missing shared runtime file for ${normalizedName}; register attached database files before opening it` + ); + } + file.sharedFile = sharedFile; + file.accessHandle = new SABMemoryAccessHandle(sharedFile, this.sharedRuntime, this.#logger); + file.retryResult = null; + } + + file.lockState = VFS.SQLITE_LOCK_NONE; + file.lockingMode = 'normal'; + file.readLock ??= new LazyLock(`${normalizedName}#read`); + file.writeLock ??= new LazyLock(`${normalizedName}#write`); + file.useLazyLock = 'readwrite'; + file.timeout = -1; + file.synchronous = 1; + file.writeHint = null; + file.pageSize = null; + file.overwrite = false; + file.attachedTransactionLockOwnerId ??= createFileLockOwnerId(); + const runtimeOwnerId = getMemoryWriteAheadRuntimeOwnerId(this.sharedRuntime); + if (!runtimeOwnerId) { + throw new Error(`Missing runtime owner for attached database ${normalizedName}`); + } + file.attachedTransactionLockRuntimeOwnerId ??= runtimeOwnerId; + } else if (flags & VFS.SQLITE_OPEN_WAL) { + throw new Error('WAL files are managed internally'); + } else if (flags & VFS.SQLITE_OPEN_SUPER_JOURNAL) { + file.accessHandle = this.#openTemporaryHandle(normalizedName, true); } else if (!file.accessHandle) { file.accessHandle = this.#openTemporaryHandle(normalizedName, !!(flags & VFS.SQLITE_OPEN_CREATE)); } @@ -543,6 +609,7 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) { + this.#releaseAttachedDatabaseLock(file); file.writeAhead?.close(); file.accessHandle?.close(); file.waHandles?.forEach((handle) => handle.close()); @@ -616,9 +683,13 @@ export class MemoryWriteAheadVFS extends FacadeVFS { if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) { const isPageResize = file.overwrite && file.pageSize !== pData.byteLength; - file.writeAhead?.write(iOffset, pData, { - dstPageSize: isPageResize ? (file.pageSize ?? null) : null, - }); + if (file.writeAhead) { + file.writeAhead.write(iOffset, pData, { + dstPageSize: isPageResize ? (file.pageSize ?? null) : null, + }); + } else { + file.accessHandle?.write(pData.subarray(), { at: iOffset }); + } return VFS.SQLITE_OK; } @@ -645,7 +716,11 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) { - file.writeAhead?.truncate(iSize); + if (file.writeAhead) { + file.writeAhead.truncate(iSize); + } else { + file.accessHandle?.truncate(iSize); + } return VFS.SQLITE_OK; } @@ -709,6 +784,27 @@ export class MemoryWriteAheadVFS extends FacadeVFS { return VFS.SQLITE_IOERR_LOCK; } + if (file.flags & VFS.SQLITE_OPEN_MAIN_DB && !file.writeAhead) { + if (file.lockState === VFS.SQLITE_LOCK_NONE && lockType === VFS.SQLITE_LOCK_SHARED) { + if (file.retryResult instanceof Error) { + const error = file.retryResult; + file.retryResult = null; + throw error; + } + + if (!file.attachedTransactionLockHeld && !this.#tryLockAttachedDatabase(file)) { + file.retryResult = {}; + module.retryOps.push(this.#retryLockAttachedDatabase(file)); + return VFS.SQLITE_BUSY; + } + + file.retryResult = null; + } + + file.lockState = lockType; + return VFS.SQLITE_OK; + } + if (file.lockState === VFS.SQLITE_LOCK_NONE && lockType === VFS.SQLITE_LOCK_SHARED) { if (file.retryResult === null) { if (file.lockingMode === 'exclusive') { @@ -737,14 +833,39 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } file.retryResult = null; - } else if (lockType >= VFS.SQLITE_LOCK_RESERVED && !file.writeLock?.mode) { - throw new Error('Write transaction cannot use BEGIN DEFERRED'); + } else if (lockType >= VFS.SQLITE_LOCK_RESERVED && file.writeAhead) { + if (file.retryResult instanceof Error) { + const error = file.retryResult; + file.retryResult = null; + throw error; + } + + if (file.writeAhead.isolationState !== 'write') { + const holdsWriteLock = + file.writeAhead.isolationState === 'read' && file.writeLock?.acquireIfHeld('exclusive'); + + if (holdsWriteLock) { + if (!file.writeAhead.tryUpgradeReadIsolationToWrite()) { + file.writeLock?.release(); + throw new MemoryWriteAheadBusyError( + 'Deferred write snapshot is stale; retry the transaction' + ); + } + } else { + file.retryResult = {}; + module.retryOps.push(this.#retryUpgradeDeferredWrite(file)); + return VFS.SQLITE_BUSY; + } + } + + file.retryResult = null; } file.lockState = lockType; return VFS.SQLITE_OK; } catch (error) { - if ((error as Error).name === 'TimeoutError') { + if ((error as Error).name === 'TimeoutError' || (error as Error).name === 'MemoryWriteAheadBusyError') { + this.lastError = error as Error; return VFS.SQLITE_BUSY; } @@ -761,6 +882,13 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } if (!file.retryResult && lockType === VFS.SQLITE_LOCK_NONE) { + if (file.flags & VFS.SQLITE_OPEN_MAIN_DB && !file.writeAhead) { + this.#releaseAttachedDatabaseLock(file); + file.writeHint = null; + file.lockState = lockType; + return VFS.SQLITE_OK; + } + file.writeAhead?.rejoin(); switch (file.useLazyLock) { @@ -1000,6 +1128,8 @@ export class MemoryWriteAheadVFS extends FacadeVFS { async #pendingCheckpoint(file: MemoryWriteAheadOpenFile, mode: 'full' | 'passive' | 'restart' | 'truncate') { const onFinally: Array<() => void> = []; + const startedAtMs = getWaitNowMs(); + const writeAheadPages = file.writeAhead?.getWriteAheadSize() ?? 0; try { if (mode !== 'passive') { await file.writeLock?.acquire('exclusive'); @@ -1010,6 +1140,10 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } await file.writeAhead?.checkpoint({ isPassive: mode === 'passive' }); + const sharedFile = getMemoryWriteAheadFile(this.sharedRuntime, file.zName); + if (sharedFile && file.writeAhead) { + recordMemoryWriteAheadCheckpoint(sharedFile, getWaitNowMs() - startedAtMs, writeAheadPages); + } } finally { while (onFinally.length) { onFinally.pop()?.(); @@ -1020,7 +1154,9 @@ export class MemoryWriteAheadVFS extends FacadeVFS { async #retryLockRead(file: MemoryWriteAheadOpenFile) { const onError: Array<() => void> = []; try { - await file.readLock?.acquire('shared', file.timeout); + if (!(await file.readLock?.acquire('shared', file.timeout))) { + throw new MemoryWriteAheadBusyError(`Timed out acquiring read lock for ${file.zName}`); + } onError.push(() => file.readLock?.release()); file.writeAhead?.isolateForRead(); @@ -1037,11 +1173,15 @@ export class MemoryWriteAheadVFS extends FacadeVFS { const onError: Array<() => void> = []; try { if (file.lockingMode === 'exclusive') { - await file.readLock?.acquire('exclusive', file.timeout); + if (!(await file.readLock?.acquire('exclusive', file.timeout))) { + throw new MemoryWriteAheadBusyError(`Timed out acquiring exclusive read lock for ${file.zName}`); + } onError.push(() => file.readLock?.release()); } - await file.writeLock?.acquire('exclusive', file.timeout); + if (!(await file.writeLock?.acquire('exclusive', file.timeout))) { + throw new MemoryWriteAheadBusyError(`Timed out acquiring write lock for ${file.zName}`); + } onError.push(() => file.writeLock?.release()); file.writeAhead?.isolateForWrite(); @@ -1054,6 +1194,94 @@ export class MemoryWriteAheadVFS extends FacadeVFS { } } + async #retryLockAttachedDatabase(file: MemoryWriteAheadOpenFile) { + try { + const deadline = file.timeout && file.timeout > 0 ? getWaitNowMs() + file.timeout : null; + while (!this.#tryLockAttachedDatabase(file)) { + if (file.timeout === 0 || (deadline !== null && getWaitNowMs() >= deadline)) { + throw new MemoryWriteAheadBusyError(`Timed out acquiring attached database lock for ${file.zName}`); + } + await new Promise((resolve) => setTimeout(resolve, 1)); + } + file.retryResult = {}; + } catch (error) { + this.#releaseAttachedDatabaseLock(file); + file.retryResult = error as Error; + } + } + + #tryLockAttachedDatabase(file: MemoryWriteAheadOpenFile) { + if (file.attachedTransactionLockHeld) { + return true; + } + + const sharedFile = file.sharedFile; + const lockOwnerId = file.attachedTransactionLockOwnerId; + const runtimeOwnerId = file.attachedTransactionLockRuntimeOwnerId; + if (!sharedFile || !lockOwnerId || !runtimeOwnerId) { + throw new Error(`Missing attached database lock state for ${file.zName}`); + } + + const meta = new Int32Array(sharedFile.meta); + if (Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.abandonedTransaction) !== 0) { + throw new MemoryWriteAheadBusyError( + `Attached database ${file.zName} is quarantined after its transaction owner terminated; reset and reload the runtime from a trusted source` + ); + } + + const claim = -lockOwnerId; + if (Atomics.compareExchange(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock, 0, claim) !== 0) { + return false; + } + + Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLockRuntimeOwner, runtimeOwnerId); + if (Atomics.compareExchange(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock, claim, lockOwnerId) !== claim) { + throw new Error(`Lost attached database lock claim for ${file.zName}`); + } + + file.attachedTransactionLockHeld = true; + return true; + } + + #releaseAttachedDatabaseLock(file: MemoryWriteAheadOpenFile) { + if (!file.attachedTransactionLockHeld || !file.sharedFile || !file.attachedTransactionLockOwnerId) { + return; + } + + const meta = new Int32Array(file.sharedFile.meta); + const releaseClaim = -file.attachedTransactionLockOwnerId; + if ( + Atomics.compareExchange( + meta, + MEMORY_WRITE_AHEAD_FILE_META.transactionLock, + file.attachedTransactionLockOwnerId, + releaseClaim + ) === file.attachedTransactionLockOwnerId + ) { + Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLockRuntimeOwner, 0); + Atomics.compareExchange(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock, releaseClaim, 0); + Atomics.notify(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock); + } + file.attachedTransactionLockHeld = false; + } + + async #retryUpgradeDeferredWrite(file: MemoryWriteAheadOpenFile) { + try { + if (!(await file.writeLock?.acquire('exclusive', file.timeout))) { + throw new MemoryWriteAheadBusyError(`Timed out acquiring write lock for ${file.zName}`); + } + + if (!requireWriteAhead(file).tryUpgradeReadIsolationToWrite()) { + throw new MemoryWriteAheadBusyError('Deferred write snapshot is stale; retry the transaction'); + } + + file.retryResult = {}; + } catch (error) { + file.writeLock?.release(); + file.retryResult = error as Error; + } + } + async #retryOpen(file: MemoryWriteAheadOpenFile) { let accessHandle: MemoryWriteAheadAccessHandle | undefined; let waHandles: [MemoryWriteAheadAccessHandle, MemoryWriteAheadAccessHandle] | undefined; diff --git a/src/index.ts b/src/index.ts index 7c7ed18..0587fc2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,13 +3,20 @@ export { MEMORY_WRITE_AHEAD_VFS, MemoryWriteAheadVFS, type MemoryWriteAheadVFSOptions } from './MemoryWriteAheadVFS.js'; export { createMemoryWriteAheadSharedRuntime, + ensureMemoryWriteAheadDatabaseFiles, + getMemoryWriteAheadRuntimeOwnerId, getMemoryWriteAheadRuntimeDiagnostics, normalizeMemoryWriteAheadPathname, + recoverTerminatedMemoryWriteAheadRuntimeOwner, + registerMemoryWriteAheadSharedFiles, resetMemoryWriteAheadSharedRuntime, resolveMemoryWriteAheadRuntimeOptions, + type MemoryWriteAheadDatabaseFiles, type MemoryWriteAheadResolvedRuntimeOptions, type MemoryWriteAheadRuntimeDiagnostics, type MemoryWriteAheadRuntimeOptions, + type MemoryWriteAheadSharedFile, + type MemoryWriteAheadSharedFilesRegistration, type MemoryWriteAheadSharedRuntime, } from './memoryWriteAheadSharedRuntime.js'; export { noopMemoryWriteAheadLogger, type MemoryWriteAheadLogger } from './logger.js'; diff --git a/src/memoryWriteAheadSharedRuntime.ts b/src/memoryWriteAheadSharedRuntime.ts index a532545..8845df9 100644 --- a/src/memoryWriteAheadSharedRuntime.ts +++ b/src/memoryWriteAheadSharedRuntime.ts @@ -9,7 +9,6 @@ const DEFAULT_MAX_WRITE_AHEAD_CAPACITY_BYTES = 512 * MIB; const DEFAULT_FILE_LOCK_STALE_MS = 30_000; const LOCK_CLOCK_MODULO_MS = 0x7fffffff; const MAX_INT32_ATOMIC_COUNTER = 0x7fffffff; -const MAX_ATOMIC_FILE_SIZE_BYTES = MAX_INT32_ATOMIC_COUNTER; export const MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT = 32; const MIN_SEGMENT_COUNT = 1; @@ -24,6 +23,22 @@ export const MEMORY_WRITE_AHEAD_FILE_META = { lockOwner: 5, lockHeartbeatMs: 6, recoveredAbandonedLockCount: 7, + peakLogicalSizeBytes: 8, + peakCapacityBytes: 9, + peakAllocatedCapacityBytes: 10, + writeOperationCount: 11, + writtenBytes: 12, + truncateOperationCount: 13, + lockWaitCount: 14, + lockWaitMs: 15, + checkpointCount: 16, + checkpointMs: 17, + checkpointWriteAheadPages: 18, + lastCheckpointMs: 19, + lastCheckpointWriteAheadPages: 20, + transactionLock: 21, + transactionLockRuntimeOwner: 22, + abandonedTransaction: 23, } as const; export const MEMORY_WRITE_AHEAD_RUNTIME_META = { @@ -58,11 +73,37 @@ export type MemoryWriteAheadResolvedRuntimeOptions = { fileLockStaleMs: number; }; -export type MemoryWriteAheadSharedSegment = { - data: SharedArrayBuffer; - maxCapacityBytes: number; +type GrowableSharedArrayBuffer = SharedArrayBuffer & { + readonly maxByteLength: number; + grow(byteLength: number): void; +}; + +type GrowableSharedArrayBufferConstructor = { + new (byteLength: number, options: { maxByteLength: number }): SharedArrayBuffer; }; +export type MemoryWriteAheadSharedSegment = + | { + capacityMode: 'growable'; + data: GrowableSharedArrayBuffer; + maxCapacityBytes: number; + } + | { + capacityMode: 'fixed'; + data: SharedArrayBuffer; + maxCapacityBytes: number; + }; + +type MemoryWriteAheadSharedSegmentData = + | { + capacityMode: 'growable'; + data: GrowableSharedArrayBuffer; + } + | { + capacityMode: 'fixed'; + data: SharedArrayBuffer; + }; + export type MemoryWriteAheadSharedFile = { dbFilename: string; maxCapacityBytes: number; @@ -79,6 +120,16 @@ export type MemoryWriteAheadSharedRuntime = { registryMeta: SharedArrayBuffer; }; +export type MemoryWriteAheadDatabaseFiles = { + database: MemoryWriteAheadSharedFile; + writeAhead: [MemoryWriteAheadSharedFile, MemoryWriteAheadSharedFile]; +}; + +export type MemoryWriteAheadSharedFilesRegistration = { + files: MemoryWriteAheadSharedFile[]; + runtimeDbFilename: string; +}; + export type MemoryWriteAheadRuntimeDiagnostics = { activeOwnerLeaseCount: number; capacities: MemoryWriteAheadResolvedRuntimeOptions; @@ -86,6 +137,7 @@ export type MemoryWriteAheadRuntimeDiagnostics = { files: { allocatedCapacityBytes: number; capacityBytes: number; + currentCapacityBytes: number; logicalSizeBytes: number; maxCapacityBytes: number; pathname: string; @@ -94,10 +146,26 @@ export type MemoryWriteAheadRuntimeDiagnostics = { segmentCapacityBytes: number; segmentCount: number; totalSegmentCount: number; + peakLogicalSizeBytes: number; + peakCapacityBytes: number; + peakAllocatedCapacityBytes: number; + writeOperationCount: number; + writtenBytes: number; + truncateOperationCount: number; + lockWaitCount: number; + lockWaitMs: number; + checkpointCount: number; + checkpointMs: number; + checkpointWriteAheadPages: number; + lastCheckpointMs: number; + lastCheckpointWriteAheadPages: number; lockState: number; lockOwner: number; lockHeartbeatAgeMs: number | null; recoveredAbandonedLockCount: number; + abandonedTransaction: boolean; + transactionLockRuntimeOwner: number; + transactionLockState: number; }[]; openHandleCount: number; recoveredAbandonedHandleCount: number; @@ -108,24 +176,15 @@ type MemoryWriteAheadGlobalScope = typeof globalThis & { __MEMORY_WRITE_AHEAD_VFS_RUNTIMES__?: Map; }; -type SharedArrayBufferConstructorWithGrowth = { - new (byteLength: number, options?: { maxByteLength?: number }): SharedArrayBuffer; -}; - -type GrowableSharedArrayBuffer = SharedArrayBuffer & { - grow?: (byteLength: number) => void; - maxByteLength?: number; -}; - type LocalRuntimeLease = { handleCount: number; heartbeatIntervalId: ReturnType; ownerId: number; + runtimeDbFilename: string; slotIndex: number; }; const localRuntimeLeases = new Map(); -const fileMetaViews = new WeakMap(); const assertPositiveInteger = (value: number, label: string) => { if (!Number.isSafeInteger(value) || value <= 0) { @@ -135,11 +194,12 @@ const assertPositiveInteger = (value: number, label: string) => { return value; }; +// File sizes and reclaim counters live in atomic Int32Array slots, so capacities above 2^31 - 1 would wrap. const assertPositiveCapacityBytes = (value: number, label: string) => { const capacityBytes = assertPositiveInteger(value, label); - if (capacityBytes > MAX_ATOMIC_FILE_SIZE_BYTES) { - throw new Error(`${label} cannot exceed ${MAX_ATOMIC_FILE_SIZE_BYTES} bytes`); + if (capacityBytes > MAX_INT32_ATOMIC_COUNTER) { + throw new Error(`${label} cannot exceed ${MAX_INT32_ATOMIC_COUNTER} bytes`); } return capacityBytes; @@ -244,6 +304,23 @@ const addSaturatingAtomicCounter = (view: Int32Array, index: number, delta: numb return current; }; +const updateAtomicMaxCounter = (view: Int32Array, index: number, value: number) => { + const nextValue = Math.max(0, Math.min(MAX_INT32_ATOMIC_COUNTER, Math.trunc(value))); + let current = Atomics.load(view, index); + + while (current < nextValue) { + const previous = Atomics.compareExchange(view, index, current, nextValue); + + if (previous === current) { + return nextValue; + } + + current = previous; + } + + return current; +}; + const getOwnerLeaseHeartbeatIntervalMs = (runtime: MemoryWriteAheadSharedRuntime) => Math.max(50, Math.min(1_000, Math.trunc(runtime.capacities.fileLockStaleMs / 3))); @@ -326,6 +403,28 @@ const reduceOpenHandleCount = (runtime: MemoryWriteAheadSharedRuntime, releasedH return 0; }; +const quarantineTransactionLocksForRuntimeOwner = (runtime: MemoryWriteAheadSharedRuntime, ownerId: number) => { + let quarantinedTransactionCount = 0; + + Object.values(runtime.files).forEach((file) => { + const meta = new Int32Array(file.meta); + if (Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLockRuntimeOwner) !== ownerId) { + return; + } + + const lockState = Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock); + if (lockState !== 0) { + // Attached databases use rollback journals stored inside the owning VFS context. If that context + // terminates mid-transaction, releasing its shared lock would expose pages whose rollback journal + // disappeared with the context. Keep the file fail-closed until the application resets the runtime. + Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.abandonedTransaction, 1); + quarantinedTransactionCount += 1; + } + }); + + return quarantinedTransactionCount; +}; + const recoverAbandonedRuntimeOwnerLeases = ( runtime: MemoryWriteAheadSharedRuntime, logger: MemoryWriteAheadLogger = memoryWriteAheadRuntimeLogger @@ -361,6 +460,7 @@ const recoverAbandonedRuntimeOwnerLeases = ( } const releasedHandleCount = reduceOpenHandleCount(runtime, Atomics.load(runtimeMeta, handleCountIndex)); + const quarantinedTransactionCount = quarantineTransactionLocksForRuntimeOwner(runtime, ownerId); clearOwnerLeaseSlot(runtimeMeta, slotIndex); Atomics.add(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.recoveredAbandonedHandleCount, releasedHandleCount); recoveredHandleCount += releasedHandleCount; @@ -368,6 +468,7 @@ const recoverAbandonedRuntimeOwnerLeases = ( dbFilename: runtime.dbFilename, ownerId, recoveredHandleCount: releasedHandleCount, + quarantinedTransactionCount, slotIndex, staleAgeMs, }); @@ -448,6 +549,7 @@ const ensureLocalRuntimeLease = ( } }, getOwnerLeaseHeartbeatIntervalMs(runtime)), ownerId, + runtimeDbFilename: runtime.dbFilename, slotIndex: freeSlotIndex, }; @@ -460,31 +562,65 @@ const ensureLocalRuntimeLease = ( return lease; }; -const createGrowableSharedArrayBuffer = ( +// Without COOP/COEP response headers a browser context is not cross-origin isolated and exposes +// no SharedArrayBuffer at all; the generic capability error would hide that deployment mistake. +const createGrowableSharedArrayBufferSupportError = () => { + if (globalThis.crossOriginIsolated === false) { + const error = new Error( + 'MemoryWriteAheadVFS requires growable SharedArrayBuffer support: this page is not cross-origin isolated. Serve it with the Cross-Origin-Opener-Policy: same-origin and Cross-Origin-Embedder-Policy: require-corp response headers.' + ); + memoryWriteAheadRuntimeLogger.error('MemoryWriteAheadVFS unavailable without cross-origin isolation', { + crossOriginIsolated: false, + error: error.message, + }); + return error; + } + + return new Error('MemoryWriteAheadVFS requires growable SharedArrayBuffer support'); +}; + +const createPotentiallyGrowableSharedArrayBuffer = (initialBytes: number, maxBytes: number) => { + const Constructor = SharedArrayBuffer as unknown as GrowableSharedArrayBufferConstructor; + + return new Constructor(initialBytes, { maxByteLength: maxBytes }); +}; + +const isGrowableSharedArrayBuffer = ( + buffer: SharedArrayBuffer, + maxBytes: number +): buffer is GrowableSharedArrayBuffer => { + const candidate = buffer as Partial; + + return candidate.maxByteLength === maxBytes && typeof candidate.grow === 'function'; +}; + +const createSharedSegmentData = ( initialBytes: number, maxBytes: number, allowFixedCapacityFallback: boolean -) => { - const constructor = SharedArrayBuffer as unknown as SharedArrayBufferConstructorWithGrowth; - +): MemoryWriteAheadSharedSegmentData => { try { - const buffer = new constructor(initialBytes, { - maxByteLength: maxBytes, - }) as GrowableSharedArrayBuffer; - if (buffer.maxByteLength === maxBytes && typeof buffer.grow === 'function') { - return buffer; + const buffer = createPotentiallyGrowableSharedArrayBuffer(initialBytes, maxBytes); + if (isGrowableSharedArrayBuffer(buffer, maxBytes)) { + return { + capacityMode: 'growable', + data: buffer, + }; } } catch (_error) { if (!allowFixedCapacityFallback) { - throw new Error('MemoryWriteAheadVFS requires growable SharedArrayBuffer support'); + throw createGrowableSharedArrayBufferSupportError(); } } if (!allowFixedCapacityFallback) { - throw new Error('MemoryWriteAheadVFS requires growable SharedArrayBuffer support'); + throw createGrowableSharedArrayBufferSupportError(); } - return new SharedArrayBuffer(maxBytes) as GrowableSharedArrayBuffer; + return { + capacityMode: 'fixed', + data: new SharedArrayBuffer(maxBytes), + }; }; const createSharedSegment = ( @@ -492,7 +628,7 @@ const createSharedSegment = ( maxBytes: number, allowFixedCapacityFallback: boolean ): MemoryWriteAheadSharedSegment => ({ - data: createGrowableSharedArrayBuffer(initialBytes, maxBytes, allowFixedCapacityFallback), + ...createSharedSegmentData(initialBytes, maxBytes, allowFixedCapacityFallback), maxCapacityBytes: maxBytes, }); @@ -523,6 +659,27 @@ const createSharedFile = ( }; }; +const ensureSharedFile = ( + runtime: MemoryWriteAheadSharedRuntime, + pathname: string, + initialCapacityBytes: number, + maxCapacityBytes: number +) => { + const normalizedPathname = normalizeMemoryWriteAheadPathname(pathname); + runtime.files[normalizedPathname] ??= createSharedFile( + runtime.dbFilename, + normalizedPathname, + initialCapacityBytes, + maxCapacityBytes, + runtime.capacities.allowFixedCapacityFallback + ); + + return runtime.files[normalizedPathname]; +}; + +const fileMetaViews = new WeakMap(); +const segmentByteViews = new WeakMap(); + const getFileMeta = (file: MemoryWriteAheadSharedFile) => { const existingView = fileMetaViews.get(file); if (existingView) { @@ -534,6 +691,18 @@ const getFileMeta = (file: MemoryWriteAheadSharedFile) => { return view; }; +const getSegmentBytes = (segment: MemoryWriteAheadSharedSegment) => { + const existingView = segmentByteViews.get(segment); + + if (existingView?.byteLength === segment.data.byteLength) { + return existingView; + } + + const view = new Uint8Array(segment.data); + segmentByteViews.set(segment, view); + return view; +}; + const getActiveSegmentCount = (file: MemoryWriteAheadSharedFile) => Math.max(MIN_SEGMENT_COUNT, Atomics.load(getFileMeta(file), MEMORY_WRITE_AHEAD_FILE_META.activeSegmentCount)); @@ -543,11 +712,34 @@ const getAllocatedCapacityBytes = (file: MemoryWriteAheadSharedFile) => const getFileCapacityBytes = (file: MemoryWriteAheadSharedFile) => file.segments.slice(0, getActiveSegmentCount(file)).reduce((sum, segment) => sum + segment.data.byteLength, 0); +const getActiveFileCapacityBytes = (file: MemoryWriteAheadSharedFile, activeSegmentCount: number) => { + const activeSegment = file.segments[activeSegmentCount - 1]; + + if (!activeSegment) { + return 0; + } + + return (activeSegmentCount - 1) * file.segmentCapacityBytes + activeSegment.data.byteLength; +}; + const getFileSizeBytes = (file: MemoryWriteAheadSharedFile) => Atomics.load(getFileMeta(file), MEMORY_WRITE_AHEAD_FILE_META.size); +const recordMemoryWriteAheadFileHighWater = (file: MemoryWriteAheadSharedFile) => { + const meta = getFileMeta(file); + updateAtomicMaxCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.peakLogicalSizeBytes, getFileSizeBytes(file)); + updateAtomicMaxCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.peakCapacityBytes, getFileCapacityBytes(file)); + updateAtomicMaxCounter( + meta, + MEMORY_WRITE_AHEAD_FILE_META.peakAllocatedCapacityBytes, + getAllocatedCapacityBytes(file) + ); +}; + const setFileSizeBytes = (file: MemoryWriteAheadSharedFile, sizeBytes: number) => { - Atomics.store(getFileMeta(file), MEMORY_WRITE_AHEAD_FILE_META.size, sizeBytes); + const meta = getFileMeta(file); + Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.size, sizeBytes); + updateAtomicMaxCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.peakLogicalSizeBytes, sizeBytes); }; const setActiveSegmentCount = (file: MemoryWriteAheadSharedFile, segmentCount: number) => { @@ -566,26 +758,29 @@ const setActiveSegmentCount = (file: MemoryWriteAheadSharedFile, segmentCount: n Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.activeSegmentCount, next); Atomics.notify(meta, MEMORY_WRITE_AHEAD_FILE_META.activeSegmentCount); + updateAtomicMaxCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.peakCapacityBytes, getFileCapacityBytes(file)); + updateAtomicMaxCounter( + meta, + MEMORY_WRITE_AHEAD_FILE_META.peakAllocatedCapacityBytes, + getAllocatedCapacityBytes(file) + ); }; const growSegment = (segment: MemoryWriteAheadSharedSegment, requiredBytes: number) => { if (requiredBytes <= segment.data.byteLength) { - return; + return false; } if (requiredBytes > segment.maxCapacityBytes) { throw new Error(`MemoryWriteAheadVFS segment capacity exceeded; required ${requiredBytes} bytes`); } - const grow = (segment.data as GrowableSharedArrayBuffer).grow; - if (typeof grow === 'function') { - grow.call(segment.data, requiredBytes); - return; + if (segment.capacityMode === 'growable') { + segment.data.grow(requiredBytes); + return true; } - if (segment.data.byteLength < requiredBytes) { - throw new Error('MemoryWriteAheadVFS fixed SharedArrayBuffer segment cannot grow'); - } + throw new Error('MemoryWriteAheadVFS fixed SharedArrayBuffer segment cannot grow'); }; const findSegmentAtOffset = (file: MemoryWriteAheadSharedFile, logicalOffset: number) => { @@ -598,7 +793,9 @@ const findSegmentAtOffset = (file: MemoryWriteAheadSharedFile, logicalOffset: nu return { segment, + segmentIndex, segmentOffset: logicalOffset - segmentIndex * file.segmentCapacityBytes, + segmentStartOffset: segmentIndex * file.segmentCapacityBytes, }; }; @@ -613,12 +810,19 @@ const ensureFileCapacity = ( ); } + const activeSegmentCount = getActiveSegmentCount(file); + if (requiredCapacityBytes <= getActiveFileCapacityBytes(file, activeSegmentCount)) { + return; + } + const desiredSegmentCount = requiredCapacityBytes === 0 ? MIN_SEGMENT_COUNT : Math.max(MIN_SEGMENT_COUNT, Math.ceil(requiredCapacityBytes / file.segmentCapacityBytes)); - for (let index = 0; index < desiredSegmentCount; index += 1) { + let didGrowSegment = false; + + for (let index = Math.max(0, activeSegmentCount - 1); index < desiredSegmentCount; index += 1) { const segment = file.segments[index]; if (!segment) { throw new Error(`MemoryWriteAheadVFS segment catalog exhausted for ${file.pathname}`); @@ -628,12 +832,20 @@ const ensureFileCapacity = ( segment.maxCapacityBytes, Math.max(0, requiredCapacityBytes - index * file.segmentCapacityBytes) ); - growSegment(segment, requiredSegmentBytes); + didGrowSegment = growSegment(segment, requiredSegmentBytes) || didGrowSegment; onProgress?.(); } - if (desiredSegmentCount > getActiveSegmentCount(file)) { + if (desiredSegmentCount > activeSegmentCount) { setActiveSegmentCount(file, desiredSegmentCount); + } else if (didGrowSegment) { + const meta = getFileMeta(file); + updateAtomicMaxCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.peakCapacityBytes, getFileCapacityBytes(file)); + updateAtomicMaxCounter( + meta, + MEMORY_WRITE_AHEAD_FILE_META.peakAllocatedCapacityBytes, + getAllocatedCapacityBytes(file) + ); } }; @@ -643,6 +855,9 @@ export const createMemoryWriteAheadSharedRuntime = ( ) => { const normalizedDbFilename = normalizeMemoryWriteAheadPathname(dbFilename); const capacities = resolveMemoryWriteAheadRuntimeOptions(options); + if (capacities.allowFixedCapacityFallback === false && globalThis.crossOriginIsolated === false) { + throw createGrowableSharedArrayBufferSupportError(); + } const registry = getRuntimeRegistry(); const existing = registry.get(normalizedDbFilename); @@ -707,24 +922,78 @@ export const createMemoryWriteAheadSharedRuntime = ( return runtime; }; +export const ensureMemoryWriteAheadDatabaseFiles = ( + runtime: MemoryWriteAheadSharedRuntime, + dbFilename: string +): MemoryWriteAheadDatabaseFiles => { + const normalizedDbFilename = normalizeMemoryWriteAheadPathname(dbFilename); + + return { + database: ensureSharedFile( + runtime, + normalizedDbFilename, + runtime.capacities.initialDatabaseCapacityBytes, + runtime.capacities.maxDatabaseCapacityBytes + ), + writeAhead: [ + ensureSharedFile( + runtime, + `${normalizedDbFilename}-wa0`, + runtime.capacities.initialWriteAheadCapacityBytes, + runtime.capacities.maxWriteAheadCapacityBytes + ), + ensureSharedFile( + runtime, + `${normalizedDbFilename}-wa1`, + runtime.capacities.initialWriteAheadCapacityBytes, + runtime.capacities.maxWriteAheadCapacityBytes + ), + ], + }; +}; + export const registerMemoryWriteAheadRuntime = (runtime: MemoryWriteAheadSharedRuntime) => { getRuntimeRegistry().set(normalizeMemoryWriteAheadPathname(runtime.dbFilename), runtime); return runtime; }; +export const registerMemoryWriteAheadSharedFiles = ({ + files, + runtimeDbFilename, +}: MemoryWriteAheadSharedFilesRegistration) => { + const normalizedDbFilename = normalizeMemoryWriteAheadPathname(runtimeDbFilename); + const runtime = getRuntimeRegistry().get(normalizedDbFilename); + + if (!runtime) { + throw new Error(`MemoryWriteAheadVFS runtime is not registered for ${normalizedDbFilename}`); + } + + files.forEach((file) => { + if (normalizeMemoryWriteAheadPathname(file.dbFilename) !== normalizedDbFilename) { + throw new Error( + `MemoryWriteAheadVFS file ${file.pathname} belongs to ${file.dbFilename}, not ${normalizedDbFilename}` + ); + } + + runtime.files[normalizeMemoryWriteAheadPathname(file.pathname)] = file; + }); + + return runtime; +}; + export const retainMemoryWriteAheadRuntimeHandle = ( runtime: MemoryWriteAheadSharedRuntime, logger: MemoryWriteAheadLogger = memoryWriteAheadRuntimeLogger ) => { const lease = ensureLocalRuntimeLease(runtime, logger); const runtimeMeta = getRuntimeMeta(runtime); - const handleCountIndex = getRuntimeOwnerLeaseIndex( - lease.slotIndex, - MEMORY_WRITE_AHEAD_OWNER_LEASE_META.handleCount - ); lease.handleCount += 1; - Atomics.add(runtimeMeta, handleCountIndex, 1); Atomics.add(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.openHandleCount, 1); + Atomics.add( + runtimeMeta, + getRuntimeOwnerLeaseIndex(lease.slotIndex, MEMORY_WRITE_AHEAD_OWNER_LEASE_META.handleCount), + 1 + ); updateOwnerLeaseHeartbeat(runtime, lease); logger.debug('Retained MemoryWriteAhead runtime handle', { dbFilename: runtime.dbFilename, @@ -783,61 +1052,111 @@ export const releaseMemoryWriteAheadRuntimeHandle = ( }); }; -export const resetMemoryWriteAheadSharedRuntime = (dbFilename?: string) => { - const registry = getRuntimeRegistry(); - const runtimes = dbFilename - ? [registry.get(normalizeMemoryWriteAheadPathname(dbFilename))].filter( - (runtime): runtime is MemoryWriteAheadSharedRuntime => runtime !== undefined - ) - : Array.from(registry.values()); - - for (const runtime of runtimes) { - recoverAbandonedRuntimeOwnerLeases(runtime); - const runtimeMeta = getRuntimeMeta(runtime); - const openHandleCount = getOpenHandleCount(runtime); - - if (openHandleCount > 0) { - throw new Error(`Cannot reset MemoryWriteAheadVFS while ${openHandleCount} open handles remain`); - } +export const getMemoryWriteAheadRuntimeOwnerId = (runtime: MemoryWriteAheadSharedRuntime) => + localRuntimeLeases.get(runtime.dbFilename)?.ownerId ?? null; - Object.values(runtime.files).forEach((file) => { - const meta = getFileMeta(file); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lock, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.size, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.activeSegmentCount, MIN_SEGMENT_COUNT); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lockOwner, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lockHeartbeatMs, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.reclaimedSegmentCount, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.reclaimedBytes, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.recoveredAbandonedLockCount, 0); +export const recoverTerminatedMemoryWriteAheadRuntimeOwner = ( + runtime: MemoryWriteAheadSharedRuntime, + ownerId: number, + logger: MemoryWriteAheadLogger = memoryWriteAheadRuntimeLogger +) => { + if (ownerId <= 0) { + return 0; + } + + const runtimeMeta = getRuntimeMeta(runtime); + const slotIndex = Array.from({ length: MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT }, (_, index) => index).find( + (index) => + Atomics.load(runtimeMeta, getRuntimeOwnerLeaseIndex(index, MEMORY_WRITE_AHEAD_OWNER_LEASE_META.ownerId)) === + ownerId + ); + if (slotIndex === undefined) { + return 0; + } + + const ownerIdIndex = getRuntimeOwnerLeaseIndex(slotIndex, MEMORY_WRITE_AHEAD_OWNER_LEASE_META.ownerId); + if (Atomics.compareExchange(runtimeMeta, ownerIdIndex, ownerId, -ownerId) !== ownerId) { + return 0; + } + + const handleCountIndex = getRuntimeOwnerLeaseIndex(slotIndex, MEMORY_WRITE_AHEAD_OWNER_LEASE_META.handleCount); + const recoveredHandleCount = reduceOpenHandleCount(runtime, Atomics.load(runtimeMeta, handleCountIndex)); + const quarantinedTransactionCount = quarantineTransactionLocksForRuntimeOwner(runtime, ownerId); + clearOwnerLeaseSlot(runtimeMeta, slotIndex); + Atomics.add(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.recoveredAbandonedHandleCount, recoveredHandleCount); + logger.warn('Recovered terminated MemoryWriteAhead runtime owner', { + dbFilename: runtime.dbFilename, + ownerId, + recoveredHandleCount, + quarantinedTransactionCount, + slotIndex, + }); + return recoveredHandleCount; +}; + +const resetMemoryWriteAheadRuntimeState = (runtime: MemoryWriteAheadSharedRuntime) => { + recoverAbandonedRuntimeOwnerLeases(runtime); + const runtimeMeta = getRuntimeMeta(runtime); + const openHandleCount = getOpenHandleCount(runtime); + + if (openHandleCount > 0) { + throw new Error(`Cannot reset MemoryWriteAheadVFS while ${openHandleCount} open handles remain`); + } + + Object.values(runtime.files).forEach((file) => { + const meta = getFileMeta(file); + Object.values(MEMORY_WRITE_AHEAD_FILE_META).forEach((slot) => { + Atomics.store(meta, slot, 0); }); + Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.activeSegmentCount, MIN_SEGMENT_COUNT); + recordMemoryWriteAheadFileHighWater(file); + }); - Atomics.store(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.openHandleCount, 0); - Atomics.store(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.recoveredAbandonedHandleCount, 0); + Atomics.store(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.openHandleCount, 0); + Atomics.store(runtimeMeta, MEMORY_WRITE_AHEAD_RUNTIME_META.recoveredAbandonedHandleCount, 0); - for (let slotIndex = 0; slotIndex < MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT; slotIndex += 1) { - clearOwnerLeaseSlot(runtimeMeta, slotIndex); - } + for (let slotIndex = 0; slotIndex < MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT; slotIndex += 1) { + clearOwnerLeaseSlot(runtimeMeta, slotIndex); + } + + const localLease = localRuntimeLeases.get(runtime.dbFilename); + if (localLease) { + clearInterval(localLease.heartbeatIntervalId); + localRuntimeLeases.delete(runtime.dbFilename); + } +}; - const localLease = localRuntimeLeases.get(runtime.dbFilename); - if (localLease) { - clearInterval(localLease.heartbeatIntervalId); - localRuntimeLeases.delete(runtime.dbFilename); +// With a dbFilename, resets only that runtime so concurrent clients (e.g. other databases) keep their +// runtimes; without one, resets every registered runtime (used by test teardown for a clean slate). +export const resetMemoryWriteAheadSharedRuntime = (dbFilename?: string) => { + const registry = getRuntimeRegistry(); + + if (dbFilename !== undefined) { + const normalizedDbFilename = normalizeMemoryWriteAheadPathname(dbFilename); + const runtime = registry.get(normalizedDbFilename); + if (!runtime) { + return; } - registry.delete(runtime.dbFilename); + resetMemoryWriteAheadRuntimeState(runtime); + registry.delete(normalizedDbFilename); + return; } + + for (const runtime of registry.values()) { + resetMemoryWriteAheadRuntimeState(runtime); + } + + registry.clear(); }; export const getMemoryWriteAheadFile = (runtime: MemoryWriteAheadSharedRuntime, pathname: string) => { return runtime.files[normalizeMemoryWriteAheadPathname(pathname)] ?? runtime.files[pathname]; }; -export const getMemoryWriteAheadFileCapacityBytes = (file: MemoryWriteAheadSharedFile) => getFileCapacityBytes(file); - export const getMemoryWriteAheadFileSizeBytes = (file: MemoryWriteAheadSharedFile) => getFileSizeBytes(file); -// Segment reads are lock-free; mutating helpers in this section assume the caller holds the file lock. +// Segment reads are lock-free; mutating helpers below assume the caller holds the file lock. export const readMemoryWriteAheadFile = ( file: MemoryWriteAheadSharedFile, target: Uint8Array | DataView, @@ -906,27 +1225,60 @@ export const writeMemoryWriteAheadFile = ( ensureMemoryWriteAheadFileCapacity(file, requiredCapacityBytes, onProgress, logger); let bytesWritten = 0; - let logicalOffset = offset; + if (input.byteLength === 0) { + setFileSizeBytes(file, Math.max(getFileSizeBytes(file), requiredCapacityBytes)); + addSaturatingAtomicCounter(getFileMeta(file), MEMORY_WRITE_AHEAD_FILE_META.writeOperationCount, 1); + return bytesWritten; + } - while (bytesWritten < input.byteLength) { - const segmentEntry = findSegmentAtOffset(file, logicalOffset); - if (!segmentEntry) { - throw new Error(`Unable to locate segmented storage for ${file.pathname} at ${logicalOffset}`); - } + const firstSegmentEntry = findSegmentAtOffset(file, offset); - const bytesAvailableInSegment = segmentEntry.segment.data.byteLength - segmentEntry.segmentOffset; - const bytesRemainingInTarget = input.byteLength - bytesWritten; - const bytesToCopy = Math.min(bytesAvailableInSegment, bytesRemainingInTarget); - new Uint8Array(segmentEntry.segment.data, segmentEntry.segmentOffset, bytesToCopy).set( - input.subarray(bytesWritten, bytesWritten + bytesToCopy) - ); + if (!firstSegmentEntry) { + throw new Error(`Unable to locate segmented storage for ${file.pathname} at ${offset}`); + } - bytesWritten += bytesToCopy; - logicalOffset += bytesToCopy; + const firstSegmentBytes = Math.min( + firstSegmentEntry.segment.data.byteLength - firstSegmentEntry.segmentOffset, + input.byteLength + ); + + if (firstSegmentBytes === input.byteLength) { + getSegmentBytes(firstSegmentEntry.segment).set(input, firstSegmentEntry.segmentOffset); + bytesWritten = firstSegmentBytes; onProgress?.(); + } else { + getSegmentBytes(firstSegmentEntry.segment).set( + input.subarray(0, firstSegmentBytes), + firstSegmentEntry.segmentOffset + ); + bytesWritten = firstSegmentBytes; + let logicalOffset = offset + firstSegmentBytes; + onProgress?.(); + + while (bytesWritten < input.byteLength) { + const segmentEntry = findSegmentAtOffset(file, logicalOffset); + if (!segmentEntry) { + throw new Error(`Unable to locate segmented storage for ${file.pathname} at ${logicalOffset}`); + } + + const bytesAvailableInSegment = segmentEntry.segment.data.byteLength - segmentEntry.segmentOffset; + const bytesRemainingInTarget = input.byteLength - bytesWritten; + const bytesToCopy = Math.min(bytesAvailableInSegment, bytesRemainingInTarget); + getSegmentBytes(segmentEntry.segment).set( + input.subarray(bytesWritten, bytesWritten + bytesToCopy), + segmentEntry.segmentOffset + ); + + bytesWritten += bytesToCopy; + logicalOffset += bytesToCopy; + onProgress?.(); + } } setFileSizeBytes(file, Math.max(getFileSizeBytes(file), requiredCapacityBytes)); + const meta = getFileMeta(file); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.writeOperationCount, 1); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.writtenBytes, bytesWritten); return bytesWritten; }; @@ -947,6 +1299,9 @@ export const truncateMemoryWriteAheadFile = ( ? MIN_SEGMENT_COUNT : Math.max(MIN_SEGMENT_COUNT, Math.ceil(sizeBytes / file.segmentCapacityBytes)); setActiveSegmentCount(file, desiredSegmentCount); + const meta = getFileMeta(file); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.truncateOperationCount, 1); + recordMemoryWriteAheadFileHighWater(file); if (desiredSegmentCount < previousSegmentCount) { logger.debug('Reclaimed MemoryWriteAhead segmented file capacity', { @@ -958,16 +1313,52 @@ export const truncateMemoryWriteAheadFile = ( } }; +export const recordMemoryWriteAheadFileLockWait = ( + file: MemoryWriteAheadSharedFile, + waitCount: number, + waitMs: number +) => { + if (waitCount <= 0) { + return; + } + + const meta = getFileMeta(file); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.lockWaitCount, waitCount); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.lockWaitMs, Math.max(0, Math.round(waitMs))); +}; + +export const recordMemoryWriteAheadCheckpoint = ( + file: MemoryWriteAheadSharedFile, + durationMs: number, + writeAheadPages: number +) => { + const meta = getFileMeta(file); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.checkpointCount, 1); + addSaturatingAtomicCounter(meta, MEMORY_WRITE_AHEAD_FILE_META.checkpointMs, Math.max(0, Math.round(durationMs))); + addSaturatingAtomicCounter( + meta, + MEMORY_WRITE_AHEAD_FILE_META.checkpointWriteAheadPages, + Math.max(0, Math.round(writeAheadPages)) + ); + Atomics.store( + meta, + MEMORY_WRITE_AHEAD_FILE_META.lastCheckpointMs, + Math.max(0, Math.min(MAX_INT32_ATOMIC_COUNTER, Math.round(durationMs))) + ); + Atomics.store( + meta, + MEMORY_WRITE_AHEAD_FILE_META.lastCheckpointWriteAheadPages, + Math.max(0, Math.min(MAX_INT32_ATOMIC_COUNTER, Math.round(writeAheadPages))) + ); +}; + export const clearMemoryWriteAheadFile = (file: MemoryWriteAheadSharedFile) => { const meta = getFileMeta(file); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lock, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lockOwner, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.lockHeartbeatMs, 0); + Object.values(MEMORY_WRITE_AHEAD_FILE_META).forEach((slot) => { + Atomics.store(meta, slot, 0); + }); setFileSizeBytes(file, 0); setActiveSegmentCount(file, MIN_SEGMENT_COUNT); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.reclaimedSegmentCount, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.reclaimedBytes, 0); - Atomics.store(meta, MEMORY_WRITE_AHEAD_FILE_META.recoveredAbandonedLockCount, 0); }; export const getMemoryWriteAheadRuntimeDiagnostics = ( @@ -986,10 +1377,14 @@ export const getMemoryWriteAheadRuntimeDiagnostics = ( files: Object.values(runtime.files).map((file) => { const meta = getFileMeta(file); const lockState = Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.lock); + const allocatedCapacityBytes = getAllocatedCapacityBytes(file); + const capacityBytes = getFileCapacityBytes(file); + const logicalSizeBytes = getFileSizeBytes(file); return { - allocatedCapacityBytes: getAllocatedCapacityBytes(file), - capacityBytes: getFileCapacityBytes(file), - logicalSizeBytes: getFileSizeBytes(file), + allocatedCapacityBytes, + capacityBytes, + currentCapacityBytes: capacityBytes, + logicalSizeBytes, maxCapacityBytes: file.maxCapacityBytes, pathname: file.pathname, reclaimedBytes: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.reclaimedBytes), @@ -997,6 +1392,31 @@ export const getMemoryWriteAheadRuntimeDiagnostics = ( segmentCapacityBytes: file.segmentCapacityBytes, segmentCount: getActiveSegmentCount(file), totalSegmentCount: file.segments.length, + peakLogicalSizeBytes: Math.max( + logicalSizeBytes, + Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.peakLogicalSizeBytes) + ), + peakCapacityBytes: Math.max( + capacityBytes, + Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.peakCapacityBytes) + ), + peakAllocatedCapacityBytes: Math.max( + allocatedCapacityBytes, + Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.peakAllocatedCapacityBytes) + ), + writeOperationCount: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.writeOperationCount), + writtenBytes: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.writtenBytes), + truncateOperationCount: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.truncateOperationCount), + lockWaitCount: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.lockWaitCount), + lockWaitMs: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.lockWaitMs), + checkpointCount: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.checkpointCount), + checkpointMs: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.checkpointMs), + checkpointWriteAheadPages: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.checkpointWriteAheadPages), + lastCheckpointMs: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.lastCheckpointMs), + lastCheckpointWriteAheadPages: Atomics.load( + meta, + MEMORY_WRITE_AHEAD_FILE_META.lastCheckpointWriteAheadPages + ), lockState, lockOwner: lockState === 0 ? 0 : Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.lockOwner), lockHeartbeatAgeMs: @@ -1009,6 +1429,12 @@ export const getMemoryWriteAheadRuntimeDiagnostics = ( meta, MEMORY_WRITE_AHEAD_FILE_META.recoveredAbandonedLockCount ), + abandonedTransaction: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.abandonedTransaction) !== 0, + transactionLockRuntimeOwner: Atomics.load( + meta, + MEMORY_WRITE_AHEAD_FILE_META.transactionLockRuntimeOwner + ), + transactionLockState: Atomics.load(meta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock), }; }), openHandleCount: getOpenHandleCount(runtime), diff --git a/src/vendor/WriteAhead.d.ts b/src/vendor/WriteAhead.d.ts new file mode 100644 index 0000000..58b233c --- /dev/null +++ b/src/vendor/WriteAhead.d.ts @@ -0,0 +1,29 @@ +type WriteAheadDurability = 'relaxed' | 'strict'; + +export class WriteAhead { + readonly isolationState: 'read' | 'write' | null; + log: ((...args: unknown[]) => void) | null; + options: { + autoCheckpoint: number; + backstopInterval: number; + journalSizeLimit: number; + }; + constructor(dbName: string, dbHandle: object, writeAheadHandles: [object, object]); + checkpoint(options?: { isPassive?: boolean }): Promise; + close(): void; + commit(): void; + getFileSize(): number; + getWriteAheadSize(): number; + isolateForRead(): void; + isolateForWrite(): void; + isTransactionPending(): boolean; + read(offset: number): Uint8Array | undefined; + ready(): Promise; + rejoin(): void; + rollback(): void; + setBackstopInterval(value: number): void; + sync(options?: { durability?: WriteAheadDurability }): void; + truncate(size: number): void; + tryUpgradeReadIsolationToWrite(): boolean; + write(offset: number, data: Uint8Array, options?: { dstPageSize?: number | null }): void; +} diff --git a/src/vendor/WriteAhead.js b/src/vendor/WriteAhead.js new file mode 100644 index 0000000..081c6d2 --- /dev/null +++ b/src/vendor/WriteAhead.js @@ -0,0 +1,1208 @@ +// Derived from @journeyapps/wa-sqlite 1.7.0's examples/WriteAhead.js. +// Local changes expose and safely upgrade read isolation for deferred SQLite writes. +import { Lock } from '@journeyapps/wa-sqlite/src/examples/Lock.js'; + +const DEFAULT_JOURNAL_SIZE_LIMIT = 1000; +const DEFAULT_BACKSTOP_INTERVAL = 30_000; + +const MAGIC = 0x377f0684; +const FILE_HEADER_SIZE = 32; +const FRAME_HEADER_SIZE = 32; +const FRAME_TYPE_PAGE = 0; +const FRAME_TYPE_COMMIT = 1; +const FRAME_TYPE_END = 2; + +/** + * @typedef PageEntry + * @property {number} waOffset location in WAL file + * @property {number} waSalt1 WAL2 file identifier + * @property {number} pageSize + * @property {Uint8Array} [pageData] + */ + +/** + * @typedef Transaction + * @property {number} id + * @property {Map} pages address to page data mapping + * @property {number} dbFileSize + * @property {number} [newPageSize] + * @property {number} waSalt1 WAL2 file identifier + * @property {number} waOffsetEnd + */ + +/** + * @typedef WriteAheadOptions + * @property {number} [autoCheckpoint] + * @property {number} [backstopInterval] + * @property {number} [journalSizeLimit] + */ + +export class WriteAhead { + + log = null; + /** @type {WriteAheadOptions} */ options = { + autoCheckpoint: 1, + backstopInterval: DEFAULT_BACKSTOP_INTERVAL, + journalSizeLimit: DEFAULT_JOURNAL_SIZE_LIMIT, + }; + + #zName; + #dbHandle; + + /** @type {FileSystemSyncAccessHandle[]} */ #waHandles; + /** @type {FileSystemSyncAccessHandle} */ #activeHandle; + /** @type {{nextTxId: number, salt1: number, salt2: number}} */ #activeHeader; + /** @type {number} */ #activeOffset; + /** @type {number} */ #txId = 0; + /** @type {Transaction} */ #txInProgress = null; + + #dbFileSize = 0; + + /** @type {Promise} */ #ready; + /** @type {'read'|'write'} */ #isolationState = null; + + /** @type {Lock} */ #txIdLock = null; + + /** @type {Map} */ #waOverlay = new Map(); + /** @type {Map} */ #mapIdToTx = new Map(); + /** @type {Map} */ #mapIdToPendingTx = new Map(); + #approxPageCount = 0; + + /** @type {BroadcastChannel} */ #broadcastChannel; + + /** @type {number} */ #backstopTimer; + /** @type {number} */ #backstopTimestamp = 0; + + #abortController = new AbortController(); + + /** + * @param {string} zName + * @param {FileSystemSyncAccessHandle} dbHandle + * @param {FileSystemSyncAccessHandle[]} waHandles + * @param {WriteAheadOptions} options + */ + constructor(zName, dbHandle, waHandles, options = {}) { + this.#zName = zName; + this.#dbHandle = dbHandle; + this.#waHandles = waHandles; + this.options = Object.assign(this.options, options); + + // All the asynchronous initialization is done here. + this.#ready = (async () => { + // Set our advertised txId to zero until we know the proper value. + await this.#updateTxIdLock(); + + // Listen for transactions and checkpoints from other connections. + this.#broadcastChannel = new BroadcastChannel(`${zName}#wa`); + this.#broadcastChannel.onmessage = (event) => { + this.#handleMessage(event); + }; + + // Read headers from both WAL files and use the one with the + // lower nextTxId. If neither header is valid, create a new header. + const fileHeader = this.#waHandles + .map(handle => this.#readFileHeader(handle)) + .filter(h => h) + .sort((a, b) => a.nextTxId - b.nextTxId)[0] + ?? this.#writeFileHeader(Math.floor(Math.random() * 0xffffffff)); + + this.#activeHeader = fileHeader; + this.#activeHandle = this.#waHandles[fileHeader.salt1 & 1]; + this.#activeOffset = FILE_HEADER_SIZE; + this.#txId = fileHeader.nextTxId - 1; + + // Load all the transactions from the WAL. + for (const tx of this.#readAllTx()) { + this.#activateTx(tx); + } + this.#updateTxIdLock(); // doesn't need await + + // Schedule backstop. The backstop is a guard against a crash in + // another context between persisting a transaction and broadcasting + // it. + this.#backstopTimestamp = performance.now(); + this.#backstop(); + })(); + } + + /** + * @returns {Promise} + */ + ready() { + return this.#ready; + } + + close() { + this.#abortController.abort(); + + // Stop asynchronous maintenance. + this.#broadcastChannel.onmessage = null; + clearTimeout(this.#backstopTimer); + + this.#txIdLock?.release(); + this.#broadcastChannel.close(); + } + + /** + * Freeze our view of the database. + * The view includes the transactions received so far but is not + * guaranteed to be completely up to date. Unfreeze the view with rejoin(). + */ + isolateForRead() { + if (this.#isolationState !== null) { + throw new Error('Already in isolated state'); + } + this.#isolationState = 'read'; + + // Disable backstop during isolation. + clearTimeout(this.#backstopTimer); + this.#backstopTimer = null; + } + + /** + * Freeze our view of the database for writing. + * The view includes all transactions. Unfreeze the view with rejoin(). + */ + isolateForWrite() { + if (this.#isolationState !== null) { + throw new Error('Already in isolated state'); + } + this.#isolationState = 'write'; + + // Disable backstop during isolation. + clearTimeout(this.#backstopTimer); + this.#backstopTimer = null; + + // A writer needs all previous transactions assimilated. + this.#advanceTxId({ readToCurrent: true }); + } + + /** + * The authoritative isolation state used by the VFS lock transition. + * @returns {'read'|'write'|null} + */ + get isolationState() { + return this.#isolationState; + } + + /** + * Upgrade read isolation without changing its frozen snapshot. + * The caller must already hold the database write lock. + * @returns {boolean} + */ + tryUpgradeReadIsolationToWrite() { + if (this.#isolationState !== 'read') { + throw new Error('Not in read isolated state'); + } + + if (this.#mapIdToPendingTx.size > 0) { + return false; + } + + // A frame after the frozen view means another connection wrote after the + // read began. A partial tail also cannot be overwritten safely. + if (this.#readFrame(this.#activeOffset) !== null) { + return false; + } + + this.#isolationState = 'write'; + return true; + } + + rejoin() { + if (this.#isolationState === 'read') { + // Catch up on new transactions that arrived while isolated. + this.#advanceTxId({ autoCheckpoint: true }); + } + this.#isolationState = null; + + // Resume backstop after isolation. + this.#backstop(); + } + + /** + * @param {number} offset + * @return {Uint8Array?} + */ + read(offset) { + // First look for the page in any write transaction in progress. + // If the page is not found in the transaction overlay, look in the + // write-ahead overlay. + const pageEntry = this.#txInProgress?.pages.get(offset) ?? this.#waOverlay.get(offset); + if (pageEntry) { + if (pageEntry.pageData) { + // Page data is cached. + this.log?.(`%cread page at ${offset} from WAL ${pageEntry.waSalt1 & 1}:${pageEntry.waOffset} (cached)`, 'background-color: gold;'); + return pageEntry.pageData; + } + + // Read the page from the WAL file. + this.log?.(`%cread page at ${offset} from WAL ${pageEntry.waSalt1 & 1}:${pageEntry.waOffset}`, 'background-color: gold;'); + return this.#fetchPage(pageEntry); + } + return null; + } + + /** + * @param {number} offset + * @param {Uint8Array} data + * @param {{dstPageSize: number?}} options + */ + write(offset, data, options) { + if (this.#isolationState !== 'write') { + throw new Error('Not in write isolated state'); + } + + if (!this.#txInProgress) { + // There is no active transaction so we need to create one. But + // first check whether to move to the other WAL file. + const nPageThreshold = this.options.journalSizeLimit > 0 ? + this.options.journalSizeLimit : + DEFAULT_JOURNAL_SIZE_LIMIT; + if (this.#approxPageCount >= nPageThreshold && this.#isInactiveFileEmpty()) { + this.log?.(`%cchange WAL file at ${this.#approxPageCount} pages`, 'background-color: lightskyblue;'); + this.#swapActiveFile(); + } + + this.#beginTx(); + if (options.dstPageSize !== data.byteLength) { + // This is a VACUUM to a new page size. The incoming writes are at + // the old page size, but we want to write to the WAL with the new + // size. + this.#txInProgress.newPageSize = options.dstPageSize; + } + } + + if (this.#txInProgress.newPageSize) { + // The incoming data is not a single page because the page size + // is changing. The two cases are when the new page size is + // smaller or larger than the old page size. + const frameSize = FRAME_HEADER_SIZE + this.#txInProgress.newPageSize; + if (data.byteLength > this.#txInProgress.newPageSize) { + // New page size is smaller. Write multiple pages of the new + // page size. + for (let i = 0; i < data.byteLength; i += this.#txInProgress.newPageSize) { + const pageData = data.slice(i, i + this.#txInProgress.newPageSize); + const waOffset = this.#writePage(offset + i, pageData); + this.log?.(`%cwrite page at ${offset + i} to WAL ${this.#activeHeader.salt1 & 1}:${waOffset}`, 'background-color: lightskyblue;'); + } + } else { + // New page size is larger. Save the page data to the WAL file + // so it can be read back and rewritten as frames with the new + // page size. + const pageOffset = offset % this.#txInProgress.newPageSize; + const waOffset = this.#activeOffset + + (offset - pageOffset) / this.#txInProgress.newPageSize * frameSize + + FRAME_HEADER_SIZE + + pageOffset; + this.#activeHandle.write(data.subarray(), { at: waOffset }); + this.log?.(`%cwrite page at ${offset} to WAL ${this.#activeHeader.salt1 & 1}:${waOffset}`, 'background-color: lightskyblue;'); + } + } else { + // This is the normal case without a page size change. + const waOffset = this.#writePage(offset, data.slice()); + this.log?.(`%cwrite page at ${offset} to WAL ${this.#activeHeader.salt1 & 1}:${waOffset}`, 'background-color: lightskyblue;'); + } + } + + /** + * @param {number} newSize + */ + truncate(newSize) { + // Ignore truncation that happens outside of a transaction. That + // only happens (e.g. post-VACUUM) to ensure the file size matches + // the database header. + if (this.#txInProgress) { + // Remove any pages past the truncation point. + for (const offset of this.#txInProgress.pages.keys()) { + if (offset >= newSize) { + this.#txInProgress.pages.delete(offset); + } + } + } + } + + getFileSize() { + return this.#txInProgress?.dbFileSize ?? this.#dbFileSize; + } + + commit() { + const tx = this.#txInProgress; + if (tx.newPageSize && tx.pages.size === 0) { + // This transaction is a VACUUM with a page size increase. All + // the database pages have been written to the WAL file at their + // new size with blank frame headers. Read the page data back + // from the WAL file and rewrite as frames. + let pageCount = 1; // to be replaced on the first iteration + for (let i = 0; i < pageCount; i++) { + // Read the page data. + const pageData = new Uint8Array(tx.newPageSize); + const waOffset = this.#activeOffset + + i * (FRAME_HEADER_SIZE + tx.newPageSize) + + FRAME_HEADER_SIZE; + this.#activeHandle.read(pageData, { at: waOffset }); + + if (i === 0) { + // Get the actual page count from the file header. + const headerView = new DataView(pageData.buffer); + pageCount = headerView.getUint32(28); + } + + // Write back as a frame. + this.#writePage(i * tx.newPageSize, pageData); + } + } + + const page1 = this.#txInProgress.pages.get(0)?.pageData; + if (page1) { + const page1View = new DataView(page1.buffer, page1.byteOffset, page1.byteLength); + const pageCount = page1View.getUint32(28); + this.#txInProgress.dbFileSize = pageCount * page1.byteLength; + } else { + // The transaction doesn't include page 1, so this must be a + // non-batch-atomic rollback. + this.rollback(); + return; + } + + // Persist the final pending transaction page with the database size. + this.#commitTx(); + + // Incorporate the transaction locally. + this.#activateTx(tx); + this.#updateTxIdLock(); + + // Send the transaction to other connections. + const payload = { type: 'tx', tx }; + this.#broadcastChannel.postMessage(payload); + + this.#autoCheckpoint(); + this.#backstopTimestamp = performance.now(); + } + + rollback() { + // Discard transaction pages. + this.#abortTx(); + } + + /** + * @param {{durability: 'strict'|'relaxed'}} options + */ + sync(options) { + if (options.durability === 'strict') { + this.#flushActiveFile(); + } + } + + /** + * Move pages from write-ahead to main database file. + * + * @param {{isPassive: boolean}} options + */ + async checkpoint(options = { isPassive: true }) { + // Passive checkpointing is abandoned if another connection is + // already checkpointing. + const lockOptions = { + ifAvailable: options.isPassive, + }; + + await navigator.locks.request(`${this.#zName}#ckpt`, lockOptions, async lock => { + if (!lock) return; + if (this.#abortController.signal.aborted) return; + + let ckptId = this.#getActiveFileStartingTxId() - 1; + if (options.isPassive) { + if (!this.#mapIdToTx.has(ckptId)) { + // There are no transactions to checkpoint. + return; + } + + // Scan the txId locks to find the oldest txId. + const busyTxId = (await this.#getTxIdLocks()) + .reduce((min, value) => Math.min(min, value.maxTxId), this.#txId); + + if (busyTxId < ckptId) { + // The inactive WAL file is still being used. + return; + } + } else { + // Wait for all connections to reach the current txId. + await this.#waitForTxIdLocks(value => value.maxTxId >= this.#txId); + ckptId = this.#txId; + } + this.log?.(`%ccheckpoint through txId ${ckptId}`, 'background-color: lightgreen;'); + + // Sync the WAL file. This ensures that if there is a crash after + // part of the WAL has been copied, the uncopied part will still be + // available afterwards. + this.#flushInactiveFile(); + if (!options.isPassive) { + this.#flushActiveFile(); + } + + // Starting at ckptId and going backwards (higher to lower txId), + // write transaction pages to the main database file. Do not overwrite + // a page written by a more recent transaction. + const writtenOffsets = new Set(); + let dbFileSize = this.#dbHandle.getSize(); + for (let tx = this.#mapIdToTx.get(ckptId); tx; tx = this.#mapIdToTx.get(tx.id - 1)) { + if (tx.id === ckptId && dbFileSize !== tx.dbFileSize) { + // Set the file size from the latest transaction. + dbFileSize = tx.dbFileSize; + this.#dbHandle.truncate(dbFileSize); + } + + for (const [offset, pageEntry] of tx.pages) { + if (offset < dbFileSize && !writtenOffsets.has(offset)) { + // Fetch the page data from the WAL file if not cached. + const pageData = pageEntry.pageData ?? this.#fetchPage(pageEntry); + + // Write the page to the database file. + const nWritten = this.#dbHandle.write(pageData, { at: offset }); + if (nWritten !== pageData.byteLength) { + throw new Error('Checkpoint write failed'); + } + writtenOffsets.add(offset); + this.log?.(`%ccheckpoint wrote txId ${tx.id} page at ${offset} to database`, 'background-color: lightgreen;'); + } + } + + if (tx.newPageSize) { + // This transaction used a new page size to overwrite the entire + // database file so no older pages need to be written. This is + // not just an optimization; it prevents incorrectly writing + // older smaller pages at addresses that aren't multiples of + // the new page size. + break; + } + } + + // Ensure that database writes are durable. + this.log?.(`%ccheckpoint flush database file`, 'background-color: lightgreen;'); + this.#dbHandle.flush(); + + // Notify other connections and ourselves of the checkpoint. + this.#broadcastChannel.postMessage({ + type: 'ckpt', + ckptId, + }); + this.#handleCheckpoint(ckptId); + + // Wait for all connections to update their overlay. + this.log?.(`%ccheckpoint waiting for connection updates`, 'background-color: lightgreen;'); + await this.#waitForTxIdLocks(value => value.minTxId > ckptId); + + // Truncate the inactive WAL file. This prevents new connections from + // unnecessarily reading checkpointed data, and allows writers to make + // it active when their conditions are met. + this.#truncateInactiveFile(); + this.log?.(`%ccheckpoint complete`, 'background-color: lightgreen;'); + }); + } + + /** + * Return the approximate number of write-ahead pages. This is the + * sum of the number of unique page indices for each transaction, + * so it can be fewer than the number of pages if any transaction + * contains multiple frames for the same page. + * @returns {number} + */ + getWriteAheadSize() { + return this.#approxPageCount; + } + + isTransactionPending() { + return !!this.#txInProgress; + } + + setBackstopInterval(intervalMillis) { + this.options.backstopInterval = intervalMillis; + if (intervalMillis > 0 && this.#isolationState) { + this.#backstop(); + } + } + + /** + * Incorporate a transaction into our view of the database. + * @param {Transaction} tx + */ + #activateTx(tx) { + // Transfer to the active collection of transactions. + this.#mapIdToTx.set(tx.id, tx); + this.#approxPageCount += tx.pages.size; + + // Add transaction pages to the write-ahead overlay. + for (const [offset, pageEntry] of tx.pages) { + this.#waOverlay.set(offset, pageEntry); + } + this.#dbFileSize = tx.dbFileSize; + } + + /** + * Advance the local view of the database. By default, advance to the + * last broadcast transaction. Optionally, also advance through any + * additional transactions in the WAL file to be fully current. + * + * @param {{readToCurrent?: boolean, autoCheckpoint?: boolean}} options + */ + #advanceTxId(options = {}) { + let didAdvance = false; + while (this.#mapIdToPendingTx.size) { + // Fetch the next transaction in sequence. Usually this will come + // from pendingTx, but if it is missing then read it from the file. + const nextTxId = this.#txId + 1; + let tx; + if (this.#mapIdToPendingTx.has(nextTxId)) { + // This transaction arrived via message. + tx = this.#mapIdToPendingTx.get(nextTxId); + this.#mapIdToPendingTx.delete(tx.id); + + // Move the WAL file offset past this transaction. + this.#skipTx(tx); + } else { + // Read the transaction from the WAL file. + tx = this.#readTx(); + } + + this.#activateTx(tx); + didAdvance = true; + } + + if (options.readToCurrent) { + // Read all additional transactions from the WAL file. + for (const tx of this.#readAllTx()) { + this.#activateTx(tx); + didAdvance = true; + } + } + + if (didAdvance) { + // Publish our new view txId. + this.#updateTxIdLock(); + + if (options.autoCheckpoint) { + this.#autoCheckpoint(); + } + } + + if (options.readToCurrent || didAdvance) { + // The WAL has been accessed, so reset the backstop. + // Calling #backstop() here is not necessary because if we are + // in an isolated state then rejoin() will schedule the next call, + // and if we are not in an isolated state then the next call + // should already be scheduled. + this.#backstopTimestamp = performance.now(); + } + } + + #autoCheckpoint() { + if (this.options.autoCheckpoint > 0) { + this.checkpoint({ isPassive: true }); + } + } + + /** + * After a checkpoint, remove checkpointed pages from write-ahead. + * The checkpoint may be been done locally or by another connection. + * @param {number} ckptId + */ + #handleCheckpoint(ckptId) { + this.log?.(`%capply checkpoint through txId ${ckptId}`, 'background-color: lightgreen;'); + + // Loop backwards from ckptId. + for (let tx = this.#mapIdToTx.get(ckptId); tx; tx = this.#mapIdToTx.get(tx.id - 1)) { + // Remove pages from write-ahead overlay. + for (const [offset, pageEntry] of tx.pages.entries()) { + // Be sure not to remove a newer version of the page. + const overlayEntry = this.#waOverlay.get(offset); + if (overlayEntry === pageEntry) { + this.log?.(`%cremove txId ${tx.id} page at offset ${offset}`, 'background-color: lightgreen;'); + this.#waOverlay.delete(offset); + } + } + + // Remove transaction. + this.#mapIdToTx.delete(tx.id); + this.#approxPageCount -= tx.pages.size; + } + this.#updateTxIdLock(); + } + + /** + * @param {MessageEvent} event + */ + #handleMessage(event) { + if (event.data.type === 'tx') { + // New transaction from another connection. Don't use it if we + // already have it. + /** @type {Transaction} */ const tx = event.data.tx; + if (tx.id > this.#txId) { + this.#mapIdToPendingTx.set(tx.id, tx); + if (this.#isolationState === null) { + // Not in an isolated state, so advance our view of the database. + this.#advanceTxId({ autoCheckpoint: true }); + } + } + } else if (event.data.type === 'ckpt') { + // Checkpoint notification from another connection. + /** @type {number} */ const ckptId = event.data.ckptId; + this.#handleCheckpoint(ckptId); + } + } + + /** + * Periodic check for recovering from lost transaction broadcasts. + */ + #backstop() { + if (this.options.backstopInterval <= 0) { + // Backstop is disabled. + return; + } + + if (this.#isolationState) { + throw new Error('Backstop was invoked in an isolated state'); + } + + const now = performance.now(); + if (now >= this.#backstopTimestamp + this.options.backstopInterval) { + // The time since the last WAL access (read, write, or skip) has + // exceeded the backstop interval. Check for transactions in the + // write-ahead log that have not arrived via message. + const oldTxId = this.#txId; + this.#advanceTxId({ readToCurrent: true }); + if (this.#txId > oldTxId) { + this.log?.(`%cbackstop txId ${oldTxId} -> ${this.#txId}`, 'background-color: lightyellow;'); + } + this.#backstopTimestamp = performance.now(); + } + + // Schedule next backstop. + const delay = this.#backstopTimestamp + this.options.backstopInterval - performance.now(); + clearTimeout(this.#backstopTimer); + this.#backstopTimer = self.setTimeout(() => { + this.#backstop(); + }, delay); + } + + /** + * Update the lock that publishes our current txId. + */ + async #updateTxIdLock() { + // Our view of the database, i.e. the txId, is encoded into the name + // of a lock so other connections can see it. When our txId changes, + // we acquire a new lock and release the old one. We must not release + // the old lock until the new one is in place. + const oldLock = this.#txIdLock; + const newLockName = this.#encodeTxIdLockName(); + if (oldLock?.name !== newLockName) { + this.#txIdLock = new Lock(newLockName); + await this.#txIdLock.acquire('shared').then(() => { + // The new lock is acquired. + oldLock?.release(); + }); + + if (this.log) { + const { minTxId, maxTxId } = this.#decodeTxIdLockName(newLockName); + this.log?.(`%ctxId to ${minTxId}:${maxTxId}`, 'background-color: pink;'); + } + } + } + + /** + * Get all txId locks for this database. + * @returns {Promise<{name: string, minTxId: number, maxTxId: number, encoded: string}[]>} + */ + async #getTxIdLocks() { + const { held } = await navigator.locks.query(); + return held + .map(lock => this.#decodeTxIdLockName(lock.name)) + .filter(value => value !== null); + } + + /** + * @returns {string} + */ + #encodeTxIdLockName() { + // The maxTxId is our current view of the database. The minTxId is + // the lowest txId we get pages from the WAL for, which is the lowest + // key in mapIdToTx. If mapIdToTx is empty then we aren't reading + // from the WAL at all - in this case we arbitrarily set minTxId to + // invalid value maxTxId + 1. + // + // Use radix 36 to encode integer values to reduce the lock name length. + const maxTxId = this.#txId; + const minTxId = this.#mapIdToTx.keys().next().value ?? (maxTxId + 1); + return `${this.#zName}-txId<${minTxId.toString(36)}:${maxTxId.toString(36)}>`; + } + + /** + * @param {string} lockName + * @returns {{name: string, minTxId: number, maxTxId: number, encoded: string}?} + */ + #decodeTxIdLockName(lockName) { + const match = lockName.match(/^(.*)-txId<([0-9a-z]+):([0-9a-z]+)>$/); + if (match?.[1] === this.#zName) { + // This txId lock is for this database. + return { + name: match[1], + minTxId: parseInt(match[2], 36), + maxTxId: parseInt(match[3], 36), + encoded: lockName + }; + } + return null; + } + + /** + * Wait for all txId locks that fail the provided predicate. + * @param {(lock: {name: string, minTxId: number, maxTxId: number}) => boolean} predicate + */ + async #waitForTxIdLocks(predicate) { + /** @type {string[]} */ let failingLockNames = []; + do { + // Wait for all connections that fail the predicate. + if (failingLockNames.length > 0) { + await Promise.all( + failingLockNames.map(name => navigator.locks.request(name, async () => {})) + ); + } + + // Refresh the list of failing locks. + failingLockNames = (await this.#getTxIdLocks()) + .filter(value => !predicate(value)) + .map(value => value.encoded); + } while (failingLockNames.length > 0); + } + + /** + * @param {PageEntry} pageEntry + * @returns {Uint8Array} + */ + #fetchPage(pageEntry) { + // Get the appropriate access handle based on salt parity. + const accessHandle = this.#waHandles[pageEntry.waSalt1 & 1]; + + // Read the page. + const pageData = new Uint8Array(pageEntry.pageSize); + const nBytesRead = accessHandle.read(pageData, { at: pageEntry.waOffset }); + + if (nBytesRead !== pageEntry.pageSize) { + throw new Error(`Short WAL read: expected ${pageEntry.pageSize} bytes, got ${nBytesRead}`); + } + return pageData; + } + + *#readAllTx() { + while (true) { + const tx = this.#readTx(); + if (!tx) break; + yield tx; + } + } + + /** + * @returns {Transaction?} + */ + #readTx() { + // Read the next complete transaction or return null. + /** @type {Transaction} */ const tx = { + id: 0, // placeholder + pages: new Map(), + dbFileSize: 0, // placeholder + waSalt1: 0, // placeholder + waOffsetEnd: 0, // placeholder + }; + + // The property this.#activeOffset is only advanced on a successful + // transition to the other WAL file or on reading a complete + // transaction. Use a local variable to track our progress. + let offset = this.#activeOffset; + while (true) { + const frame = this.#readFrame(offset); + if (!frame) return null; + + if (frame.frameType === FRAME_TYPE_PAGE) { + tx.pages.set( + frame.pageOffset, + { + pageSize: frame.pageData.byteLength, + waOffset: offset + FRAME_HEADER_SIZE, + waSalt1: this.#activeHeader.salt1, + } + ); + } else if (frame.frameType === FRAME_TYPE_COMMIT) { + // The transaction is complete. Update the instance state. + this.#txId += 1; + this.#activeOffset = offset + frame.byteLength; + + // Finalize the transaction fields and return it. + tx.id = this.#txId; + tx.dbFileSize = frame.dbFileSize; + tx.waSalt1 = this.#activeHeader.salt1; + tx.newPageSize = (frame.flags & 1) ? tx.pages.get(0).pageSize : null; + tx.waOffsetEnd = this.#activeOffset; + return tx; + } else if (frame.frameType === FRAME_TYPE_END) { + // No more transactions on the current WAL file. Switch to the + // other file. + this.#followFileChange(frame.fileHeader); + offset = this.#activeOffset; + continue; + } + + offset += frame.byteLength; + } + } + + /** + * This method is called when transaction(s) have been received by other + * means than readTx(), e.g. via BroadcastChannel. + * + * @param {Transaction} tx + */ + #skipTx(tx) { + if (tx.waSalt1 !== this.#activeHeader.salt1) { + // This transaction is on the other WAL file. + if (!this.#followFileChange(null)) { + throw new Error('invalid WAL file'); + } + } + + this.#txId = tx.id; + this.#activeOffset = tx.waOffsetEnd; + } + + /** + * @param {{overwrite?: boolean}} options + * @returns {Transaction} + */ + #beginTx(options = {}) { + this.#txInProgress = { + id: this.#txId + 1, + pages: new Map(), + dbFileSize: this.#dbFileSize, + waSalt1: this.#activeHeader.salt1, + waOffsetEnd: this.#activeOffset, + }; + return this.#txInProgress; + } + + /** + * Write a page frame to the WAL file. + * + * @param {number} pageOffset + * @param {Uint8Array} pageData + */ + #writePage(pageOffset, pageData) { + const headerView = new DataView(new ArrayBuffer(FRAME_HEADER_SIZE)); + headerView.setUint8(0, FRAME_TYPE_PAGE); + headerView.setUint16(2, pageData.byteLength === 65536 ? 1 : pageData.byteLength); + headerView.setBigUint64(8, BigInt(pageOffset)); + headerView.setUint32(16, this.#activeHeader.salt1); + headerView.setUint32(20, this.#activeHeader.salt2); + + const checksum = new Checksum(); + checksum.update(new Uint8Array(headerView.buffer, 0, FRAME_HEADER_SIZE - 8)); + checksum.update(pageData); + headerView.setUint32(24, checksum.s0); + headerView.setUint32(28, checksum.s1); + + const bytesWritten = + this.#activeHandle.write(headerView, { at: this.#txInProgress.waOffsetEnd }) + + this.#activeHandle.write(pageData, { + at: this.#txInProgress.waOffsetEnd + FRAME_HEADER_SIZE, + }); + if (bytesWritten !== headerView.byteLength + pageData.byteLength) { + throw new Error('write failed'); + } + + // Cache page 1 as a performance optimization and to exercise the + // cache code path. + const pageEntry = { + pageSize: pageData.byteLength, + waOffset: this.#txInProgress.waOffsetEnd + FRAME_HEADER_SIZE, + waSalt1: this.#activeHeader.salt1, + pageData: pageOffset === 0 ? pageData : undefined + }; + this.#txInProgress.pages.set(pageOffset, pageEntry); + this.#txInProgress.waOffsetEnd += bytesWritten; + + return pageEntry.waOffset; + } + + /** + * @returns {Transaction} + */ + #commitTx() { + // Write a commit frame - which is a special frame header with no + // body - to the WAL file. + const headerView = new DataView(new ArrayBuffer(FRAME_HEADER_SIZE)); + headerView.setUint8(0, FRAME_TYPE_COMMIT); + headerView.setUint8(1, this.#txInProgress.newPageSize ? 1 : 0); + headerView.setBigUint64(8, BigInt(this.#txInProgress.dbFileSize)); + headerView.setUint32(16, this.#activeHeader.salt1); + headerView.setUint32(20, this.#activeHeader.salt2); + + const checksum = new Checksum(); + checksum.update(new Uint8Array(headerView.buffer, 0, FRAME_HEADER_SIZE - 8)); + headerView.setUint32(24, checksum.s0); + headerView.setUint32(28, checksum.s1); + + const bytesWritten = this.#activeHandle.write(headerView, { + at: this.#txInProgress.waOffsetEnd, + }); + if (bytesWritten !== headerView.byteLength) { + throw new Error('write failed'); + } + this.#txInProgress.waOffsetEnd += bytesWritten; + + const tx = this.#txInProgress; + this.#txInProgress = null; + this.#activeOffset = tx.waOffsetEnd; + this.#txId = tx.id; + return tx; + } + + #abortTx() { + this.#txInProgress = null; + this.#activeHandle.truncate(this.#activeOffset); + } + + /** + * Switch the active WAL file prior to writing the next transaction. + */ + #swapActiveFile() { + // Write an end frame to terminate the currently active WAL file. + const frameView = new DataView(new ArrayBuffer(FRAME_HEADER_SIZE)); + frameView.setUint8(0, FRAME_TYPE_END); + frameView.setUint32(16, this.#activeHeader.salt1); + frameView.setUint32(20, this.#activeHeader.salt2); + + const checksum = new Checksum(); + checksum.update(new Uint8Array(frameView.buffer, 0, FRAME_HEADER_SIZE - 8)); + frameView.setUint32(24, checksum.s0); + frameView.setUint32(28, checksum.s1); + + const bytesWritten = this.#activeHandle.write(frameView, { at: this.#activeOffset }); + if (bytesWritten !== frameView.byteLength) { + throw new Error('write failed'); + } + + // Initialize the other WAL file and make it active. + this.#activeHeader = this.#writeFileHeader(); + this.#activeHandle = this.#getInactiveHandle(); + this.#activeOffset = FILE_HEADER_SIZE; + } + + #getActiveFileStartingTxId() { + return this.#activeHeader.nextTxId; + } + + #flushActiveFile() { + this.#activeHandle.flush(); + } + + #flushInactiveFile() { + const accessHandle = this.#getInactiveHandle(); + accessHandle.flush(); + } + + #isInactiveFileEmpty() { + if (this.#mapIdToTx.has(this.#activeHeader.nextTxId - 1)) { + // At least one transaction on the inactive file has not been + // checkpointed. + return false; + } + + const inactiveHandle = this.#getInactiveHandle(); + if (inactiveHandle.getSize() < FILE_HEADER_SIZE) { + // The inactive file is smaller than the minimum size for a valid + // WAL file. + return true; + } + + // This test is sufficient by itself but the previous tests are + // less expensive. + return this.#readFileHeader(inactiveHandle) === null; + } + + #truncateInactiveFile() { + const accessHandle = this.#getInactiveHandle(); + accessHandle.truncate(0); + } + + /** + * This method is called after reading an end frame to switch to the + * other WAL file. + * @param {{nextTxId: number, salt1: number, salt2: number}?} fileHeader + */ + #followFileChange(fileHeader) { + // As an optimization, the file header can be passed as an argument + // if it has already been read and validated. Otherwise that is + // done here. + const accessHandle = this.#getInactiveHandle(); + if (!fileHeader) { + fileHeader = this.#readFileHeader(accessHandle); + if (fileHeader?.salt1 !== ((this.#activeHeader.salt1 + 1) >>> 0)) return null; + } + + this.#activeHandle = accessHandle; + this.#activeHeader = fileHeader; + this.#activeOffset = FILE_HEADER_SIZE; + return fileHeader; + } + + #getInactiveHandle() { + return this.#activeHandle !== this.#waHandles[0] ? + this.#waHandles[0] : + this.#waHandles[1]; + } + + /** + * @param {FileSystemSyncAccessHandle} accessHandle + */ + #readFileHeader(accessHandle) { + const headerView = new DataView(new ArrayBuffer(FILE_HEADER_SIZE)); + if (accessHandle.read(headerView, { at: 0 }) !== headerView.byteLength) { + return null; + } + + if (headerView.getUint32(0) !== MAGIC) return null; + + const checksum = new Checksum(); + checksum.update(new Uint8Array(headerView.buffer, 0, FILE_HEADER_SIZE - 8)); + if (!checksum.matches(headerView.getUint32(24), headerView.getUint32(28))) { + return null; + } + + return { + nextTxId: Number(headerView.getBigUint64(8)), + salt1: headerView.getUint32(16), + salt2: headerView.getUint32(20), + }; + } + + /** + * @param {number} offset + */ + #readFrame(offset) { + const headerView = new DataView(new ArrayBuffer(FRAME_HEADER_SIZE)); + if (this.#activeHandle.read(headerView, { at: offset }) !== headerView.byteLength) { + // EOF, not an error. + return null; + } + + // Verify the frame header salt values match the file header. + const frameSalt1 = headerView.getUint32(16); + const frameSalt2 = headerView.getUint32(20); + if (frameSalt1 !== this.#activeHeader.salt1 || frameSalt2 !== this.#activeHeader.salt2) { + // Not necessarily an error, could be from a restart without truncation. + return null; + } + + const payloadSize = (size => size === 1 ? 65536 : size)(headerView.getUint16(2)); + /** @type {Uint8Array} */ let payloadData; + if (payloadSize) { + payloadData = new Uint8Array(payloadSize); + const payloadBytesRead = this.#activeHandle.read( + payloadData, + { at: offset + FRAME_HEADER_SIZE } + ); + if (payloadBytesRead !== payloadSize) return null; + } + + const checksum = new Checksum(); + checksum.update(new Uint8Array(headerView.buffer, 0, FRAME_HEADER_SIZE - 8)); + if (payloadData) { + checksum.update(payloadData); + } + if (!checksum.matches(headerView.getUint32(24), headerView.getUint32(28))) { + // Not necessarily an error, could be from a restart without truncation. + return null; + } + + const frameType = headerView.getUint8(0); + if (frameType === FRAME_TYPE_PAGE) { + return { + frameType, + byteLength: FRAME_HEADER_SIZE + payloadSize, + pageOffset: Number(headerView.getBigUint64(8)), + pageData: payloadData, + }; + } else if (frameType === FRAME_TYPE_COMMIT) { + return { + frameType, + byteLength: FRAME_HEADER_SIZE, + flags: headerView.getUint8(1), + dbFileSize: Number(headerView.getBigUint64(8)), + }; + } else if (frameType === FRAME_TYPE_END) { + // Handling the end frame and new file header must be atomic, so + // we validate the new file header before returning the frame. + // If the file header is corrupt, the end frame effectively does + // not exist. + // + // A corrupt file header should be repaired by the next writer + // that attempts to swap WAL files. + const fileHeader = this.#readFileHeader(this.#getInactiveHandle()); + if (fileHeader?.salt1 !== ((this.#activeHeader.salt1 + 1) >>> 0)) return null; + + return { + frameType, + byteLength: FRAME_HEADER_SIZE, + fileHeader, + }; + } + throw new Error(`Invalid frame type: ${frameType}`); + } + + #writeFileHeader(prevSalt1 = this.#activeHeader.salt1) { + // Derive new values from the previous values. + const nextTxId = this.#txId + 1; + const salt1 = (prevSalt1 + 1) >>> 0; + const salt2 = Math.floor(Math.random() * 0xffffffff) >>> 0; + const headerView = new DataView(new ArrayBuffer(FILE_HEADER_SIZE)); + headerView.setUint32(0, MAGIC); + headerView.setBigUint64(8, BigInt(nextTxId)); + headerView.setUint32(16, salt1); + headerView.setUint32(20, salt2); + + const checksum = new Checksum(); + checksum.update(new Uint8Array(headerView.buffer, 0, FILE_HEADER_SIZE - 8)); + headerView.setUint32(24, checksum.s0); + headerView.setUint32(28, checksum.s1); + + // The even/odd parity of salt1 determines which file is written to. + const accessHandle = this.#waHandles[salt1 & 1]; + const bytesWritten = accessHandle.write(headerView, { at: 0 }); + if (bytesWritten !== headerView.byteLength) { + throw new Error('write failed'); + } + + return { nextTxId, salt1, salt2 }; + } +} + +// https://www.sqlite.org/fileformat.html#checksum_algorithm +class Checksum { + /** @type {number} */ s0 = 0; + /** @type {number} */ s1 = 0; + + /** + * @param {ArrayBuffer|ArrayBufferView} data + */ + update(data) { + if ((data.byteLength % 8) !== 0) throw new Error('Data must be a multiple of 8 bytes'); + const words = ArrayBuffer.isView(data) ? + new Uint32Array(data.buffer, data.byteOffset, data.byteLength / 4) : + new Uint32Array(data); + for (let i = 0; i < words.length; i += 2) { + this.s0 = (this.s0 + words[i] + this.s1) >>> 0; + this.s1 = (this.s1 + words[i + 1] + this.s0) >>> 0; + } + } + + matches(s0, s1) { + return this.s0 === s0 && this.s1 === s1; + } +} diff --git a/src/wa-sqlite-private.d.ts b/src/wa-sqlite-private.d.ts index 49a571c..a6da628 100644 --- a/src/wa-sqlite-private.d.ts +++ b/src/wa-sqlite-private.d.ts @@ -58,36 +58,6 @@ declare module '@journeyapps/wa-sqlite/src/examples/LazyLock.js' { } } -declare module '@journeyapps/wa-sqlite/src/examples/WriteAhead.js' { - type WriteAheadDurability = 'relaxed' | 'strict'; - - export class WriteAhead { - log: ((...args: unknown[]) => void) | null; - options: { - autoCheckpoint: number; - backstopInterval: number; - journalSizeLimit: number; - }; - constructor(dbName: string, dbHandle: object, writeAheadHandles: [object, object]); - checkpoint(options?: { isPassive?: boolean }): Promise; - close(): void; - commit(): void; - getFileSize(): number; - getWriteAheadSize(): number; - isolateForRead(): void; - isolateForWrite(): void; - isTransactionPending(): boolean; - read(offset: number): Uint8Array | undefined; - ready(): Promise; - rejoin(): void; - rollback(): void; - setBackstopInterval(value: number): void; - sync(options?: { durability?: WriteAheadDurability }): void; - truncate(size: number): void; - write(offset: number, data: Uint8Array, options?: { dstPageSize?: number | null }): void; - } -} - declare module '@journeyapps/wa-sqlite/dist/wa-sqlite.mjs' { const factory: () => Promise; export default factory; diff --git a/tests/browser/memoryWriteAheadVfs.spec.ts b/tests/browser/memoryWriteAheadVfs.spec.ts index 1325ae2..8fbcf23 100644 --- a/tests/browser/memoryWriteAheadVfs.spec.ts +++ b/tests/browser/memoryWriteAheadVfs.spec.ts @@ -2,9 +2,13 @@ import { expect, type Page, test } from '@playwright/test'; const browserFixtureNames = [ 'runAbandonedFileLockWorkerRecovery', + 'runAbandonedAttachedTransactionRecovery', 'runAbandonedWorkerLeaseRecovery', + 'runAttachedDatabaseAtomicTransaction', + 'runAttachedDatabaseConcurrentWrites', 'runCapacityExceededFailure', 'runDuplicatePendingOpenRejection', + 'runDeferredWriteUpgrade', 'runFileLockRegression', 'runMultipleRuntimeIsolation', 'runOwnerLeaseClaimRecovery', @@ -19,9 +23,11 @@ const browserFixtureNames = [ 'runSegmentReclaimAccounting', 'runSegmentReclaimAccountingSaturation', 'runSnapshotIsolation', + 'runStaleDeferredWriteRejection', 'runStaleOwnedLeaseSlotPreservation', 'runWalRoundTrip', 'runWorkerConcurrency', + 'runZeroByteWriteAndDiagnostics', ] as const; type BrowserApi = typeof import('../fixtures/browserApi.js'); @@ -80,6 +86,9 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.count).toBe(40); expect(result.payloadBytes).toBe(40 * 2048); + expect(result.checkpointCount).toBeGreaterThanOrEqual(1); + expect(result.checkpointWriteAheadPages).toBeGreaterThan(0); + expect(result.lastCheckpointWriteAheadPages).toBeGreaterThan(0); expect(result.writeAheadExpandedBeforeCheckpoint).toBe(true); expect(result.writeAheadLogicalSizeBeforeCheckpoint).toBeGreaterThan(0); expect(result.writeAheadSegmentCountAfterCheckpoint).toBeGreaterThanOrEqual(2); @@ -103,6 +112,18 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.reclaimedSegmentCount).toBe(0x7fffffff); }); + test('extends logical size for zero-byte writes and records high-water diagnostics', async ({ page }) => { + const result = await runBrowserFixture(page, 'runZeroByteWriteAndDiagnostics'); + + expect(result.bytesWritten).toBe(0); + expect(result.logicalSizeBytes).toBe(64 * 1024); + expect(result.peakLogicalSizeBytes).toBe(64 * 1024); + expect(result.peakCapacityBytes).toBeGreaterThanOrEqual(64 * 1024); + expect(result.peakAllocatedCapacityBytes).toBeGreaterThanOrEqual(64 * 1024); + expect(result.writeOperationCount).toBe(1); + expect(result.writtenBytes).toBe(0); + }); + test('fails at configured capacity and logs the underlying VFS capacity error', async ({ page }) => { const result = await runBrowserFixture(page, 'runCapacityExceededFailure'); @@ -119,6 +140,36 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.after).toEqual(['before', 'during-1', 'during-2']); }); + test('upgrades deferred and savepoint transactions from reads to writes', async ({ page }) => { + const rows = await runBrowserFixture(page, 'runDeferredWriteUpgrade'); + + expect(rows).toEqual(['warmup', 'transaction', 'savepoint']); + }); + + test('rejects a deferred write when another connection made its snapshot stale', async ({ page }) => { + const result = await runBrowserFixture(page, 'runStaleDeferredWriteRejection'); + + expect(result.rejectionMessage).toContain('database is locked'); + expect(result.detailedRejectionMessage).toContain('snapshot is stale'); + expect(result.detailedRejectionMessage).not.toContain('Not in write isolated state'); + expect(result.rows).toEqual(['warmup', 'concurrent', 'recovered']); + }); + + test('commits atomically across the primary and an attached shared database', async ({ page }) => { + const result = await runBrowserFixture(page, 'runAttachedDatabaseAtomicTransaction'); + + expect(result.mainRows).toEqual(['main']); + expect(result.attachedRows).toEqual(['attached']); + expect(result.fileCount).toBe(6); + }); + + test('serializes concurrent transactions opened directly on an attached shared database', async ({ page }) => { + const result = await runBrowserFixture(page, 'runAttachedDatabaseConcurrentWrites'); + + expect(result.value).toBe(40); + expect(result.integrity).toBe('ok'); + }); + test('preserves rollback semantics and VFS pragma round-trips', async ({ page }) => { const result = await runBrowserFixture(page, 'runRollbackAndPragmaBehavior'); @@ -230,14 +281,26 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.openHandleCountAfterClose).toBe(0); }); - test('recovers abandoned worker runtime leases after stale heartbeat', async ({ page }) => { + test('recovers terminated-worker runtime leases immediately', async ({ page }) => { const result = await runBrowserFixture(page, 'runAbandonedWorkerLeaseRecovery'); expect(result.openHandleCountBeforeTerminate).toBe(3); expect(result.activeOwnerLeaseCountBeforeTerminate).toBe(1); + expect(result.recoveredHandleCount).toBe(3); expect(result.diagnosticsAfterReset).toBeNull(); }); + test('quarantines an attached database when its transaction owner terminates', async ({ page }) => { + const result = await runBrowserFixture(page, 'runAbandonedAttachedTransactionRecovery'); + + expect(result.recoveredHandleCount).toBeGreaterThan(0); + expect(result.transactionLockBeforeTerminate).toBeGreaterThan(0); + expect(result.transactionLockAfterRecovery).toBe(result.transactionLockBeforeTerminate); + expect(result.transactionLockOwnerAfterRecovery).toBeGreaterThan(0); + expect(result.attachedReadMessage).toContain('database is locked'); + expect(result.lastVfsError).toContain('quarantined'); + }); + test('keeps named runtimes isolated when one runtime is reset', async ({ page }) => { const result = await runBrowserFixture(page, 'runMultipleRuntimeIsolation'); diff --git a/tests/fixtures/browserApi.ts b/tests/fixtures/browserApi.ts index 4127bad..3084bb7 100644 --- a/tests/fixtures/browserApi.ts +++ b/tests/fixtures/browserApi.ts @@ -6,8 +6,10 @@ import { MEMORY_WRITE_AHEAD_VFS, MemoryWriteAheadVFS, createMemoryWriteAheadSharedRuntime, + ensureMemoryWriteAheadDatabaseFiles, getMemoryWriteAheadRuntimeDiagnostics, normalizeMemoryWriteAheadPathname, + recoverTerminatedMemoryWriteAheadRuntimeOwner, resolveMemoryWriteAheadRuntimeOptions, type MemoryWriteAheadRuntimeOptions, type MemoryWriteAheadSharedRuntime, @@ -310,7 +312,10 @@ export const runSegmentGrowthAndCheckpointReclaim = async () => { return { count, + checkpointCount: databaseFileAfter?.checkpointCount ?? 0, + checkpointWriteAheadPages: databaseFileAfter?.checkpointWriteAheadPages ?? 0, databaseSegmentCountAfterCheckpoint: databaseFileAfter?.segmentCount ?? 0, + lastCheckpointWriteAheadPages: databaseFileAfter?.lastCheckpointWriteAheadPages ?? 0, payloadBytes, writeAheadLogicalSizeBeforeCheckpoint: writeAheadFilesBefore.reduce( (sum, file) => sum + file.logicalSizeBytes, @@ -395,6 +400,39 @@ export const runSegmentReclaimAccountingSaturation = () => { } }; +export const runZeroByteWriteAndDiagnostics = () => { + ensureCrossOriginIsolated(); + + const dbFilename = nextDbFilename('zero-byte-write'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename, { + initialDatabaseCapacityBytes: 64 * 1024, + maxDatabaseCapacityBytes: 128 * 1024, + initialWriteAheadCapacityBytes: 16 * 1024, + maxWriteAheadCapacityBytes: 128 * 1024, + }); + const file = getMemoryWriteAheadFile(runtime, dbFilename); + if (!file) { + throw new Error(`Missing MemoryWriteAhead file for ${dbFilename}`); + } + + try { + const bytesWritten = writeMemoryWriteAheadFile(file, new Uint8Array(0), 64 * 1024); + const diagnostics = fileDiagnosticsFor(dbFilename, dbFilename); + + return { + bytesWritten, + logicalSizeBytes: diagnostics.logicalSizeBytes, + peakAllocatedCapacityBytes: diagnostics.peakAllocatedCapacityBytes, + peakCapacityBytes: diagnostics.peakCapacityBytes, + peakLogicalSizeBytes: diagnostics.peakLogicalSizeBytes, + writeOperationCount: diagnostics.writeOperationCount, + writtenBytes: diagnostics.writtenBytes, + }; + } finally { + resetRuntime(dbFilename); + } +}; + export const runOwnerLeaseClaimRecovery = () => { ensureCrossOriginIsolated(); @@ -872,6 +910,154 @@ export const runSnapshotIsolation = async () => { } }; +export const runDeferredWriteUpgrade = async () => { + const dbFilename = nextDbFilename('deferred-write'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + + try { + await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('PRAGMA journal_mode=WAL'); + await connection.exec('CREATE TABLE deferred_rows (id INTEGER PRIMARY KEY, value TEXT NOT NULL)'); + await connection.exec("INSERT INTO deferred_rows (value) VALUES ('warmup')"); + }); + + return await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('BEGIN'); + await connection.exec('SELECT COUNT(*) FROM deferred_rows'); + await connection.exec("INSERT INTO deferred_rows (value) VALUES ('transaction')"); + await connection.exec('COMMIT'); + + await connection.exec('SAVEPOINT top_level'); + await connection.exec('SELECT COUNT(*) FROM deferred_rows'); + await connection.exec("INSERT INTO deferred_rows (value) VALUES ('savepoint')"); + await connection.exec('RELEASE SAVEPOINT top_level'); + + const rows = await connection.exec('SELECT value FROM deferred_rows ORDER BY id'); + return rows.rows.map(([value]) => value); + }); + } finally { + await waitForOpenHandleCount(dbFilename, 0); + resetRuntime(dbFilename); + } +}; + +export const runAttachedDatabaseConcurrentWrites = async () => { + const dbFilename = nextDbFilename('attached-lock-owner'); + const attachedDbFilename = nextDbFilename('attached-lock-target'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + ensureMemoryWriteAheadDatabaseFiles(runtime, attachedDbFilename); + const connections: BrowserSqliteConnection[] = []; + + const increment = async (connection: BrowserSqliteConnection, iterations: number) => { + for (let index = 0; index < iterations; index += 1) { + await connection.exec('BEGIN'); + const current = await connection.exec('SELECT value FROM counter WHERE id = 1'); + await connection.exec(`UPDATE counter SET value = ${firstNumber(current.rows) + 1} WHERE id = 1`); + await connection.exec('COMMIT'); + } + }; + + try { + await withMemoryWriteAheadDatabase(attachedDbFilename, runtime, async (connection) => { + await connection.exec('CREATE TABLE counter (id INTEGER PRIMARY KEY, value INTEGER NOT NULL)'); + await connection.exec('INSERT INTO counter (id, value) VALUES (1, 0)'); + }); + + const first = await openMemoryWriteAheadDatabase(attachedDbFilename, runtime); + connections.push(first); + const second = await openMemoryWriteAheadDatabase(attachedDbFilename, runtime); + connections.push(second); + + await Promise.all([increment(first, 20), increment(second, 20)]); + const counter = await first.exec('SELECT value FROM counter WHERE id = 1'); + const integrity = await first.exec('PRAGMA integrity_check'); + + return { + integrity: firstCell(integrity.rows), + value: firstNumber(counter.rows), + }; + } finally { + await Promise.all(connections.map((connection) => connection.close())); + await waitForOpenHandleCount(dbFilename, 0); + resetRuntime(dbFilename); + } +}; + +export const runStaleDeferredWriteRejection = async () => { + const dbFilename = nextDbFilename('stale-deferred-write'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + const connections: BrowserSqliteConnection[] = []; + + try { + const connection = await openMemoryWriteAheadDatabase(dbFilename, runtime); + connections.push(connection); + await connection.exec('PRAGMA journal_mode=WAL'); + await connection.exec('CREATE TABLE deferred_rows (id INTEGER PRIMARY KEY, value TEXT NOT NULL)'); + await connection.exec('BEGIN IMMEDIATE'); + await connection.exec("INSERT INTO deferred_rows (value) VALUES ('warmup')"); + await connection.exec('COMMIT'); + + const otherConnection = await openMemoryWriteAheadDatabase(dbFilename, runtime); + connections.push(otherConnection); + await connection.exec('BEGIN'); + await connection.exec('SELECT COUNT(*) FROM deferred_rows'); + await otherConnection.exec("INSERT INTO deferred_rows (value) VALUES ('concurrent')"); + const rejectionMessage = await captureErrorMessage(() => + connection.exec("INSERT INTO deferred_rows (value) VALUES ('stale')") + ); + const detailedRejectionMessage = connection.getLastVfsError(); + + await connection.exec('ROLLBACK').catch(() => undefined); + await connection.exec("INSERT INTO deferred_rows (value) VALUES ('recovered')"); + const rows = await connection.exec('SELECT value FROM deferred_rows ORDER BY id'); + + return { + detailedRejectionMessage, + rejectionMessage, + rows: rows.rows.map(([value]) => value), + }; + } finally { + await Promise.all(connections.map((connection) => connection.close())); + await waitForOpenHandleCount(dbFilename, 0); + resetRuntime(dbFilename); + } +}; + +export const runAttachedDatabaseAtomicTransaction = async () => { + const dbFilename = nextDbFilename('attached-main'); + const attachedDbFilename = nextDbFilename('attached-secondary'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + ensureMemoryWriteAheadDatabaseFiles(runtime, attachedDbFilename); + + try { + await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('PRAGMA journal_mode=WAL'); + await connection.exec('CREATE TABLE main_rows (id INTEGER PRIMARY KEY, value TEXT NOT NULL)'); + await connection.exec(`ATTACH DATABASE '${attachedDbFilename}' AS secondary`); + await connection.exec('CREATE TABLE secondary.attached_rows (id INTEGER PRIMARY KEY, value TEXT NOT NULL)'); + await connection.exec('BEGIN IMMEDIATE'); + await connection.exec("INSERT INTO main_rows (value) VALUES ('main')"); + await connection.exec("INSERT INTO secondary.attached_rows (value) VALUES ('attached')"); + await connection.exec('COMMIT'); + }); + + return await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec(`ATTACH DATABASE '${attachedDbFilename}' AS secondary`); + const mainRows = await connection.exec('SELECT value FROM main_rows ORDER BY id'); + const attachedRows = await connection.exec('SELECT value FROM secondary.attached_rows ORDER BY id'); + + return { + attachedRows: attachedRows.rows.map(([value]) => value), + fileCount: diagnosticsFor(dbFilename).files.length, + mainRows: mainRows.rows.map(([value]) => value), + }; + }); + } finally { + await waitForOpenHandleCount(dbFilename, 0); + resetRuntime(dbFilename); + } +}; + export const runRollbackAndPragmaBehavior = async () => { const dbFilename = nextDbFilename('rollback'); const runtime = createMemoryWriteAheadSharedRuntime(dbFilename, { @@ -1040,7 +1226,7 @@ export const runWorkerConcurrency = async () => { } }; -type LeaseWorkerMessage = { ok: true } | { ok: false; message: string; stack?: string }; +type LeaseWorkerMessage = { ok: true; ownerId: number } | { ok: false; message: string; stack?: string }; const waitForLeaseWorkerReady = (worker: Worker) => new Promise((resolve, reject) => { @@ -1076,13 +1262,92 @@ export const runAbandonedWorkerLeaseRecovery = async () => { const beforeTerminate = await waitForOpenHandleCount(dbFilename, 3); worker.terminate(); - await wait(350); + const recoveredHandleCount = recoverTerminatedMemoryWriteAheadRuntimeOwner(runtime, ready.ownerId); + await waitForOpenHandleCount(dbFilename, 0); resetRuntime(dbFilename); return { activeOwnerLeaseCountBeforeTerminate: beforeTerminate.activeOwnerLeaseCount, diagnosticsAfterReset: getMemoryWriteAheadRuntimeDiagnostics(dbFilename), openHandleCountBeforeTerminate: beforeTerminate.openHandleCount, + recoveredHandleCount, + }; + } finally { + worker.terminate(); + if (getMemoryWriteAheadRuntimeDiagnostics(dbFilename)) { + await waitForOpenHandleCount(dbFilename, 0).catch(() => undefined); + resetRuntime(dbFilename); + } + } +}; + +export const runAbandonedAttachedTransactionRecovery = async () => { + const dbFilename = nextDbFilename('lease-dirty-main'); + const attachedDbFilename = nextDbFilename('lease-dirty-attached'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename, { + initialDatabaseCapacityBytes: 64 * 1024, + maxDatabaseCapacityBytes: 16 * 1024 * 1024, + initialWriteAheadCapacityBytes: 16 * 1024, + maxWriteAheadCapacityBytes: 16 * 1024 * 1024, + fileLockStaleMs: 100, + }); + const attachedFile = ensureMemoryWriteAheadDatabaseFiles(runtime, attachedDbFilename).database; + const attachedMeta = new Int32Array(attachedFile.meta); + const worker = new Worker(new URL('./leaseWorker.ts', import.meta.url), { type: 'module' }); + + try { + await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('PRAGMA journal_mode=WAL'); + await connection.exec(`ATTACH DATABASE '${attachedDbFilename}' AS secondary`); + await connection.exec( + 'CREATE TABLE secondary.attached_rows (id INTEGER PRIMARY KEY, payload BLOB NOT NULL)' + ); + await connection.exec( + `WITH RECURSIVE rows(id) AS ( + SELECT 1 + UNION ALL + SELECT id + 1 FROM rows WHERE id < 1000 + ) + INSERT INTO secondary.attached_rows (id, payload) + SELECT id, zeroblob(4000) FROM rows` + ); + }); + + const readyPromise = waitForLeaseWorkerReady(worker); + worker.postMessage({ attachedDbFilename, dbFilename, runtime }); + const ready = await readyPromise; + if (!ready.ok) { + throw new Error(`${ready.message}\n${ready.stack ?? ''}`); + } + + const transactionLockBeforeTerminate = Atomics.load(attachedMeta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock); + worker.terminate(); + const recoveredHandleCount = recoverTerminatedMemoryWriteAheadRuntimeOwner(runtime, ready.ownerId); + await waitForOpenHandleCount(dbFilename, 0); + + let attachedReadMessage = ''; + let lastVfsError = ''; + await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('PRAGMA journal_mode=WAL'); + attachedReadMessage = await captureErrorMessage(() => + connection.exec(`ATTACH DATABASE '${attachedDbFilename}' AS secondary`) + ); + lastVfsError = connection.getLastVfsError() ?? ''; + }); + + const transactionLockAfterRecovery = Atomics.load(attachedMeta, MEMORY_WRITE_AHEAD_FILE_META.transactionLock); + const transactionLockOwnerAfterRecovery = Atomics.load( + attachedMeta, + MEMORY_WRITE_AHEAD_FILE_META.transactionLockRuntimeOwner + ); + + return { + attachedReadMessage, + lastVfsError, + recoveredHandleCount, + transactionLockAfterRecovery, + transactionLockBeforeTerminate, + transactionLockOwnerAfterRecovery, }; } finally { worker.terminate(); diff --git a/tests/fixtures/leaseWorker.ts b/tests/fixtures/leaseWorker.ts index 2133d79..cde4725 100644 --- a/tests/fixtures/leaseWorker.ts +++ b/tests/fixtures/leaseWorker.ts @@ -1,12 +1,13 @@ import { openMemoryWriteAheadDatabase } from './sqliteHarness.js'; -import type { MemoryWriteAheadSharedRuntime } from '../../src/index.js'; +import { getMemoryWriteAheadRuntimeOwnerId, type MemoryWriteAheadSharedRuntime } from '../../src/index.js'; type LeaseWorkerRequest = { + attachedDbFilename?: string; dbFilename: string; runtime: MemoryWriteAheadSharedRuntime; }; -type LeaseWorkerResponse = { ok: true } | { ok: false; message: string; stack?: string }; +type LeaseWorkerResponse = { ok: true; ownerId: number } | { ok: false; message: string; stack?: string }; let keepAliveConnection: Awaited> | null = null; @@ -14,9 +15,19 @@ self.onmessage = async (event: MessageEvent) => { try { keepAliveConnection = await openMemoryWriteAheadDatabase(event.data.dbFilename, event.data.runtime); await keepAliveConnection.exec('PRAGMA journal_mode=WAL'); + if (event.data.attachedDbFilename) { + await keepAliveConnection.exec(`ATTACH DATABASE '${event.data.attachedDbFilename}' AS secondary`); + await keepAliveConnection.exec('PRAGMA secondary.cache_size=5'); + await keepAliveConnection.exec('BEGIN'); + await keepAliveConnection.exec('UPDATE secondary.attached_rows SET payload = zeroblob(3500)'); + } await keepAliveConnection.exec('CREATE TABLE IF NOT EXISTS lease_rows (value TEXT NOT NULL)'); await keepAliveConnection.exec("INSERT INTO lease_rows (value) VALUES ('leased')"); - const response: LeaseWorkerResponse = { ok: true }; + const ownerId = getMemoryWriteAheadRuntimeOwnerId(event.data.runtime); + if (!ownerId) { + throw new Error('MemoryWriteAhead runtime owner was not registered'); + } + const response: LeaseWorkerResponse = { ok: true, ownerId }; self.postMessage(response); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); diff --git a/tests/fixtures/sqliteHarness.ts b/tests/fixtures/sqliteHarness.ts index e8cb5df..a045220 100644 --- a/tests/fixtures/sqliteHarness.ts +++ b/tests/fixtures/sqliteHarness.ts @@ -19,6 +19,7 @@ type QueryResult = { export type BrowserSqliteConnection = { close(): Promise; exec(sql: string): Promise; + getLastVfsError(): string | null; }; export const openMemoryWriteAheadDatabase = async ( @@ -54,6 +55,9 @@ export const openMemoryWriteAheadDatabase = async ( return { columns, rows }; }, + getLastVfsError() { + return vfs.lastError?.message ?? null; + }, }; }; diff --git a/tsconfig.json b/tsconfig.json index 0edc1a7..4358fbe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,5 +15,13 @@ "verbatimModuleSyntax": true, "allowSyntheticDefaultImports": true }, - "include": ["src/**/*.ts", "tests/**/*.ts", "tests/**/*.d.ts", "playwright.config.ts", "vite.config.ts"] + "include": [ + "src/**/*.ts", + "tests/**/*.ts", + "tests/**/*.d.ts", + "benchmarks/**/*.ts", + "playwright.config.ts", + "playwright.benchmark.config.ts", + "vite.config.ts" + ] }