From 2626790f540393667b040145bb0ff23e29b802ba Mon Sep 17 00:00:00 2001 From: Yahya JIRARI Date: Wed, 7 Oct 2026 18:56:08 +0200 Subject: [PATCH 1/4] fix: name the allocator error when a growable SharedArrayBuffer reservation fails When the engine supports growable SharedArrayBuffer but refuses the reservation itself (for example once a page has reserved more address space than the browser allows), runtime creation threw "MemoryWriteAheadVFS requires growable SharedArrayBuffer support" and dropped the allocator's error. That message points at a missing browser feature, so a user could not tell an exhausted reservation budget from an unsupported engine. The error now reads "MemoryWriteAheadVFS could not reserve a growable SharedArrayBuffer: " and carries the allocator error as its cause. Engines that return a non-growable buffer still get the original message, and pages without cross-origin isolation keep their COOP/COEP hint. --- src/memoryWriteAheadSharedRuntime.ts | 15 ++++++++--- tests/browser/memoryWriteAheadVfs.spec.ts | 11 ++++++++ tests/fixtures/browserApi.ts | 32 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/memoryWriteAheadSharedRuntime.ts b/src/memoryWriteAheadSharedRuntime.ts index 8845df9..c97997f 100644 --- a/src/memoryWriteAheadSharedRuntime.ts +++ b/src/memoryWriteAheadSharedRuntime.ts @@ -564,7 +564,7 @@ const ensureLocalRuntimeLease = ( // Without COOP/COEP response headers a browser context is not cross-origin isolated and exposes // no SharedArrayBuffer at all; the generic capability error would hide that deployment mistake. -const createGrowableSharedArrayBufferSupportError = () => { +const createGrowableSharedArrayBufferSupportError = (reservationError?: unknown) => { if (globalThis.crossOriginIsolated === false) { const error = new Error( 'MemoryWriteAheadVFS requires growable SharedArrayBuffer support: this page is not cross-origin isolated. Serve it with the Cross-Origin-Opener-Policy: same-origin and Cross-Origin-Embedder-Policy: require-corp response headers.' @@ -576,6 +576,15 @@ const createGrowableSharedArrayBufferSupportError = () => { return error; } + if (reservationError !== undefined) { + return new Error( + `MemoryWriteAheadVFS could not reserve a growable SharedArrayBuffer: ${ + reservationError instanceof Error ? reservationError.message : String(reservationError) + }`, + { cause: reservationError } + ); + } + return new Error('MemoryWriteAheadVFS requires growable SharedArrayBuffer support'); }; @@ -607,9 +616,9 @@ const createSharedSegmentData = ( data: buffer, }; } - } catch (_error) { + } catch (reservationError) { if (!allowFixedCapacityFallback) { - throw createGrowableSharedArrayBufferSupportError(); + throw createGrowableSharedArrayBufferSupportError(reservationError); } } diff --git a/tests/browser/memoryWriteAheadVfs.spec.ts b/tests/browser/memoryWriteAheadVfs.spec.ts index 8fbcf23..7127ad0 100644 --- a/tests/browser/memoryWriteAheadVfs.spec.ts +++ b/tests/browser/memoryWriteAheadVfs.spec.ts @@ -14,6 +14,7 @@ const browserFixtureNames = [ 'runOwnerLeaseClaimRecovery', 'runOwnerLeaseExhaustionRecovery', 'runReadmeQuickstart', + 'runRejectedGrowableReservation', 'runRollbackAndPragmaBehavior', 'runRuntimeLifecycle', 'runRuntimeOptionValidation', @@ -61,6 +62,16 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { ]); }); + test('names the allocator error when a growable SharedArrayBuffer reservation is rejected', async ({ page }) => { + const result = await runBrowserFixture(page, 'runRejectedGrowableReservation'); + + expect(result.message).toBe( + 'MemoryWriteAheadVFS could not reserve a growable SharedArrayBuffer: Array buffer allocation failed' + ); + expect(result.causeIsAllocatorError).toBe(true); + expect(result.runtimeRegistered).toBe(false); + }); + test('tracks lifecycle diagnostics and refuses reset while handles are open', async ({ page }) => { const result = await runBrowserFixture(page, 'runRuntimeLifecycle'); diff --git a/tests/fixtures/browserApi.ts b/tests/fixtures/browserApi.ts index 3084bb7..388cd94 100644 --- a/tests/fixtures/browserApi.ts +++ b/tests/fixtures/browserApi.ts @@ -177,6 +177,38 @@ export const runRuntimeOptionValidation = () => { }; }; +export const runRejectedGrowableReservation = () => { + ensureCrossOriginIsolated(); + + const dbFilename = nextDbFilename('rejected-reservation'); + const reservationError = new RangeError('Array buffer allocation failed'); + const NativeSharedArrayBuffer = globalThis.SharedArrayBuffer; + globalThis.SharedArrayBuffer = new Proxy(NativeSharedArrayBuffer, { + construct(target, [byteLength, options]: [number, { maxByteLength?: number }?]) { + if (options?.maxByteLength !== undefined) { + throw reservationError; + } + + return new target(byteLength); + }, + }); + + try { + createMemoryWriteAheadSharedRuntime(dbFilename); + } catch (error) { + return { + causeIsAllocatorError: error instanceof Error && error.cause === reservationError, + message: error instanceof Error ? error.message : String(error), + runtimeRegistered: getMemoryWriteAheadRuntimeDiagnostics(dbFilename) !== null, + }; + } finally { + globalThis.SharedArrayBuffer = NativeSharedArrayBuffer; + } + + resetRuntime(dbFilename); + throw new Error('Expected runtime creation to fail when the growable reservation is rejected'); +}; + export const runRuntimeLifecycle = async () => { ensureCrossOriginIsolated(); From 90e6da0fa5084eb6a4a2f10b78d2bf45d96d746b Mon Sep 17 00:00:00 2001 From: Yahya JIRARI Date: Wed, 7 Oct 2026 18:57:18 +0200 Subject: [PATCH 2/4] fix: stop reserving write-ahead files for attached databases ensureMemoryWriteAheadDatabaseFiles gave every attached database a -wa0/-wa1 pair next to its database file. Only the runtime's primary database is ever opened with a WriteAhead; an attached database takes the rollback-journal path and never reads those files. With the default options each attached database therefore reserved an extra 1 GiB of growable SharedArrayBuffer address space and committed 16 MiB of memory it could not use. A browser that caps the total reservation per page (seen on iOS WebKit) refused to attach databases sooner than it had to. An attached database now gets its database file only, and `writeAhead` is an empty list for it; the primary database still gets its pair. Diagnostics list one file per attached database instead of three. The `writeAhead` type is now `MemoryWriteAheadSharedFile[]` instead of a two-element tuple, and the README example registers the attached database file alone. --- README.md | 4 +-- src/memoryWriteAheadSharedRuntime.ts | 21 ++++++++----- tests/browser/memoryWriteAheadVfs.spec.ts | 12 ++++++- tests/fixtures/browserApi.ts | 38 +++++++++++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b125bbc..c4bc006 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ When an application deliberately terminates a worker, it can recover that worker ## Attached databases -Register each attached database in the shared runtime before executing `ATTACH DATABASE`. While the owning context remains live, atomic transactions spanning the primary and attached databases are coordinated through SQLite's super-journal path. Transactions opened directly on an attached database are serialized across connections because attached databases use SQLite's rollback-journal protocol rather than the primary database's write-ahead path. +Register each attached database in the shared runtime before executing `ATTACH DATABASE`. While the owning context remains live, atomic transactions spanning the primary and attached databases are coordinated through SQLite's super-journal path. Transactions opened directly on an attached database are serialized across connections because attached databases use SQLite's rollback-journal protocol rather than the primary database's write-ahead path. For the same reason an attached database reserves only its database file: `ensureMemoryWriteAheadDatabaseFiles` returns an empty `writeAhead` list for any filename other than the runtime's primary database. ```ts import { ensureMemoryWriteAheadDatabaseFiles } from 'memory-write-ahead-vfs'; @@ -149,7 +149,7 @@ import { ensureMemoryWriteAheadDatabaseFiles, registerMemoryWriteAheadSharedFile // Worker that first opens the attached database: const attachedFiles = ensureMemoryWriteAheadDatabaseFiles(workerRuntime, '/analytics.sqlite'); -postMessage({ files: [attachedFiles.database, ...attachedFiles.writeAhead] }); +postMessage({ files: [attachedFiles.database] }); // Runtime-owning parent after receiving that structured clone: registerMemoryWriteAheadSharedFiles({ diff --git a/src/memoryWriteAheadSharedRuntime.ts b/src/memoryWriteAheadSharedRuntime.ts index c97997f..4b0c62b 100644 --- a/src/memoryWriteAheadSharedRuntime.ts +++ b/src/memoryWriteAheadSharedRuntime.ts @@ -122,7 +122,7 @@ export type MemoryWriteAheadSharedRuntime = { export type MemoryWriteAheadDatabaseFiles = { database: MemoryWriteAheadSharedFile; - writeAhead: [MemoryWriteAheadSharedFile, MemoryWriteAheadSharedFile]; + writeAhead: MemoryWriteAheadSharedFile[]; }; export type MemoryWriteAheadSharedFilesRegistration = { @@ -936,14 +936,21 @@ export const ensureMemoryWriteAheadDatabaseFiles = ( dbFilename: string ): MemoryWriteAheadDatabaseFiles => { const normalizedDbFilename = normalizeMemoryWriteAheadPathname(dbFilename); + const database = ensureSharedFile( + runtime, + normalizedDbFilename, + runtime.capacities.initialDatabaseCapacityBytes, + runtime.capacities.maxDatabaseCapacityBytes + ); + + // Only the runtime's primary database is opened with a WriteAhead. Attached databases use SQLite's rollback + // journal, so a -wa0/-wa1 pair for them would only consume growable SharedArrayBuffer reservation. + if (normalizedDbFilename !== runtime.dbFilename) { + return { database, writeAhead: [] }; + } return { - database: ensureSharedFile( - runtime, - normalizedDbFilename, - runtime.capacities.initialDatabaseCapacityBytes, - runtime.capacities.maxDatabaseCapacityBytes - ), + database, writeAhead: [ ensureSharedFile( runtime, diff --git a/tests/browser/memoryWriteAheadVfs.spec.ts b/tests/browser/memoryWriteAheadVfs.spec.ts index 7127ad0..26ddfd1 100644 --- a/tests/browser/memoryWriteAheadVfs.spec.ts +++ b/tests/browser/memoryWriteAheadVfs.spec.ts @@ -6,6 +6,7 @@ const browserFixtureNames = [ 'runAbandonedWorkerLeaseRecovery', 'runAttachedDatabaseAtomicTransaction', 'runAttachedDatabaseConcurrentWrites', + 'runAttachedDatabaseWithoutWriteAheadFiles', 'runCapacityExceededFailure', 'runDuplicatePendingOpenRejection', 'runDeferredWriteUpgrade', @@ -171,7 +172,16 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.mainRows).toEqual(['main']); expect(result.attachedRows).toEqual(['attached']); - expect(result.fileCount).toBe(6); + expect(result.fileCount).toBe(4); + }); + + test('reserves no write-ahead files for an attached database', async ({ page }) => { + const result = await runBrowserFixture(page, 'runAttachedDatabaseWithoutWriteAheadFiles'); + + expect(result.primaryWriteAheadFileCount).toBe(2); + expect(result.attachedWriteAheadFileCount).toBe(0); + expect(result.pathnames).toEqual(['main', 'main-wa0', 'main-wa1', 'attached']); + expect(result.attachedRows).toEqual(['attached']); }); test('serializes concurrent transactions opened directly on an attached shared database', async ({ page }) => { diff --git a/tests/fixtures/browserApi.ts b/tests/fixtures/browserApi.ts index 388cd94..b711d04 100644 --- a/tests/fixtures/browserApi.ts +++ b/tests/fixtures/browserApi.ts @@ -1090,6 +1090,44 @@ export const runAttachedDatabaseAtomicTransaction = async () => { } }; +export const runAttachedDatabaseWithoutWriteAheadFiles = async () => { + const dbFilename = nextDbFilename('attached-files-main'); + const attachedDbFilename = nextDbFilename('attached-files-secondary'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + const primaryFiles = ensureMemoryWriteAheadDatabaseFiles(runtime, dbFilename); + const attachedFiles = ensureMemoryWriteAheadDatabaseFiles(runtime, attachedDbFilename); + const labelPathname = (pathname: string) => + [ + { filename: dbFilename, label: 'main' }, + { filename: attachedDbFilename, label: 'attached' }, + ].reduce( + (labelled, { filename, label }) => + labelled.startsWith(filename) ? `${label}${labelled.slice(filename.length)}` : labelled, + pathname + ); + + try { + const attachedRows = await withMemoryWriteAheadDatabase(dbFilename, runtime, async (connection) => { + await connection.exec('PRAGMA journal_mode=WAL'); + await connection.exec(`ATTACH DATABASE '${attachedDbFilename}' AS secondary`); + await connection.exec('CREATE TABLE secondary.attached_rows (id INTEGER PRIMARY KEY, value TEXT NOT NULL)'); + await connection.exec("INSERT INTO secondary.attached_rows (value) VALUES ('attached')"); + const rows = await connection.exec('SELECT value FROM secondary.attached_rows ORDER BY id'); + return rows.rows.map(([value]) => value); + }); + + return { + attachedRows, + attachedWriteAheadFileCount: attachedFiles.writeAhead.length, + pathnames: diagnosticsFor(dbFilename).files.map((file) => labelPathname(file.pathname)), + primaryWriteAheadFileCount: primaryFiles.writeAhead.length, + }; + } finally { + await waitForOpenHandleCount(dbFilename, 0); + resetRuntime(dbFilename); + } +}; + export const runRollbackAndPragmaBehavior = async () => { const dbFilename = nextDbFilename('rollback'); const runtime = createMemoryWriteAheadSharedRuntime(dbFilename, { From 895a58222dab6e8c4d03161eda89f04b7ac3d306 Mon Sep 17 00:00:00 2001 From: Yahya JIRARI Date: Wed, 7 Oct 2026 18:59:13 +0200 Subject: [PATCH 3/4] fix: let the clean-slate runtime reset finish when a handle was never released resetMemoryWriteAheadSharedRuntime() without a filename (also reached through MemoryWriteAheadVFS.resetRuntime()) is meant to hand the next caller a clean slate, but it threw on the first runtime that still counted an open handle. Runtimes visited before that one were already zeroed yet stayed registered, runtimes after it were left untouched, and the registry was never cleared. One connection that a test forgot to close therefore made every later createMemoryWriteAheadSharedRuntime call reuse stale, half-reset files. The no-filename form now resets and deregisters every runtime, open handles or not. The targeted form is unchanged: it still refuses while its runtime has open handles, and it refuses before touching any state, which a new test now pins. --- src/memoryWriteAheadSharedRuntime.ts | 15 +++--- tests/browser/memoryWriteAheadVfs.spec.ts | 19 +++++++ tests/fixtures/browserApi.ts | 65 +++++++++++++++++++++++ 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/memoryWriteAheadSharedRuntime.ts b/src/memoryWriteAheadSharedRuntime.ts index 4b0c62b..d7c0df5 100644 --- a/src/memoryWriteAheadSharedRuntime.ts +++ b/src/memoryWriteAheadSharedRuntime.ts @@ -1111,13 +1111,7 @@ export const recoverTerminatedMemoryWriteAheadRuntimeOwner = ( }; const resetMemoryWriteAheadRuntimeState = (runtime: MemoryWriteAheadSharedRuntime) => { - recoverAbandonedRuntimeOwnerLeases(runtime); const runtimeMeta = getRuntimeMeta(runtime); - const openHandleCount = getOpenHandleCount(runtime); - - if (openHandleCount > 0) { - throw new Error(`Cannot reset MemoryWriteAheadVFS while ${openHandleCount} open handles remain`); - } Object.values(runtime.files).forEach((file) => { const meta = getFileMeta(file); @@ -1143,7 +1137,8 @@ const resetMemoryWriteAheadRuntimeState = (runtime: MemoryWriteAheadSharedRuntim }; // With a dbFilename, resets only that runtime so concurrent clients (e.g. other databases) keep their -// runtimes; without one, resets every registered runtime (used by test teardown for a clean slate). +// runtimes, and refuses while that runtime has open handles. Without one, resets and deregisters every +// registered runtime even when handles were never released (used by test teardown for a clean slate). export const resetMemoryWriteAheadSharedRuntime = (dbFilename?: string) => { const registry = getRuntimeRegistry(); @@ -1154,6 +1149,12 @@ export const resetMemoryWriteAheadSharedRuntime = (dbFilename?: string) => { return; } + recoverAbandonedRuntimeOwnerLeases(runtime); + const openHandleCount = getOpenHandleCount(runtime); + if (openHandleCount > 0) { + throw new Error(`Cannot reset MemoryWriteAheadVFS while ${openHandleCount} open handles remain`); + } + resetMemoryWriteAheadRuntimeState(runtime); registry.delete(normalizedDbFilename); return; diff --git a/tests/browser/memoryWriteAheadVfs.spec.ts b/tests/browser/memoryWriteAheadVfs.spec.ts index 26ddfd1..b1d60ae 100644 --- a/tests/browser/memoryWriteAheadVfs.spec.ts +++ b/tests/browser/memoryWriteAheadVfs.spec.ts @@ -8,6 +8,7 @@ const browserFixtureNames = [ 'runAttachedDatabaseConcurrentWrites', 'runAttachedDatabaseWithoutWriteAheadFiles', 'runCapacityExceededFailure', + 'runCleanSlateResetWithOpenHandle', 'runDuplicatePendingOpenRejection', 'runDeferredWriteUpgrade', 'runFileLockRegression', @@ -27,6 +28,7 @@ const browserFixtureNames = [ 'runSnapshotIsolation', 'runStaleDeferredWriteRejection', 'runStaleOwnedLeaseSlotPreservation', + 'runTargetedResetRefusal', 'runWalRoundTrip', 'runWorkerConcurrency', 'runZeroByteWriteAndDiagnostics', @@ -85,6 +87,23 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.resetWhileOpenMessage).toBe('Cannot reset MemoryWriteAheadVFS while 3 open handles remain'); }); + test('resets and deregisters every runtime even when one still holds an open handle', async ({ page }) => { + const result = await runBrowserFixture(page, 'runCleanSlateResetWithOpenHandle'); + + expect(result.resetError).toBeNull(); + expect(result.leakingRuntimeRegistered).toBe(false); + expect(result.followingRuntimeRegistered).toBe(false); + expect(result.followingFileSizeAfterReset).toBe(0); + }); + + test('refuses a targeted reset with open handles without touching the runtime', async ({ page }) => { + const result = await runBrowserFixture(page, 'runTargetedResetRefusal'); + + expect(result.resetMessage).toBe('Cannot reset MemoryWriteAheadVFS while 1 open handles remain'); + expect(result.openHandleCountAfterRefusal).toBe(1); + expect(result.fileSizeAfterRefusal).toBe(4); + }); + test('reopens committed WAL data from the shared in-memory runtime', async ({ page }) => { const result = await runBrowserFixture(page, 'runWalRoundTrip'); diff --git a/tests/fixtures/browserApi.ts b/tests/fixtures/browserApi.ts index b711d04..2be1bb5 100644 --- a/tests/fixtures/browserApi.ts +++ b/tests/fixtures/browserApi.ts @@ -25,6 +25,7 @@ import { MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT, MEMORY_WRITE_AHEAD_RUNTIME_META, getMemoryWriteAheadFile, + getMemoryWriteAheadFileSizeBytes, getMemoryWriteAheadLockClockMs, getMemoryWriteAheadRuntimeOwnerSlotOffset, releaseMemoryWriteAheadRuntimeHandle, @@ -261,6 +262,70 @@ export const runRuntimeLifecycle = async () => { } }; +export const runCleanSlateResetWithOpenHandle = () => { + ensureCrossOriginIsolated(); + + const leakingDbFilename = nextDbFilename('reset-leaking'); + const followingDbFilename = nextDbFilename('reset-following'); + const leakingRuntime = createMemoryWriteAheadSharedRuntime(leakingDbFilename); + const followingRuntime = createMemoryWriteAheadSharedRuntime(followingDbFilename); + const followingFile = getMemoryWriteAheadFile(followingRuntime, followingDbFilename); + if (!followingFile) { + throw new Error(`Missing MemoryWriteAhead file for ${followingDbFilename}`); + } + + retainMemoryWriteAheadRuntimeHandle(leakingRuntime); + writeMemoryWriteAheadFile(followingFile, new Uint8Array([1, 2, 3, 4]), 0); + + try { + const resetError = (() => { + try { + MemoryWriteAheadVFS.resetRuntime(); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + + return { + followingFileSizeAfterReset: getMemoryWriteAheadFileSizeBytes(followingFile), + followingRuntimeRegistered: getMemoryWriteAheadRuntimeDiagnostics(followingDbFilename) !== null, + leakingRuntimeRegistered: getMemoryWriteAheadRuntimeDiagnostics(leakingDbFilename) !== null, + resetError, + }; + } finally { + releaseMemoryWriteAheadRuntimeHandle(leakingRuntime); + MemoryWriteAheadVFS.resetRuntime(); + } +}; + +export const runTargetedResetRefusal = async () => { + ensureCrossOriginIsolated(); + + const dbFilename = nextDbFilename('reset-targeted'); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename); + const file = getMemoryWriteAheadFile(runtime, dbFilename); + if (!file) { + throw new Error(`Missing MemoryWriteAhead file for ${dbFilename}`); + } + + retainMemoryWriteAheadRuntimeHandle(runtime); + writeMemoryWriteAheadFile(file, new Uint8Array([1, 2, 3, 4]), 0); + + try { + const resetMessage = await captureErrorMessage(() => resetRuntime(dbFilename)); + + return { + fileSizeAfterRefusal: getMemoryWriteAheadFileSizeBytes(file), + openHandleCountAfterRefusal: diagnosticsFor(dbFilename).openHandleCount, + resetMessage, + }; + } finally { + releaseMemoryWriteAheadRuntimeHandle(runtime); + resetRuntime(dbFilename); + } +}; + export const runWalRoundTrip = async () => { ensureCrossOriginIsolated(); From bf2545720be1891670dc34f36226d9ee2e94434f Mon Sep 17 00:00:00 2001 From: Yahya JIRARI Date: Wed, 7 Oct 2026 19:01:40 +0200 Subject: [PATCH 4/4] perf: grow shared file segments in bounded steps instead of to the exact byte Once a file outgrew its first segment, every write past the end grew the next segment's SharedArrayBuffer to exactly the bytes it needed. SQLite appends one page at a time, so a database or write-ahead file larger than its initial capacity resized a buffer and rebuilt its byte view on almost every write. A growth now doubles the segment, starting at 64 KiB and adding at most 1 MiB per step, or exactly what a larger write needs, and never past the segment's maximum. Reads still stop at the logical file size, so the spare room is invisible to SQLite, and cloned runtimes see the grown buffer as before. Fixed-capacity fallback buffers are unaffected. Measured in headless Chromium on the default options: appending 48 MiB in 4 KiB writes past a database's first segment went from 12,288 grow calls and 115-150 ms to 60 grow calls and about 7 ms; inserting 64 MiB of rows and running a TRUNCATE checkpoint went from 399-436 ms to 229-284 ms. Diagnostics now report allocated capacity up to 1 MiB above the logical size for a segment that is still growing. --- README.md | 2 +- src/memoryWriteAheadSharedRuntime.ts | 9 +- tests/browser/memoryWriteAheadVfs.spec.ts | 23 +++++ tests/fixtures/browserApi.ts | 107 ++++++++++++++++++++++ 4 files changed, 139 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c4bc006..5d6394d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ This library targets the gap between those options: an ephemeral SQLite database A plain `SharedArrayBuffer` is fixed-size at construction, which forces a bad choice for a database with a large ceiling: either commit the maximum capacity up front (hundreds of MiB for a mostly-small database), or re-allocate bigger buffers as data grows and re-broadcast the new handles to every worker mid-flight, copying bytes and re-coordinating every open connection. -`SharedArrayBuffer.prototype.grow` removes that dilemma. Buffers start at the initial capacity and grow in place up to `maxByteLength`, and every worker that already holds the runtime observes the new length immediately — no re-sharing, no copies. Files are split into segments so each buffer's reservation stays bounded, and write-ahead segments past the logical end drop out of the active set after `PRAGMA wal_checkpoint(TRUNCATE)` (tracked in diagnostics; the underlying allocation is not returned to the OS, since shared buffers cannot shrink). +`SharedArrayBuffer.prototype.grow` removes that dilemma. Buffers start at the initial capacity and grow in place up to `maxByteLength`, and every worker that already holds the runtime observes the new length immediately — no re-sharing, no copies. Each growth doubles the buffer, starting at 64 KiB and adding at most 1 MiB per step (or exactly what a larger write needs), so a run of page-sized appends does not resize a buffer on every write. Files are split into segments so each buffer's reservation stays bounded, and write-ahead segments past the logical end drop out of the active set after `PRAGMA wal_checkpoint(TRUNCATE)` (tracked in diagnostics; the underlying allocation is not returned to the OS, since shared buffers cannot shrink). On engines without growable `SharedArrayBuffer`, `allowFixedCapacityFallback: true` restores the fixed-size strategy at the cost of allocating each file's max capacity immediately. diff --git a/src/memoryWriteAheadSharedRuntime.ts b/src/memoryWriteAheadSharedRuntime.ts index d7c0df5..a87f5e5 100644 --- a/src/memoryWriteAheadSharedRuntime.ts +++ b/src/memoryWriteAheadSharedRuntime.ts @@ -11,6 +11,8 @@ const LOCK_CLOCK_MODULO_MS = 0x7fffffff; const MAX_INT32_ATOMIC_COUNTER = 0x7fffffff; export const MEMORY_WRITE_AHEAD_OWNER_LEASE_SLOT_COUNT = 32; const MIN_SEGMENT_COUNT = 1; +const MIN_AMORTIZED_SEGMENT_CAPACITY_BYTES = 64 * KIB; +const MAX_SEGMENT_GROWTH_STEP_BYTES = MIB; const memoryWriteAheadRuntimeLogger = noopMemoryWriteAheadLogger; @@ -785,7 +787,12 @@ const growSegment = (segment: MemoryWriteAheadSharedSegment, requiredBytes: numb } if (segment.capacityMode === 'growable') { - segment.data.grow(requiredBytes); + const currentCapacityBytes = segment.data.byteLength; + const amortizedCapacityBytes = Math.min( + Math.max(MIN_AMORTIZED_SEGMENT_CAPACITY_BYTES, currentCapacityBytes * 2), + currentCapacityBytes + MAX_SEGMENT_GROWTH_STEP_BYTES + ); + segment.data.grow(Math.min(segment.maxCapacityBytes, Math.max(requiredBytes, amortizedCapacityBytes))); return true; } diff --git a/tests/browser/memoryWriteAheadVfs.spec.ts b/tests/browser/memoryWriteAheadVfs.spec.ts index b1d60ae..bfe2124 100644 --- a/tests/browser/memoryWriteAheadVfs.spec.ts +++ b/tests/browser/memoryWriteAheadVfs.spec.ts @@ -7,6 +7,7 @@ const browserFixtureNames = [ 'runAttachedDatabaseAtomicTransaction', 'runAttachedDatabaseConcurrentWrites', 'runAttachedDatabaseWithoutWriteAheadFiles', + 'runBoundedSegmentGrowth', 'runCapacityExceededFailure', 'runCleanSlateResetWithOpenHandle', 'runDuplicatePendingOpenRejection', @@ -143,6 +144,28 @@ test.describe('MemoryWriteAheadVFS browser integration', () => { expect(result.reclaimedSegmentCount).toBe(0x7fffffff); }); + test('grows segments in bounded steps and keeps grown bytes visible to cloned runtimes', async ({ page }) => { + const { clonedReader, finalPartialSegment, largeSegment, smallAppends } = await runBrowserFixture( + page, + 'runBoundedSegmentGrowth' + ); + + expect(smallAppends.afterFirstAppend).toBe(64 * 1024); + expect(smallAppends.afterSecondAppend).toBe(64 * 1024); + expect(smallAppends.logicalSizeBytes).toBe(1024 * 1024 + 2 * 4096); + expect(smallAppends.bytesReadPastEnd).toBe(0); + expect(smallAppends.pastEndUntouched).toBe(true); + expect(largeSegment.secondSegmentBytes).toBe(4 * 1024 * 1024); + expect(finalPartialSegment.secondSegmentBytes).toBe(1000); + expect(finalPartialSegment.bytesRead).toBe(64 * 1024 + 1000); + expect(finalPartialSegment.readbackMatches).toBe(true); + expect(clonedReader.bytesRead).toBe(64 * 1024 + 4096); + expect(clonedReader.readbackMatches).toBe(true); + expect(clonedReader.tailUntouched).toBe(true); + expect(clonedReader.readerLogicalSizeBytes).toBe(64 * 1024 + 4096); + expect(clonedReader.readerSecondSegmentBytes).toBe(clonedReader.writerSecondSegmentBytes); + }); + test('extends logical size for zero-byte writes and records high-water diagnostics', async ({ page }) => { const result = await runBrowserFixture(page, 'runZeroByteWriteAndDiagnostics'); diff --git a/tests/fixtures/browserApi.ts b/tests/fixtures/browserApi.ts index 2be1bb5..f2ce366 100644 --- a/tests/fixtures/browserApi.ts +++ b/tests/fixtures/browserApi.ts @@ -12,6 +12,7 @@ import { recoverTerminatedMemoryWriteAheadRuntimeOwner, resolveMemoryWriteAheadRuntimeOptions, type MemoryWriteAheadRuntimeOptions, + type MemoryWriteAheadSharedFile, type MemoryWriteAheadSharedRuntime, } from '../../src/index.js'; import { @@ -28,6 +29,7 @@ import { getMemoryWriteAheadFileSizeBytes, getMemoryWriteAheadLockClockMs, getMemoryWriteAheadRuntimeOwnerSlotOffset, + readMemoryWriteAheadFile, releaseMemoryWriteAheadRuntimeHandle, retainMemoryWriteAheadRuntimeHandle, truncateMemoryWriteAheadFile, @@ -497,6 +499,111 @@ export const runSegmentReclaimAccountingSaturation = () => { } }; +export const runBoundedSegmentGrowth = () => { + ensureCrossOriginIsolated(); + + const kib = 1024; + const mib = 1024 * kib; + const withRuntimeFile = ( + label: string, + options: MemoryWriteAheadRuntimeOptions, + operation: (file: MemoryWriteAheadSharedFile, runtime: MemoryWriteAheadSharedRuntime) => TResult + ) => { + const dbFilename = nextDbFilename(label); + const runtime = createMemoryWriteAheadSharedRuntime(dbFilename, { + initialWriteAheadCapacityBytes: 64 * kib, + maxWriteAheadCapacityBytes: 128 * kib, + ...options, + }); + const file = getMemoryWriteAheadFile(runtime, dbFilename); + if (!file) { + throw new Error(`Missing MemoryWriteAhead file for ${dbFilename}`); + } + + try { + return operation(file, runtime); + } finally { + resetRuntime(dbFilename); + } + }; + const secondSegmentBytes = (file: MemoryWriteAheadSharedFile) => file.segments[1]?.data.byteLength ?? 0; + + const smallAppends = withRuntimeFile( + 'growth-small-appends', + { initialDatabaseCapacityBytes: mib, maxDatabaseCapacityBytes: 2 * mib }, + (file) => { + const pageBytes = new Uint8Array(4096).fill(23); + writeMemoryWriteAheadFile(file, pageBytes, mib); + const afterFirstAppend = secondSegmentBytes(file); + writeMemoryWriteAheadFile(file, pageBytes, mib + pageBytes.byteLength); + const pastEnd = new Uint8Array(32).fill(37); + + return { + afterFirstAppend, + afterSecondAppend: secondSegmentBytes(file), + bytesReadPastEnd: readMemoryWriteAheadFile(file, pastEnd, mib + 2 * pageBytes.byteLength), + logicalSizeBytes: getMemoryWriteAheadFileSizeBytes(file), + pastEndUntouched: pastEnd.every((byte) => byte === 37), + }; + } + ); + + const largeSegment = withRuntimeFile( + 'growth-large-segment', + { initialDatabaseCapacityBytes: 8 * mib, maxDatabaseCapacityBytes: 16 * mib }, + (file) => { + writeMemoryWriteAheadFile(file, new Uint8Array(3 * mib), 8 * mib); + writeMemoryWriteAheadFile(file, new Uint8Array(32), 11 * mib); + + return { secondSegmentBytes: secondSegmentBytes(file) }; + } + ); + + const finalPartialSegment = withRuntimeFile( + 'growth-final-partial-segment', + { initialDatabaseCapacityBytes: 64 * kib, maxDatabaseCapacityBytes: 64 * kib + 1000 }, + (file) => { + const payload = new Uint8Array(64 * kib + 1000).fill(23); + writeMemoryWriteAheadFile(file, payload, 0); + const readback = new Uint8Array(payload.byteLength); + const bytesRead = readMemoryWriteAheadFile(file, readback, 0); + + return { + bytesRead, + readbackMatches: readback.every((byte) => byte === 23), + secondSegmentBytes: secondSegmentBytes(file), + }; + } + ); + + const clonedReader = withRuntimeFile( + 'growth-cloned-reader', + { initialDatabaseCapacityBytes: 64 * kib, maxDatabaseCapacityBytes: 3 * 64 * kib }, + (file, runtime) => { + const readerFile = getMemoryWriteAheadFile(structuredClone(runtime), file.pathname); + if (!readerFile) { + throw new Error(`Missing cloned MemoryWriteAhead file for ${file.pathname}`); + } + + const payload = new Uint8Array(64 * kib + 4096).fill(23); + writeMemoryWriteAheadFile(file, payload, 0); + const readback = new Uint8Array(payload.byteLength + 1).fill(37); + const bytesRead = readMemoryWriteAheadFile(readerFile, readback, 0); + + return { + bytesRead, + readbackMatches: readback.subarray(0, payload.byteLength).every((byte) => byte === 23), + readerLogicalSizeBytes: getMemoryWriteAheadFileSizeBytes(readerFile), + readerSecondSegmentBytes: secondSegmentBytes(readerFile), + tailUntouched: readback[payload.byteLength] === 37, + writerSecondSegmentBytes: secondSegmentBytes(file), + }; + } + ); + + return { clonedReader, finalPartialSegment, largeSegment, smallAppends }; +}; + export const runZeroByteWriteAndDiagnostics = () => { ensureCrossOriginIsolated();