From 08af1d3f130e776a48e2f409081fde5f2b075bcf Mon Sep 17 00:00:00 2001 From: Tanner Sharon Date: Wed, 28 Oct 2026 23:36:06 -0500 Subject: [PATCH 1/5] fix(server): skip the update backup when the target adds no migrations Before every service update, the launcher copied the whole database while no server ran. On ext4 there are no reflinks, so a 43 GB statev2.sqlite took about 8 minutes of downtime, even for nightlies with no migrations. The target's __service-preflight now prints its migration manifest. The launcher compares it with effect_sql_migrations before stopping the server. When every target migration is already applied, it skips the disk-space check and the backup, and records skipDatabaseBackup on the pending update. Any failure in that check takes the backup as before. Rollback without a backup already restores nothing, since restore and its marker key off the backup directory. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/servicePreflight.ts | 10 +- apps/server/src/cloud/serviceProtocol.ts | 11 +- apps/server/src/serviceLauncher.test.ts | 120 ++++++++++++++++++++++ apps/server/src/serviceLauncher.ts | 78 ++++++++++++-- 4 files changed, 210 insertions(+), 9 deletions(-) diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts index 8865ce7b3d11f..22981eb463c71 100644 --- a/apps/server/src/cloud/servicePreflight.ts +++ b/apps/server/src/cloud/servicePreflight.ts @@ -1,4 +1,5 @@ import packageJson from "../../package.json" with { type: "json" }; +import { migrationManifest } from "../persistence/Migrations.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; export type ServicePreflightResult = @@ -6,6 +7,8 @@ export type ServicePreflightResult = readonly status: "ready"; readonly version: string; readonly launcherProtocol: typeof SERVICE_LAUNCHER_PROTOCOL; + /** The launcher compares these with the database to decide whether the update needs a backup. */ + readonly migrations?: typeof migrationManifest; } | { readonly status: "blocked"; @@ -34,7 +37,12 @@ export function runServicePreflight(input: { }; } - return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; + return { + status: "ready", + version, + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + migrations: migrationManifest, + }; } export function decodeServicePreflightResult(value: unknown): ServicePreflightResult | undefined { diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 3b8ea548d93da..096341da6e577 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -16,6 +16,8 @@ export interface PendingServiceUpdate { readonly targetVersion: string; readonly dbPath: string; readonly status: "pending"; + /** Set when the target applies no migration the database lacks, so the previous version can still open it. */ + readonly skipDatabaseBackup?: true; } export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome; @@ -84,7 +86,14 @@ function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { } if (status === "pending") { return typeof value.dbPath === "string" && value.dbPath.trim() !== "" - ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status } + ? { + id, + fromVersion, + targetVersion, + dbPath: value.dbPath, + status, + ...(value.skipDatabaseBackup === true ? { skipDatabaseBackup: true } : {}), + } : undefined; } if ( diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 70550f2379272..ffd2f90240c91 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,5 +1,6 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins. import type * as NodeChildProcess from "node:child_process"; +import * as NodeSqlite from "node:sqlite"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -461,6 +462,125 @@ if (context.update?.status === "pending") { }), ); + it.effect.each([ + { + target: "adds no migrations, even with a full disk", + migrations: [[1, "Init"]], + freeBytes: 0, + sawBackup: "false", + afterRollback: "after trial", + }, + { + target: "adds a migration", + migrations: [ + [1, "Init"], + [2, "Next"], + ], + freeBytes: Number.MAX_SAFE_INTEGER, + sawBackup: "true", + afterRollback: "before trial", + }, + ])( + "backs up only when the target $target, and rolls back either way", + ({ migrations, freeBytes, sawBackup, afterRollback }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-skip-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const observedPath = path.join(root, "trial-saw-backup"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + const database = new NodeSqlite.DatabaseSync(databasePath); + database.exec(` + CREATE TABLE effect_sql_migrations (migration_id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO effect_sql_migrations VALUES (1, 'Init'); + CREATE TABLE notes (body TEXT NOT NULL); + INSERT INTO notes VALUES ('before trial'); + `); + database.close(); + const encoded = JSON.stringify({ root, databasePath, observedPath, migrations }); + const childSource = ` +import { existsSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +const paths = ${encoded}; +if (process.argv.includes("__service-preflight")) { + console.log(JSON.stringify({ status: "ready", version: "1.1.0", launcherProtocol: ${SERVICE_LAUNCHER_PROTOCOL}, migrations: paths.migrations })); +} else { + const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); + if (context.update?.status === "pending") { + writeFileSync(paths.observedPath, String(existsSync(join(paths.root, "runtime", "db-backup", context.update.id)))); + const database = new DatabaseSync(context.update.dbPath); + database.exec("UPDATE notes SET body = 'after trial'"); + database.close(); + process.exit(1); + } else if (context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-rejected") process.exit(2); + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }); + setInterval(() => {}, 1_000); + } else { + process.exit(0); + } +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + backupSpaceProbe: { + readFreeBytes: () => Promise.resolve(freeBytes), + canClone: () => Promise.resolve(false), + }, + }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + "candidate-exited:1", + ); + assert.equal(yield* fs.readFileString(observedPath), sawBackup); + const after = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); + try { + assert.deepEqual( + after + .prepare("SELECT body FROM notes") + .all() + .map((row) => row.body), + [afterRollback], + ); + } finally { + after.close(); + } + }), + ); + const requestRejectedUpdate = (backupSpaceProbe: Partial) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9f764b788efe2..e3e6570884a3d 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -6,6 +6,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; import * as NodeUtil from "node:util"; import type { @@ -30,6 +31,8 @@ import { } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; +// The requesting server gives up on the launcher's reply after 30 seconds. +const MIGRATION_CHECK_TIMEOUT_MS = 10_000; /** * Migrations count toward it and can scale with the database: tens of GB on a cold page cache * take minutes. A missed window rolls back the database, so the next attempt pays the same cost. @@ -242,6 +245,61 @@ export async function databaseBackupBlocker( return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`; } +const isMigrationEntry = (entry: unknown): entry is readonly [number, string] => + Array.isArray(entry) && + entry.length === 2 && + typeof entry[0] === "number" && + typeof entry[1] === "string"; + +// Older targets omit their migrations from preflight; any failure here means "back up". +async function targetAddsMigrations( + baseDir: string, + targetVersion: string, + dbPath: string, + environment: Readonly, +): Promise { + try { + const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)( + runtimePaths(baseDir, targetVersion).entryPath, + [ + "__service-preflight", + "--database-path", + dbPath, + "--launcher-protocol", + String(SERVICE_LAUNCHER_PROTOCOL), + ], + { env: environment, timeout: MIGRATION_CHECK_TIMEOUT_MS }, + ); + const preflight: unknown = JSON.parse(stdout.trim()); + const migrations = + typeof preflight === "object" && preflight !== null && "migrations" in preflight + ? preflight.migrations + : undefined; + if ( + !Array.isArray(migrations) || + migrations.length === 0 || + !migrations.every(isMigrationEntry) + ) { + return true; + } + + const database = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + try { + const applied = new Set( + database + .prepare("SELECT migration_id, name FROM effect_sql_migrations") + .all() + .map((row) => `${String(row.migration_id)}_${String(row.name)}`), + ); + return migrations.some(([id, name]) => !applied.has(`${id}_${name}`)); + } finally { + database.close(); + } + } catch { + return true; + } +} + // Preserve completed backups; restarted trials may already have changed the live database. async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); @@ -553,7 +611,8 @@ export class Launcher { async #startTrial(pending: PendingServiceUpdate): Promise { try { - await backupDatabaseOnce(this.#baseDir, pending); + // Without a backup, rollback restores nothing: restore and its marker key off the backup directory. + if (pending.skipDatabaseBackup !== true) await backupDatabaseOnce(this.#baseDir, pending); } catch { await this.#returnToPrevious(pending, "failed", "db-backup-failed"); return; @@ -661,15 +720,19 @@ export class Launcher { await reject("The requested target runtime is missing or incomplete."); return; } - // Failed checks must refuse updates rather than stop the server with unknown backup capacity. - const backupBlocker = await databaseBackupBlocker( + const backupDatabase = await targetAddsMigrations( this.#baseDir, + message.targetVersion, message.dbPath, - this.#backupSpaceProbe, - ).catch( - (error: unknown) => - `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`, + this.#environment, ); + // Failed checks must refuse updates rather than stop the server with unknown backup capacity. + const backupBlocker = backupDatabase + ? await databaseBackupBlocker(this.#baseDir, message.dbPath, this.#backupSpaceProbe).catch( + (error: unknown) => + `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`, + ) + : undefined; if (!child.process.connected) return; if (backupBlocker !== undefined) { await reject(backupBlocker); @@ -682,6 +745,7 @@ export class Launcher { targetVersion: message.targetVersion, dbPath: message.dbPath, status: "pending", + ...(backupDatabase ? {} : { skipDatabaseBackup: true }), }; const next: ServiceState = { ...this.#state, update: pending }; await writeServiceState(this.#statePath, next); From c1f7bb7d5aab4abc8b150225632660f1dfba0ff7 Mon Sep 17 00:00:00 2001 From: Tanner Sharon Date: Wed, 28 Oct 2026 23:50:02 -0500 Subject: [PATCH 2/5] fix(server): flag rollbacks that had no backup, and hard-kill a stuck preflight Review round 1 on #134. A rollback without a backup now ends its reason in :no-backup, since the previous version may not decode rows the trial wrote. The preflight gets SIGKILL on timeout. A new test case covers a migration recorded under another name at the same id, and the update docs describe when the snapshot is skipped. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/serviceLauncher.test.ts | 14 ++++++++++++-- apps/server/src/serviceLauncher.ts | 10 +++++++--- docs/internals/server-updates.md | 12 +++++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index ffd2f90240c91..d7b708f958f58 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -469,6 +469,15 @@ if (context.update?.status === "pending") { freeBytes: 0, sawBackup: "false", afterRollback: "after trial", + reason: "candidate-exited:1:no-backup", + }, + { + target: "renames a recorded migration", + migrations: [[1, "Renamed"]], + freeBytes: Number.MAX_SAFE_INTEGER, + sawBackup: "true", + afterRollback: "before trial", + reason: "candidate-exited:1", }, { target: "adds a migration", @@ -479,10 +488,11 @@ if (context.update?.status === "pending") { freeBytes: Number.MAX_SAFE_INTEGER, sawBackup: "true", afterRollback: "before trial", + reason: "candidate-exited:1", }, ])( "backs up only when the target $target, and rolls back either way", - ({ migrations, freeBytes, sawBackup, afterRollback }) => + ({ migrations, freeBytes, sawBackup, afterRollback, reason }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -563,7 +573,7 @@ if (process.argv.includes("__service-preflight")) { assert.equal(state.update?.status, "rolled-back"); assert.equal( state.update?.status === "rolled-back" ? state.update.reason : undefined, - "candidate-exited:1", + reason, ); assert.equal(yield* fs.readFileString(observedPath), sawBackup); const after = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index e3e6570884a3d..02e5171a9a812 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -268,7 +268,7 @@ async function targetAddsMigrations( "--launcher-protocol", String(SERVICE_LAUNCHER_PROTOCOL), ], - { env: environment, timeout: MIGRATION_CHECK_TIMEOUT_MS }, + { env: environment, timeout: MIGRATION_CHECK_TIMEOUT_MS, killSignal: "SIGKILL" }, ); const preflight: unknown = JSON.parse(stdout.trim()); const migrations = @@ -611,7 +611,7 @@ export class Launcher { async #startTrial(pending: PendingServiceUpdate): Promise { try { - // Without a backup, rollback restores nothing: restore and its marker key off the backup directory. + // Without a backup, rollback keeps the trial's writes, which the previous version may not decode. if (pending.skipDatabaseBackup !== true) await backupDatabaseOnce(this.#baseDir, pending); } catch { await this.#returnToPrevious(pending, "failed", "db-backup-failed"); @@ -868,7 +868,11 @@ export class Launcher { await terminateChild(child.process); } await restoreDatabaseBackup(this.#baseDir, pending); - const outcome = terminalUpdate({ pending, status, reason }); + const outcome = terminalUpdate({ + pending, + status, + reason: pending.skipDatabaseBackup === true ? `${reason}:no-backup` : reason, + }); const next: ServiceState = { ...this.#state, activeVersion: pending.fromVersion, diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1536884578d4b..29775ff8fb451 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -32,7 +32,7 @@ is authoritative and the service manager's ordinary restart policy applies. The prepared deadline is long because migrations can scale with the database and block the trial's main thread, so it cannot report progress. A missed -deadline restores the snapshot, and the retry pays the same cost again. A +deadline restores the snapshot, if one was taken, and the retry pays the same cost again. A crashed trial is caught when it exits; the deadline bounds only a stuck one. Client update waits must outlast the deadline, or a slow update shows as failed. @@ -49,6 +49,16 @@ interrupted restore finish before either version boots. Keep the snapshot until commit, or until both restoration and the terminal rollback state are durable. Attachments and other files outside SQLite are outside this rollback boundary. +The launcher skips the snapshot, and the disk check, when the target ships no +migration that `effect_sql_migrations` lacks. It learns the target's migrations +from its `__service-preflight` output before stopping the old child. A database +snapshot on a filesystem without reflinks is a full copy made while no server +runs, which took 8 minutes for a 43 GB database. If the check fails, times out, +or meets an older target that doesn't report migrations, the launcher takes the +snapshot. A rollback without a snapshot keeps the trial's writes. The previous +version can open the schema, but it may not decode rows written in a new format, +so the rollback reason ends in `:no-backup` to flag that case. + ## Client acknowledgement An accepted update is still pending. Clients correlate the launcher's update ID From 1d975463cfe8a016c3b5ff263a09de0ced16868d Mon Sep 17 00:00:00 2001 From: Tanner Sharon Date: Wed, 28 Oct 2026 23:58:20 -0500 Subject: [PATCH 3/5] fix(server): flag no-backup rollbacks only when a trial ran Review round 2 on #134. Failures before the target starts, such as a missing runtime, no longer get the :no-backup suffix. The update docs now describe the backup decision in place instead of appending an exception. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/serviceLauncher.test.ts | 45 +++++++++++++++++++++++++ apps/server/src/serviceLauncher.ts | 20 +++++++---- docs/internals/server-updates.md | 33 +++++++++--------- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index d7b708f958f58..6e530553fb690 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -591,6 +591,51 @@ if (process.argv.includes("__service-preflight")) { }), ); + it.effect("does not flag a skipped backup when no trial ran", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-no-trial-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + "process.exit(0);", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: path.join(root, "userdata", "state.sqlite"), + status: "pending", + skipDatabaseBackup: true, + }, + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "failed"); + assert.equal( + state.update?.status === "failed" ? state.update.reason : undefined, + "target-runtime-missing", + ); + }), + ); + const requestRejectedUpdate = (backupSpaceProbe: Partial) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 02e5171a9a812..21b80b078a35d 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -300,6 +300,11 @@ async function targetAddsMigrations( } } +// A trial that ran without a backup leaves its writes behind, which the previous version may not decode. +function trialRollbackReason(pending: PendingServiceUpdate, reason: string): string { + return pending.skipDatabaseBackup === true ? `${reason}:no-backup` : reason; +} + // Preserve completed backups; restarted trials may already have changed the live database. async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); @@ -799,7 +804,12 @@ export class Launcher { return; } this.#timer = undefined; - await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child); + await this.#returnToPrevious( + pending, + "rolled-back", + trialRollbackReason(pending, "prepared-timeout"), + child, + ); } async #handleExit( @@ -822,7 +832,7 @@ export class Launcher { await this.#returnToPrevious( pending, "rolled-back", - `candidate-exited:${String(code ?? signal ?? "unknown")}`, + trialRollbackReason(pending, `candidate-exited:${String(code ?? signal ?? "unknown")}`), ); return; } @@ -868,11 +878,7 @@ export class Launcher { await terminateChild(child.process); } await restoreDatabaseBackup(this.#baseDir, pending); - const outcome = terminalUpdate({ - pending, - status, - reason: pending.skipDatabaseBackup === true ? `${reason}:no-backup` : reason, - }); + const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { ...this.#state, activeVersion: pending.fromVersion, diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 29775ff8fb451..5717fd69b0027 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -32,32 +32,35 @@ is authoritative and the service manager's ordinary restart policy applies. The prepared deadline is long because migrations can scale with the database and block the trial's main thread, so it cannot report progress. A missed -deadline restores the snapshot, if one was taken, and the retry pays the same cost again. A +deadline restores the snapshot, if the update took one, and the retry pays the same cost again. A crashed trial is caught when it exits; the deadline bounds only a stuck one. Client update waits must outlast the deadline, or a slow update shows as failed. ## Database rollback -Before stopping the old child, the launcher refuses an update whose snapshot will -not fit on disk, so a full disk does not cause an outage. After the old child -exits, the launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes trial migrations reversible without down -migrations. The snapshot is made once per update and survives launcher restarts; -replacing it during a retry could capture changes from the failed trial. +Before stopping the old child, the launcher decides whether the update needs a +database snapshot. It runs the target's `__service-preflight`, which prints the +target's migrations, and compares them with `effect_sql_migrations`. If the +target ships no migration the database lacks, the update takes no snapshot. If +the check fails, times out, or meets an older target that doesn't report +migrations, the update takes one. A snapshot on a filesystem without reflinks is +a full copy made while no server runs, so skipping it keeps updates short. + +An update that takes a snapshot is refused first if the snapshot will not fit on +disk, so a full disk does not cause an outage. After the old child exits, the +launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes +trial migrations reversible without down migrations. The snapshot is made once +per update and survives launcher restarts; replacing it during a retry could +capture changes from the failed trial. Rollback stops the trial before restoring. A durable restore marker makes an interrupted restore finish before either version boots. Keep the snapshot until commit, or until both restoration and the terminal rollback state are durable. Attachments and other files outside SQLite are outside this rollback boundary. -The launcher skips the snapshot, and the disk check, when the target ships no -migration that `effect_sql_migrations` lacks. It learns the target's migrations -from its `__service-preflight` output before stopping the old child. A database -snapshot on a filesystem without reflinks is a full copy made while no server -runs, which took 8 minutes for a 43 GB database. If the check fails, times out, -or meets an older target that doesn't report migrations, the launcher takes the -snapshot. A rollback without a snapshot keeps the trial's writes. The previous -version can open the schema, but it may not decode rows written in a new format, -so the rollback reason ends in `:no-backup` to flag that case. +A rollback without a snapshot keeps the trial's writes. The previous version can +open the schema, but it may not decode rows written in a new format. When a +trial ran without a snapshot, its rollback reason ends in `:no-backup`. ## Client acknowledgement From 2e91e14843fa3294a68c72d7032b53844e062f87 Mon Sep 17 00:00:00 2001 From: Tanner Sharon Date: Thu, 29 Oct 2026 00:03:24 -0500 Subject: [PATCH 4/5] fix(server): flag invalid-prepared rollbacks that had no backup Review round 3 on #134. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/serviceLauncher.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 21b80b078a35d..44f1249da147e 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -779,7 +779,12 @@ export class Launcher { pending.targetVersion !== child.version ) { if (child.role === "trial" && pending?.status === "pending") { - await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child); + await this.#returnToPrevious( + pending, + "rolled-back", + trialRollbackReason(pending, "invalid-prepared"), + child, + ); return; } throw new Error("Trial child reported prepared for an unexpected update."); From 491ac2460bb8d361361fd3fd9635753b3796087e Mon Sep 17 00:00:00 2001 From: Tanner Sharon Date: Thu, 29 Oct 2026 06:15:12 -0500 Subject: [PATCH 5/5] test(server): check the ready preflight lists migrations The launcher tests feed in a hand-written migration list, so they would still pass if the preflight stopped printing its migrations. Updates would then take the full backup again without any test failing. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/servicePreflight.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index 18a5323cd590c..7efeb2aac652b 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "@effect/vitest"; +import { migrationManifest } from "../persistence/Migrations.ts"; import { runServicePreflight } from "./servicePreflight.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; @@ -20,3 +21,19 @@ it.each([1])("blocks legacy launcher protocol %i with exact recovery steps", (la expect(reason).toContain("t3 --version"); expect(reason).toContain("which -a t3"); }); + +// The launcher skips the update backup only when every migration listed here is already applied. +it("lists every migration as id and name pairs when ready", () => { + const result = runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }); + + const printed: unknown = JSON.parse(JSON.stringify(result)); + expect(printed).toMatchObject({ status: "ready", launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }); + expect(printed).toHaveProperty( + "migrations", + migrationManifest.map(([id, name]) => [id, name]), + ); + expect(migrationManifest.length).toBeGreaterThan(0); +});