diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index 18a5323cd590c..7efeb2aac652b 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "@effect/vitest"; +import { migrationManifest } from "../persistence/Migrations.ts"; import { runServicePreflight } from "./servicePreflight.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; @@ -20,3 +21,19 @@ it.each([1])("blocks legacy launcher protocol %i with exact recovery steps", (la expect(reason).toContain("t3 --version"); expect(reason).toContain("which -a t3"); }); + +// The launcher skips the update backup only when every migration listed here is already applied. +it("lists every migration as id and name pairs when ready", () => { + const result = runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }); + + const printed: unknown = JSON.parse(JSON.stringify(result)); + expect(printed).toMatchObject({ status: "ready", launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }); + expect(printed).toHaveProperty( + "migrations", + migrationManifest.map(([id, name]) => [id, name]), + ); + expect(migrationManifest.length).toBeGreaterThan(0); +}); diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts index 8865ce7b3d11f..22981eb463c71 100644 --- a/apps/server/src/cloud/servicePreflight.ts +++ b/apps/server/src/cloud/servicePreflight.ts @@ -1,4 +1,5 @@ import packageJson from "../../package.json" with { type: "json" }; +import { migrationManifest } from "../persistence/Migrations.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; export type ServicePreflightResult = @@ -6,6 +7,8 @@ export type ServicePreflightResult = readonly status: "ready"; readonly version: string; readonly launcherProtocol: typeof SERVICE_LAUNCHER_PROTOCOL; + /** The launcher compares these with the database to decide whether the update needs a backup. */ + readonly migrations?: typeof migrationManifest; } | { readonly status: "blocked"; @@ -34,7 +37,12 @@ export function runServicePreflight(input: { }; } - return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; + return { + status: "ready", + version, + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + migrations: migrationManifest, + }; } export function decodeServicePreflightResult(value: unknown): ServicePreflightResult | undefined { diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 3b8ea548d93da..096341da6e577 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -16,6 +16,8 @@ export interface PendingServiceUpdate { readonly targetVersion: string; readonly dbPath: string; readonly status: "pending"; + /** Set when the target applies no migration the database lacks, so the previous version can still open it. */ + readonly skipDatabaseBackup?: true; } export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome; @@ -84,7 +86,14 @@ function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { } if (status === "pending") { return typeof value.dbPath === "string" && value.dbPath.trim() !== "" - ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status } + ? { + id, + fromVersion, + targetVersion, + dbPath: value.dbPath, + status, + ...(value.skipDatabaseBackup === true ? { skipDatabaseBackup: true } : {}), + } : undefined; } if ( diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 70550f2379272..6e530553fb690 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,5 +1,6 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins. import type * as NodeChildProcess from "node:child_process"; +import * as NodeSqlite from "node:sqlite"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -461,6 +462,180 @@ if (context.update?.status === "pending") { }), ); + it.effect.each([ + { + target: "adds no migrations, even with a full disk", + migrations: [[1, "Init"]], + freeBytes: 0, + sawBackup: "false", + afterRollback: "after trial", + reason: "candidate-exited:1:no-backup", + }, + { + target: "renames a recorded migration", + migrations: [[1, "Renamed"]], + freeBytes: Number.MAX_SAFE_INTEGER, + sawBackup: "true", + afterRollback: "before trial", + reason: "candidate-exited:1", + }, + { + target: "adds a migration", + migrations: [ + [1, "Init"], + [2, "Next"], + ], + freeBytes: Number.MAX_SAFE_INTEGER, + sawBackup: "true", + afterRollback: "before trial", + reason: "candidate-exited:1", + }, + ])( + "backs up only when the target $target, and rolls back either way", + ({ migrations, freeBytes, sawBackup, afterRollback, reason }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-skip-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const observedPath = path.join(root, "trial-saw-backup"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + const database = new NodeSqlite.DatabaseSync(databasePath); + database.exec(` + CREATE TABLE effect_sql_migrations (migration_id INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO effect_sql_migrations VALUES (1, 'Init'); + CREATE TABLE notes (body TEXT NOT NULL); + INSERT INTO notes VALUES ('before trial'); + `); + database.close(); + const encoded = JSON.stringify({ root, databasePath, observedPath, migrations }); + const childSource = ` +import { existsSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +const paths = ${encoded}; +if (process.argv.includes("__service-preflight")) { + console.log(JSON.stringify({ status: "ready", version: "1.1.0", launcherProtocol: ${SERVICE_LAUNCHER_PROTOCOL}, migrations: paths.migrations })); +} else { + const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); + if (context.update?.status === "pending") { + writeFileSync(paths.observedPath, String(existsSync(join(paths.root, "runtime", "db-backup", context.update.id)))); + const database = new DatabaseSync(context.update.dbPath); + database.exec("UPDATE notes SET body = 'after trial'"); + database.close(); + process.exit(1); + } else if (context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-rejected") process.exit(2); + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }); + setInterval(() => {}, 1_000); + } else { + process.exit(0); + } +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + backupSpaceProbe: { + readFreeBytes: () => Promise.resolve(freeBytes), + canClone: () => Promise.resolve(false), + }, + }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + reason, + ); + assert.equal(yield* fs.readFileString(observedPath), sawBackup); + const after = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); + try { + assert.deepEqual( + after + .prepare("SELECT body FROM notes") + .all() + .map((row) => row.body), + [afterRollback], + ); + } finally { + after.close(); + } + }), + ); + + it.effect("does not flag a skipped backup when no trial ran", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-no-trial-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + "process.exit(0);", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: path.join(root, "userdata", "state.sqlite"), + status: "pending", + skipDatabaseBackup: true, + }, + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "failed"); + assert.equal( + state.update?.status === "failed" ? state.update.reason : undefined, + "target-runtime-missing", + ); + }), + ); + const requestRejectedUpdate = (backupSpaceProbe: Partial) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9f764b788efe2..44f1249da147e 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -6,6 +6,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; import * as NodeUtil from "node:util"; import type { @@ -30,6 +31,8 @@ import { } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; +// The requesting server gives up on the launcher's reply after 30 seconds. +const MIGRATION_CHECK_TIMEOUT_MS = 10_000; /** * Migrations count toward it and can scale with the database: tens of GB on a cold page cache * take minutes. A missed window rolls back the database, so the next attempt pays the same cost. @@ -242,6 +245,66 @@ export async function databaseBackupBlocker( return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`; } +const isMigrationEntry = (entry: unknown): entry is readonly [number, string] => + Array.isArray(entry) && + entry.length === 2 && + typeof entry[0] === "number" && + typeof entry[1] === "string"; + +// Older targets omit their migrations from preflight; any failure here means "back up". +async function targetAddsMigrations( + baseDir: string, + targetVersion: string, + dbPath: string, + environment: Readonly, +): Promise { + try { + const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)( + runtimePaths(baseDir, targetVersion).entryPath, + [ + "__service-preflight", + "--database-path", + dbPath, + "--launcher-protocol", + String(SERVICE_LAUNCHER_PROTOCOL), + ], + { env: environment, timeout: MIGRATION_CHECK_TIMEOUT_MS, killSignal: "SIGKILL" }, + ); + const preflight: unknown = JSON.parse(stdout.trim()); + const migrations = + typeof preflight === "object" && preflight !== null && "migrations" in preflight + ? preflight.migrations + : undefined; + if ( + !Array.isArray(migrations) || + migrations.length === 0 || + !migrations.every(isMigrationEntry) + ) { + return true; + } + + const database = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + try { + const applied = new Set( + database + .prepare("SELECT migration_id, name FROM effect_sql_migrations") + .all() + .map((row) => `${String(row.migration_id)}_${String(row.name)}`), + ); + return migrations.some(([id, name]) => !applied.has(`${id}_${name}`)); + } finally { + database.close(); + } + } catch { + return true; + } +} + +// A trial that ran without a backup leaves its writes behind, which the previous version may not decode. +function trialRollbackReason(pending: PendingServiceUpdate, reason: string): string { + return pending.skipDatabaseBackup === true ? `${reason}:no-backup` : reason; +} + // Preserve completed backups; restarted trials may already have changed the live database. async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); @@ -553,7 +616,8 @@ export class Launcher { async #startTrial(pending: PendingServiceUpdate): Promise { try { - await backupDatabaseOnce(this.#baseDir, pending); + // Without a backup, rollback keeps the trial's writes, which the previous version may not decode. + if (pending.skipDatabaseBackup !== true) await backupDatabaseOnce(this.#baseDir, pending); } catch { await this.#returnToPrevious(pending, "failed", "db-backup-failed"); return; @@ -661,15 +725,19 @@ export class Launcher { await reject("The requested target runtime is missing or incomplete."); return; } - // Failed checks must refuse updates rather than stop the server with unknown backup capacity. - const backupBlocker = await databaseBackupBlocker( + const backupDatabase = await targetAddsMigrations( this.#baseDir, + message.targetVersion, message.dbPath, - this.#backupSpaceProbe, - ).catch( - (error: unknown) => - `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`, + this.#environment, ); + // Failed checks must refuse updates rather than stop the server with unknown backup capacity. + const backupBlocker = backupDatabase + ? await databaseBackupBlocker(this.#baseDir, message.dbPath, this.#backupSpaceProbe).catch( + (error: unknown) => + `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`, + ) + : undefined; if (!child.process.connected) return; if (backupBlocker !== undefined) { await reject(backupBlocker); @@ -682,6 +750,7 @@ export class Launcher { targetVersion: message.targetVersion, dbPath: message.dbPath, status: "pending", + ...(backupDatabase ? {} : { skipDatabaseBackup: true }), }; const next: ServiceState = { ...this.#state, update: pending }; await writeServiceState(this.#statePath, next); @@ -710,7 +779,12 @@ export class Launcher { pending.targetVersion !== child.version ) { if (child.role === "trial" && pending?.status === "pending") { - await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child); + await this.#returnToPrevious( + pending, + "rolled-back", + trialRollbackReason(pending, "invalid-prepared"), + child, + ); return; } throw new Error("Trial child reported prepared for an unexpected update."); @@ -735,7 +809,12 @@ export class Launcher { return; } this.#timer = undefined; - await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child); + await this.#returnToPrevious( + pending, + "rolled-back", + trialRollbackReason(pending, "prepared-timeout"), + child, + ); } async #handleExit( @@ -758,7 +837,7 @@ export class Launcher { await this.#returnToPrevious( pending, "rolled-back", - `candidate-exited:${String(code ?? signal ?? "unknown")}`, + trialRollbackReason(pending, `candidate-exited:${String(code ?? signal ?? "unknown")}`), ); return; } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1536884578d4b..5717fd69b0027 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -32,23 +32,36 @@ is authoritative and the service manager's ordinary restart policy applies. The prepared deadline is long because migrations can scale with the database and block the trial's main thread, so it cannot report progress. A missed -deadline restores the snapshot, and the retry pays the same cost again. A +deadline restores the snapshot, if the update took one, and the retry pays the same cost again. A crashed trial is caught when it exits; the deadline bounds only a stuck one. Client update waits must outlast the deadline, or a slow update shows as failed. ## Database rollback -Before stopping the old child, the launcher refuses an update whose snapshot will -not fit on disk, so a full disk does not cause an outage. After the old child -exits, the launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes trial migrations reversible without down -migrations. The snapshot is made once per update and survives launcher restarts; -replacing it during a retry could capture changes from the failed trial. +Before stopping the old child, the launcher decides whether the update needs a +database snapshot. It runs the target's `__service-preflight`, which prints the +target's migrations, and compares them with `effect_sql_migrations`. If the +target ships no migration the database lacks, the update takes no snapshot. If +the check fails, times out, or meets an older target that doesn't report +migrations, the update takes one. A snapshot on a filesystem without reflinks is +a full copy made while no server runs, so skipping it keeps updates short. + +An update that takes a snapshot is refused first if the snapshot will not fit on +disk, so a full disk does not cause an outage. After the old child exits, the +launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes +trial migrations reversible without down migrations. The snapshot is made once +per update and survives launcher restarts; replacing it during a retry could +capture changes from the failed trial. Rollback stops the trial before restoring. A durable restore marker makes an interrupted restore finish before either version boots. Keep the snapshot until commit, or until both restoration and the terminal rollback state are durable. Attachments and other files outside SQLite are outside this rollback boundary. +A rollback without a snapshot keeps the trial's writes. The previous version can +open the schema, but it may not decode rows written in a new format. When a +trial ran without a snapshot, its rollback reason ends in `:no-backup`. + ## Client acknowledgement An accepted update is still pending. Clients correlate the launcher's update ID