Add E2E retry support with SQLite snapshot/restore

client-v3/e2e/db-snapshot.ts

@@ -0,0 +1,83 @@
+import fs from 'fs';
+import path from 'path';
+import { spawn } from 'child_process';
+import { test } from '@playwright/test';
+import { PID_FILE, TMPDIR_FILE, SERVER_PORT, waitForServer } from './global-setup.js';
+
+function getPaths() {
+  const tempDir = fs.readFileSync(TMPDIR_FILE, 'utf-8').trim();
+  return {
+    db: path.join(tempDir, 'digiscript.sqlite'),
+    dbSnapshot: path.join(tempDir, 'digiscript.sqlite.snapshot'),
+    config: path.join(tempDir, 'digiscript.json'),
+    configSnapshot: path.join(tempDir, 'digiscript.json.snapshot'),
+    serverDir: path.resolve(process.cwd(), '..', 'server'),
+  };
+}
+
+export function snapshotExists(): boolean {
+  const { dbSnapshot } = getPaths();
+  return fs.existsSync(dbSnapshot);
+}
+
+/** Copy the DB and config to snapshot files. Call after each passing test. */
+export function snapshotState(): void {
+  const { db, dbSnapshot, config, configSnapshot } = getPaths();
+  fs.copyFileSync(db, dbSnapshot);
+  fs.copyFileSync(config, configSnapshot);
+}
+
+/**
+ * Restore DB and config from the last snapshot, then restart the server.
+ * Call at the start of a retry to bring the backend back to last known-good state.
+ */
+export async function restoreStateAndRestartServer(): Promise<void> {
+  const { db, dbSnapshot, config, configSnapshot, serverDir } = getPaths();
+
+  // Kill existing server
+  if (fs.existsSync(PID_FILE)) {
+    const pid = parseInt(fs.readFileSync(PID_FILE, 'utf-8').trim(), 10);
+    try {
+      process.kill(pid, 'SIGKILL');
+    } catch {
+      // process already gone
+    }
+    await new Promise((r) => setTimeout(r, 500));
+  }
+
+  // Remove any stale journal file (server uses DELETE journal mode, not WAL)
+  try {
+    fs.rmSync(`${db}-journal`);
+  } catch {
+    // no journal file present
+  }
+
+  // Restore DB and config from snapshot
+  fs.copyFileSync(dbSnapshot, db);
+  fs.copyFileSync(configSnapshot, config);
+
+  // Respawn server with the restored config
+  const server = spawn(
+    'python3',
+    ['main.py', `--port=${SERVER_PORT}`, `--settings_path=${config}`, '--debug=false'],
+    { cwd: serverDir, detached: true, stdio: 'ignore' }
+  );
+  fs.writeFileSync(PID_FILE, String(server.pid!));
+  server.unref();
+
+  await waitForServer();
+}
+
+/** Register beforeAll/afterEach hooks for retry support. Call once at the top of each spec file. */
+export function registerRetryHooks(): void {
+  test.beforeAll(async () => {
+    if (test.info().retry > 0 && snapshotExists()) {
+      await restoreStateAndRestartServer();
+    }
+  });
+  test.afterEach(async () => {
+    if (test.info().status === 'passed') {
+      snapshotState();
+    }
+  });
+}

client-v3/e2e/global-setup.ts

@@ -3,7 +3,7 @@ import fs from 'fs';
 import path from 'path';
 import os from 'os';
 
-const SERVER_PORT = 8888;
+export const SERVER_PORT = 8888;
 const HEALTH_URL = `http://localhost:${SERVER_PORT}/api/v1/health`;
 
 export const PID_FILE = path.join(os.tmpdir(), 'digiscript-e2e-server.pid');
@@ -85,7 +85,7 @@ async function killStaleServer(): Promise<void> {
   }
 }
 
-async function waitForServer(timeoutMs = 30_000): Promise<void> {
+export async function waitForServer(timeoutMs = 30_000): Promise<void> {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
     try {

client-v3/e2e/tests/01-first-run.spec.ts

@@ -5,9 +5,12 @@
  */
 import { test, expect, type BrowserContext, type Page } from '@playwright/test';
 import { UI_BASE, ADMIN_PASSWORD, waitForAppReady } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/02-auth.spec.ts

@@ -10,9 +10,12 @@ import {
   waitForAppReady,
   loginAsAdmin,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/03-system-config.spec.ts

@@ -11,9 +11,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/04-show-config-show.spec.ts

@@ -10,9 +10,12 @@ import {
   confirmModal,
   waitForModalClosed,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/05-show-config-acts-scenes.spec.ts

@@ -12,9 +12,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/06-show-config-characters.spec.ts

@@ -11,9 +11,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/07-show-config-stage.spec.ts

@@ -12,9 +12,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/08-show-config-cues.spec.ts

@@ -11,9 +11,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/09-show-config-mics.spec.ts

@@ -11,9 +11,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/10-show-config-script.spec.ts

@@ -12,9 +12,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/11-show-config-revisions.spec.ts

@@ -11,9 +11,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/12-show-config-sessions.spec.ts

@@ -12,9 +12,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/e2e/tests/13-live-show.spec.ts

@@ -16,9 +16,12 @@ import {
   waitForModalClosed,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let leaderCtx: BrowserContext;
 let followerCtx: BrowserContext;
 let leaderPage: Page;

client-v3/e2e/tests/14-user-settings.spec.ts

@@ -10,9 +10,12 @@ import {
   waitForAppReady,
   confirmDialog,
 } from '../helpers.js';
+import { registerRetryHooks } from '../db-snapshot.js';
 
 test.describe.configure({ mode: 'serial' });
 
+registerRetryHooks();
+
 let ctx: BrowserContext;
 let page: Page;
 

client-v3/playwright.config.ts

@@ -6,7 +6,8 @@ export default defineConfig({
   testDir: './e2e/tests',
   fullyParallel: false,
   workers: 1,
-  retries: 0,
+  retries: 3,
+  maxFailures: 1,
   reporter: [
     ['html', { outputFolder: 'playwright-report', open: 'never' }],
     ['junit', { outputFile: 'junit/playwright-results.xml' }],