diff --git a/apps/api/src/routes/admin/project-data-archive-breakers.ts b/apps/api/src/routes/admin/project-data-archive-breakers.ts new file mode 100644 index 0000000000..dbeeffc056 --- /dev/null +++ b/apps/api/src/routes/admin/project-data-archive-breakers.ts @@ -0,0 +1,86 @@ +import { Hono } from 'hono'; + +import type { Env } from '../../env'; +import { errors } from '../../middleware/error'; +import { getProjectDataArchiveRolloutListConfig } from '../../services/project-data-archive-rollout-controls'; + +/** + * Superadmin list of per-project archive-sharding circuit breakers. + * + * Mounted at `/api/admin/project-data/storage/archive-sharding/circuit-breakers`. + * D1-only and bounded: this backs the Admin → Storage tab, which is the + * phone-usable path for closing a tripped breaker (the POST lives in + * `project-data-storage.ts` as `/:projectId/archive-sharding/circuit-breaker`). + */ +export const adminProjectDataArchiveBreakerRoutes = new Hono<{ Bindings: Env }>(); + +const BREAKER_STATES = new Set(['closed', 'open', 'frozen']); + +type BreakerRow = { + project_id: unknown; + project_name: unknown; + repository: unknown; + state: unknown; + reason: unknown; + opened_at: unknown; + updated_at: unknown; +}; + +export function parseArchiveRolloutLimit(rawLimit: string | undefined, env: Env): number { + const { defaultLimit, maxLimit } = getProjectDataArchiveRolloutListConfig(env); + const parsedLimit = rawLimit ? Number.parseInt(rawLimit, 10) : defaultLimit; + if (!Number.isSafeInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > maxLimit) { + throw errors.badRequest(`limit must be between 1 and ${maxLimit}`); + } + return parsedLimit; +} + +function optionalString(value: unknown): string | null { + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function optionalNumber(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +adminProjectDataArchiveBreakerRoutes.get('/', async (c) => { + const limit = parseArchiveRolloutLimit(c.req.query('limit'), c.env); + const result = await c.env.DATABASE.prepare( + `SELECT + b.project_id, + p.name AS project_name, + p.repository AS repository, + b.state, + b.reason, + b.opened_at, + b.updated_at + FROM project_data_archive_circuit_breakers b + LEFT JOIN projects p ON p.id = b.project_id + ORDER BY CASE WHEN b.state = 'closed' THEN 1 ELSE 0 END ASC, b.updated_at DESC + LIMIT ?` + ) + .bind(limit) + .all(); + + const breakers = []; + let skippedRows = 0; + for (const row of result.results ?? []) { + const projectId = optionalString(row.project_id); + const updatedAt = optionalNumber(row.updated_at); + if (!projectId || updatedAt === null || !BREAKER_STATES.has(String(row.state))) { + skippedRows += 1; + continue; + } + breakers.push({ + projectId, + projectName: optionalString(row.project_name), + repository: optionalString(row.repository), + state: row.state as 'closed' | 'open' | 'frozen', + reason: optionalString(row.reason), + openedAt: optionalNumber(row.opened_at), + updatedAt, + }); + } + + return c.json({ breakers, skippedRows, limit }); +}); diff --git a/apps/api/src/routes/admin/project-data-storage.ts b/apps/api/src/routes/admin/project-data-storage.ts index 419ffe9a3b..0fd39e19c0 100644 --- a/apps/api/src/routes/admin/project-data-storage.ts +++ b/apps/api/src/routes/admin/project-data-storage.ts @@ -33,11 +33,14 @@ import { import { freezeProjectDataArchiveProject, getProjectDataArchiveManualCanaryConfig, - getProjectDataArchiveRolloutListConfig, getProjectDataArchiveRolloutState, listProjectDataArchiveProblemMigrations, setProjectDataArchiveCircuitBreaker, } from '../../services/project-data-archive-rollout-controls'; +import { + adminProjectDataArchiveBreakerRoutes, + parseArchiveRolloutLimit, +} from './project-data-archive-breakers'; const PROJECT_DATA_STORAGE_STATUSES = new Set(['ok', 'notice', 'warning', 'critical', 'degraded']); const PROJECT_DATA_STORAGE_CLEANUP_HEALTH_STATES = new Set([ @@ -52,6 +55,11 @@ const DEFAULT_STORAGE_TELEMETRY_LIST_MAX = 200; export const adminProjectDataStorageRoutes = new Hono<{ Bindings: Env }>(); +adminProjectDataStorageRoutes.route( + '/archive-sharding/circuit-breakers', + adminProjectDataArchiveBreakerRoutes +); + function parsePositiveIntegerConfig(raw: string | undefined, fallback: number): number { if (!raw?.trim()) return fallback; const parsed = Number.parseInt(raw, 10); @@ -83,15 +91,6 @@ function parseStorageTelemetryLimit(rawLimit: string | undefined, env: Env): num return parsedLimit; } -function parseArchiveRolloutLimit(rawLimit: string | undefined, env: Env): number { - const { defaultLimit, maxLimit } = getProjectDataArchiveRolloutListConfig(env); - const parsedLimit = rawLimit ? Number.parseInt(rawLimit, 10) : defaultLimit; - if (!Number.isSafeInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > maxLimit) { - throw errors.badRequest(`limit must be between 1 and ${maxLimit}`); - } - return parsedLimit; -} - function parseArchiveFrozenIntentLimit(rawLimit: string | undefined, env: Env): number { const { defaultLimit, maxLimit } = getProjectDataArchiveFrozenIntentInspectionConfig(env); const parsedLimit = rawLimit ? Number.parseInt(rawLimit, 10) : defaultLimit; diff --git a/apps/api/tests/unit/routes/admin-project-data-archive-breakers.test.ts b/apps/api/tests/unit/routes/admin-project-data-archive-breakers.test.ts new file mode 100644 index 0000000000..0026fddf6b --- /dev/null +++ b/apps/api/tests/unit/routes/admin-project-data-archive-breakers.test.ts @@ -0,0 +1,167 @@ +import Database from 'better-sqlite3'; +import { describe, expect, it } from 'vitest'; + +import * as schema from '../../../src/db/schema'; +import type { Env } from '../../../src/env'; +import { createSchemaTables, createSqliteD1 } from '../../helpers/sqlite-d1'; +import { createAdminProjectDataStorageApp } from './helpers/admin-project-data-storage-route'; + +const LIST_URL = + 'http://localhost/api/admin/project-data/storage/archive-sharding/circuit-breakers'; + +function makeEnv(sqlite: Database.Database): Env { + return { DATABASE: createSqliteD1(sqlite) } as Env; +} + +function createTables(sqlite: Database.Database): void { + createSchemaTables(sqlite, [schema.projects, schema.projectDataArchiveCircuitBreakers]); +} + +function seedProject(sqlite: Database.Database, id: string, name: string): void { + sqlite + .prepare( + `INSERT INTO projects (id, user_id, name, normalized_name, repository, created_at, updated_at) + VALUES (?, 'owner', ?, ?, ?, 1, 1)` + ) + .run(id, name, name.toLowerCase(), `org/${id}`); +} + +function seedBreaker( + sqlite: Database.Database, + input: { + projectId: string; + state: string; + reason?: string | null; + openedAt?: number | null; + updatedAt: number; + } +): void { + sqlite + .prepare( + `INSERT INTO project_data_archive_circuit_breakers (project_id, state, reason, opened_at, updated_at) + VALUES (?, ?, ?, ?, ?)` + ) + .run( + input.projectId, + input.state, + input.reason ?? null, + input.openedAt ?? null, + input.updatedAt + ); +} + +describe('GET /api/admin/project-data/storage/archive-sharding/circuit-breakers', () => { + it('lists breakers with project names, open breakers first, and skips malformed rows', async () => { + const sqlite = new Database(':memory:'); + createTables(sqlite); + seedProject(sqlite, 'project-open', 'SAM'); + seedProject(sqlite, 'project-closed', 'Other'); + seedBreaker(sqlite, { + projectId: 'project-closed', + state: 'closed', + reason: 'operator reset', + updatedAt: 5_000, + }); + seedBreaker(sqlite, { + projectId: 'project-open', + state: 'open', + reason: 'attempts_exhausted:Error', + openedAt: 1_000, + updatedAt: 2_000, + }); + // Breaker for a deleted project: still listed, name null. + seedBreaker(sqlite, { projectId: 'project-gone', state: 'frozen', updatedAt: 3_000 }); + // Malformed state must be skipped, not fail the list (rule 50). + seedBreaker(sqlite, { projectId: 'project-bad', state: 'bogus', updatedAt: 4_000 }); + + const app = createAdminProjectDataStorageApp(); + const response = await app.request( + LIST_URL, + { headers: { 'x-test-role': 'superadmin' } }, + makeEnv(sqlite) + ); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + breakers: Array>; + skippedRows: number; + limit: number; + }; + expect(body.skippedRows).toBe(1); + expect(body.breakers.map((b) => b.projectId)).toEqual([ + 'project-gone', + 'project-open', + 'project-closed', + ]); + expect(body.breakers[1]).toEqual({ + projectId: 'project-open', + projectName: 'SAM', + repository: 'org/project-open', + state: 'open', + reason: 'attempts_exhausted:Error', + openedAt: 1_000, + updatedAt: 2_000, + }); + expect(body.breakers[0]).toMatchObject({ + projectName: null, + repository: null, + state: 'frozen', + }); + }); + + it('rejects non-superadmin callers', async () => { + const sqlite = new Database(':memory:'); + createTables(sqlite); + const app = createAdminProjectDataStorageApp(); + const response = await app.request(LIST_URL, {}, makeEnv(sqlite)); + expect(response.status).toBe(403); + }); + + it('rejects an out-of-range limit', async () => { + const sqlite = new Database(':memory:'); + createTables(sqlite); + const app = createAdminProjectDataStorageApp(); + const response = await app.request( + `${LIST_URL}?limit=0`, + { headers: { 'x-test-role': 'superadmin' } }, + makeEnv(sqlite) + ); + expect(response.status).toBe(400); + }); + + it('closes a breaker through the control route and the list reflects it', async () => { + const sqlite = new Database(':memory:'); + createTables(sqlite); + seedProject(sqlite, 'project-open', 'SAM'); + seedBreaker(sqlite, { + projectId: 'project-open', + state: 'open', + reason: 'attempts_exhausted:Error', + openedAt: 1_000, + updatedAt: 2_000, + }); + const app = createAdminProjectDataStorageApp(); + const env = makeEnv(sqlite); + + const close = await app.request( + 'http://localhost/api/admin/project-data/storage/project-open/archive-sharding/circuit-breaker', + { + method: 'POST', + headers: { 'x-test-role': 'superadmin', 'content-type': 'application/json' }, + body: JSON.stringify({ state: 'closed', reason: 'Closed from admin UI' }), + }, + env + ); + expect(close.status).toBe(200); + + const list = await app.request(LIST_URL, { headers: { 'x-test-role': 'superadmin' } }, env); + const body = (await list.json()) as { breakers: Array> }; + expect(body.breakers).toHaveLength(1); + expect(body.breakers[0]).toMatchObject({ + projectId: 'project-open', + state: 'closed', + reason: 'Closed from admin UI', + openedAt: null, + }); + }); +}); diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index eb60f6ab63..7eadf3b938 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -50,6 +50,7 @@ const AdminProjectEvents = lazyNamed( () => import('./pages/AdminProjectEvents'), 'AdminProjectEvents' ); +const AdminStorage = lazyNamed(() => import('./pages/AdminStorage'), 'AdminStorage'); const AdminStream = lazyNamed(() => import('./pages/AdminStream'), 'AdminStream'); const AdminTrials = lazyNamed(() => import('./pages/AdminTrials'), 'AdminTrials'); const AdminUsers = lazyNamed(() => import('./pages/AdminUsers'), 'AdminUsers'); @@ -379,6 +380,7 @@ export default function App() { )} /> )} /> )} /> + )} /> )} /> )} /> )} /> diff --git a/apps/web/src/lib/api/admin-project-data-storage.ts b/apps/web/src/lib/api/admin-project-data-storage.ts new file mode 100644 index 0000000000..845209565b --- /dev/null +++ b/apps/web/src/lib/api/admin-project-data-storage.ts @@ -0,0 +1,42 @@ +import type { + AdminProjectDataArchiveCircuitBreakerControlResponse, + AdminProjectDataArchiveCircuitBreakersResponse, + AdminProjectDataStorageTelemetryResponse, +} from '@simple-agent-manager/shared'; + +import { request } from './client'; + +// ============================================================================= +// Admin ProjectData storage (Admin → Storage tab) +// ============================================================================= + +export async function fetchAdminProjectDataStorageTelemetry( + limit?: number +): Promise { + const params = limit ? `?limit=${limit}` : ''; + return request( + `/api/admin/project-data/storage${params}` + ); +} + +export async function fetchAdminProjectDataArchiveCircuitBreakers( + limit?: number +): Promise { + const params = limit ? `?limit=${limit}` : ''; + return request( + `/api/admin/project-data/storage/archive-sharding/circuit-breakers${params}` + ); +} + +export async function closeAdminProjectDataArchiveCircuitBreaker( + projectId: string, + reason: string +): Promise { + return request( + `/api/admin/project-data/storage/${encodeURIComponent(projectId)}/archive-sharding/circuit-breaker`, + { + method: 'POST', + body: JSON.stringify({ state: 'closed', reason }), + } + ); +} diff --git a/apps/web/src/lib/api/index.ts b/apps/web/src/lib/api/index.ts index cfc74593a7..1db21886c4 100644 --- a/apps/web/src/lib/api/index.ts +++ b/apps/web/src/lib/api/index.ts @@ -91,6 +91,11 @@ export { updatePlatformCredential, updateSignupApprovalConfig, } from './admin'; +export { + closeAdminProjectDataArchiveCircuitBreaker, + fetchAdminProjectDataArchiveCircuitBreakers, + fetchAdminProjectDataStorageTelemetry, +} from './admin-project-data-storage'; export { createAgentProfile, createSkill, diff --git a/apps/web/src/lib/query-options/admin-project-data-storage.ts b/apps/web/src/lib/query-options/admin-project-data-storage.ts new file mode 100644 index 0000000000..61c25091c0 --- /dev/null +++ b/apps/web/src/lib/query-options/admin-project-data-storage.ts @@ -0,0 +1,36 @@ +import { queryOptions } from '@tanstack/react-query'; + +import { + fetchAdminProjectDataArchiveCircuitBreakers, + fetchAdminProjectDataStorageTelemetry, +} from '../api'; + +export const adminProjectDataStorageQueryKeys = { + all: (queryScope: string) => ['auth', queryScope, 'admin', 'project-data-storage'] as const, + breakers: (queryScope: string, limit?: number) => + [ + ...adminProjectDataStorageQueryKeys.all(queryScope), + 'breakers', + { limit: limit ?? null }, + ] as const, + telemetry: (queryScope: string, limit?: number) => + [ + ...adminProjectDataStorageQueryKeys.all(queryScope), + 'telemetry', + { limit: limit ?? null }, + ] as const, +}; + +export function adminProjectDataArchiveBreakersQueryOptions(queryScope: string, limit?: number) { + return queryOptions({ + queryKey: adminProjectDataStorageQueryKeys.breakers(queryScope, limit), + queryFn: () => fetchAdminProjectDataArchiveCircuitBreakers(limit), + }); +} + +export function adminProjectDataStorageTelemetryQueryOptions(queryScope: string, limit?: number) { + return queryOptions({ + queryKey: adminProjectDataStorageQueryKeys.telemetry(queryScope, limit), + queryFn: () => fetchAdminProjectDataStorageTelemetry(limit), + }); +} diff --git a/apps/web/src/lib/query-options/index.ts b/apps/web/src/lib/query-options/index.ts index 0de21ecee3..2b072a64a1 100644 --- a/apps/web/src/lib/query-options/index.ts +++ b/apps/web/src/lib/query-options/index.ts @@ -29,6 +29,11 @@ * the `projects/detail` case documented in `query-persist-config.ts` shows. */ +export { + adminProjectDataArchiveBreakersQueryOptions, + adminProjectDataStorageQueryKeys, + adminProjectDataStorageTelemetryQueryOptions, +} from './admin-project-data-storage'; export { adminProjectEventInspectorQueryOptions, adminProjectEventQueryKeys, diff --git a/apps/web/src/pages/Admin.tsx b/apps/web/src/pages/Admin.tsx index 8a42d2b801..31458ab4c8 100644 --- a/apps/web/src/pages/Admin.tsx +++ b/apps/web/src/pages/Admin.tsx @@ -9,6 +9,7 @@ const ADMIN_TABS: Tab[] = [ { id: 'integrations', label: 'Integrations', path: 'integrations' }, { id: 'credentials', label: 'Credentials', path: 'credentials' }, { id: 'infrastructure', label: 'Infrastructure', path: 'infrastructure' }, + { id: 'storage', label: 'Storage', path: 'storage' }, { id: 'ai-proxy', label: 'AI Proxy', path: 'ai-proxy' }, { id: 'trials', label: 'Trials', path: 'trials' }, { id: 'costs', label: 'Costs', path: 'costs' }, diff --git a/apps/web/src/pages/AdminStorage.tsx b/apps/web/src/pages/AdminStorage.tsx new file mode 100644 index 0000000000..481d7be60d --- /dev/null +++ b/apps/web/src/pages/AdminStorage.tsx @@ -0,0 +1,317 @@ +import type { + AdminProjectDataArchiveCircuitBreaker, + AdminProjectDataStorageTelemetryRow, +} from '@simple-agent-manager/shared'; +import { Alert, Button, Card, Dialog, Input, Spinner } from '@simple-agent-manager/ui'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { ShieldCheck } from 'lucide-react'; +import { useState } from 'react'; + +import { formatBytes } from '../components/deployments/deployment-card-format'; +import { useQueryScope } from '../hooks/useQueryScope'; +import { useToast } from '../hooks/useToast'; +import { closeAdminProjectDataArchiveCircuitBreaker } from '../lib/api'; +import { + adminProjectDataArchiveBreakersQueryOptions, + adminProjectDataStorageQueryKeys, + adminProjectDataStorageTelemetryQueryOptions, +} from '../lib/query-options'; + +const DEFAULT_CLOSE_REASON = 'Closed from admin UI'; + +const BREAKER_BADGE: Record< + AdminProjectDataArchiveCircuitBreaker['state'], + { label: string; className: string } +> = { + open: { label: 'Open', className: 'bg-danger-tint text-danger-fg' }, + frozen: { label: 'Frozen', className: 'bg-warning-tint text-warning-fg' }, + closed: { label: 'Closed', className: 'bg-success-tint text-success-fg' }, +}; + +const STORAGE_STATUS_CLASS: Record = { + ok: 'bg-success-tint text-success-fg', + notice: 'bg-info-tint text-info-fg', + warning: 'bg-warning-tint text-warning-fg', + critical: 'bg-danger-tint text-danger-fg', + degraded: 'bg-danger-tint text-danger-fg', +}; + +function formatTimestamp(value: number | null): string { + if (value === null) return '—'; + return new Date(value).toLocaleString(); +} + +function formatPercent(ratio: number): string { + return `${(ratio * 100).toLocaleString(undefined, { maximumFractionDigits: 1 })}%`; +} + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error && error.message ? error.message : fallback; +} + +function Badge({ label, className }: { label: string; className: string }) { + return ( + + {label} + + ); +} + +function BreakerCard({ + breaker, + telemetry, + onClose, +}: { + breaker: AdminProjectDataArchiveCircuitBreaker; + telemetry: AdminProjectDataStorageTelemetryRow | undefined; + onClose: (breaker: AdminProjectDataArchiveCircuitBreaker) => void; +}) { + const badge = BREAKER_BADGE[breaker.state]; + return ( + +
+
+

+ {breaker.projectName ?? breaker.projectId} +

+ +
+
+
Project
+
+ {breaker.repository ?? breaker.projectId} +
+
Reason
+
{breaker.reason ?? '—'}
+
Opened
+
{formatTimestamp(breaker.openedAt)}
+
Updated
+
{formatTimestamp(breaker.updatedAt)}
+ {telemetry && ( + <> +
Storage
+
+ {formatBytes(telemetry.database_size_bytes)} of {formatBytes(telemetry.limit_bytes)}{' '} + ({formatPercent(telemetry.usage_ratio)}) +
+ + )} +
+ {breaker.state !== 'closed' && ( + + )} +
+
+ ); +} + +function TelemetryRow({ row }: { row: AdminProjectDataStorageTelemetryRow }) { + const statusClass = STORAGE_STATUS_CLASS[row.status] ?? 'bg-surface-secondary text-fg-muted'; + return ( +
  • +
    + + {row.project_name ?? row.project_id} + + +
    +
    + {formatBytes(row.database_size_bytes)} of {formatBytes(row.limit_bytes)} ( + {formatPercent(row.usage_ratio)}) + {row.growth_rate_bytes_per_day !== null && ( + <> · {formatBytes(row.growth_rate_bytes_per_day)}/day + )} + {row.estimated_days_to_limit !== null && ( + <> · ~{Math.round(row.estimated_days_to_limit).toLocaleString()} days to limit + )} + {row.cleanup_health && <> · cleanup {row.cleanup_health}} +
    + {row.last_error && ( +
    Last error: {row.last_error}
    + )} +
  • + ); +} + +export function AdminStorage() { + const queryScope = useQueryScope(); + const queryClient = useQueryClient(); + const toast = useToast(); + const [target, setTarget] = useState(null); + const [reason, setReason] = useState(DEFAULT_CLOSE_REASON); + + const breakersQuery = useQuery({ + ...adminProjectDataArchiveBreakersQueryOptions(queryScope), + enabled: Boolean(queryScope), + }); + const telemetryQuery = useQuery({ + ...adminProjectDataStorageTelemetryQueryOptions(queryScope), + enabled: Boolean(queryScope), + }); + + const closeBreaker = useMutation({ + mutationFn: (input: { projectId: string; reason: string }) => + closeAdminProjectDataArchiveCircuitBreaker(input.projectId, input.reason), + onSuccess: async (response) => { + toast.success( + `Archive breaker closed for ${target?.projectName ?? response.result.projectId}` + ); + setTarget(null); + setReason(DEFAULT_CLOSE_REASON); + await queryClient.invalidateQueries({ + queryKey: adminProjectDataStorageQueryKeys.all(queryScope), + }); + }, + onError: (error) => { + toast.error(errorMessage(error, 'Failed to close the archive breaker')); + }, + }); + + const breakers = breakersQuery.data?.breakers ?? []; + const telemetry = telemetryQuery.data?.telemetry ?? []; + const telemetryByProject = new Map(telemetry.map((row) => [row.project_id, row])); + const trimmedReason = reason.trim(); + + const openDialog = (breaker: AdminProjectDataArchiveCircuitBreaker) => { + closeBreaker.reset(); + setReason(DEFAULT_CLOSE_REASON); + setTarget(breaker); + }; + const dismissDialog = () => { + if (closeBreaker.isPending) return; + setTarget(null); + }; + + return ( +
    +
    +

    Storage

    +

    + ProjectData storage health and the per-project archive circuit breakers. A tripped breaker + stops the scheduled archive drain for that project until an admin closes it here. +

    +
    + +
    +

    Archive circuit breakers

    + {breakersQuery.isError && !breakersQuery.data && ( + + {errorMessage(breakersQuery.error, 'Failed to load archive circuit breakers')} + + )} + {breakersQuery.isPending && !breakersQuery.isError && ( +
    + +
    + )} + {breakersQuery.data && breakers.length === 0 && ( +

    No archive circuit breakers recorded.

    + )} + {breakers.map((breaker) => ( + + ))} + {breakersQuery.data && breakersQuery.data.skippedRows > 0 && ( +

    + {breakersQuery.data.skippedRows} malformed breaker row(s) were skipped. +

    + )} +
    + +
    +

    Storage telemetry

    + {telemetryQuery.isError && !telemetryQuery.data && ( + + {errorMessage(telemetryQuery.error, 'Failed to load storage telemetry')} + + )} + {telemetryQuery.isPending && !telemetryQuery.isError && ( +
    + +
    + )} + {telemetryQuery.data && telemetry.length === 0 && ( +

    No storage telemetry recorded yet.

    + )} + {telemetry.length > 0 && ( + +
      + {telemetry.map((row) => ( + + ))} +
    +
    + )} +
    + + + {target && ( +
    { + event.preventDefault(); + if (!trimmedReason || closeBreaker.isPending) return; + closeBreaker.mutate({ projectId: target.projectId, reason: trimmedReason }); + }} + > +

    + Close archive circuit breaker +

    +

    + This lets the scheduled archive sweep resume for{' '} + + {target.projectName ?? target.projectId} + + . Already frozen migrations stay frozen until they are copied back or abandoned. +

    + + {closeBreaker.isError && ( + + {errorMessage(closeBreaker.error, 'Failed to close the archive breaker')} + + )} +
    + + +
    +
    + )} +
    +
    + ); +} diff --git a/apps/web/tests/playwright/admin-storage-audit.spec.ts b/apps/web/tests/playwright/admin-storage-audit.spec.ts new file mode 100644 index 0000000000..a0a3d25774 --- /dev/null +++ b/apps/web/tests/playwright/admin-storage-audit.spec.ts @@ -0,0 +1,292 @@ +import { expect, type Page, type Route, test } from '@playwright/test'; + +import { assertNoOverflow, makeMockUser, screenshot, setupAuditRoutes } from './audit-helpers'; + +const ADMIN_USER = makeMockUser({ + email: 'admin@example.com', + name: 'Admin User', + role: 'superadmin', + sessionId: 'session-admin-storage', + userId: 'user-admin-storage', +}); + +const LONG_REASON = + 'attempts_exhausted:Error: Compact archive R2 deadline exceeded after 3 attempts on migration 67927ce6 ' + + '(session 1d438cc7, 8,962 messages) — see project_data_archive_migrations for the poisoned row 🚨'; + +const BREAKERS = { + breakers: [ + { + projectId: '01KHRJGANBBWGDY1NZ0KVF0D4J', + projectName: + 'SAM — simple-agent-manager with an intentionally very long project name to stress wrapping', + repository: 'raphaeltm/simple-agent-manager', + state: 'open', + reason: LONG_REASON, + openedAt: Date.now() - 2 * 86_400_000, + updatedAt: Date.now() - 3_600_000, + }, + { + projectId: 'project-frozen', + projectName: 'Frozen project', + repository: 'org/frozen', + state: 'frozen', + reason: 'operator freeze', + openedAt: Date.now() - 86_400_000, + updatedAt: Date.now() - 86_400_000, + }, + { + projectId: 'project-deleted', + projectName: null, + repository: null, + state: 'open', + reason: null, + openedAt: null, + updatedAt: Date.now() - 10 * 86_400_000, + }, + { + projectId: 'project-closed', + projectName: 'Healthy project', + repository: 'org/healthy', + state: 'closed', + reason: 'Closed from admin UI', + openedAt: null, + updatedAt: Date.now() - 5 * 86_400_000, + }, + ], + skippedRows: 0, + limit: 25, +}; + +const TELEMETRY = { + telemetry: [ + { + project_id: '01KHRJGANBBWGDY1NZ0KVF0D4J', + project_name: + 'SAM — simple-agent-manager with an intentionally very long project name to stress wrapping', + repository: 'raphaeltm/simple-agent-manager', + measured_at: Date.now() - 60_000, + database_size_bytes: 10_097_864_704, + limit_bytes: 10_000_000_000, + usage_ratio: 1.0097864704, + status: 'degraded', + growth_rate_bytes_per_day: 30_925_756, + estimated_days_to_limit: 0, + cleanup_health: 'running', + last_error: null, + updated_at: Date.now() - 60_000, + }, + { + project_id: 'project-frozen', + project_name: 'Frozen project', + repository: 'org/frozen', + measured_at: Date.now() - 600_000, + database_size_bytes: 4_200_000_000, + limit_bytes: 10_000_000_000, + usage_ratio: 0.42, + status: 'warning', + growth_rate_bytes_per_day: 120_000_000, + estimated_days_to_limit: 48, + cleanup_health: 'not_needed', + last_error: null, + updated_at: Date.now() - 600_000, + }, + { + project_id: 'p', + project_name: 'X', + repository: 'o/r', + measured_at: Date.now(), + database_size_bytes: 1_024, + limit_bytes: 10_000_000_000, + usage_ratio: 0.0000001, + status: 'ok', + growth_rate_bytes_per_day: null, + estimated_days_to_limit: null, + cleanup_health: null, + last_error: 'Compact archive R2 deadline exceeded', + updated_at: Date.now(), + }, + ], +}; + +/** + * The first-run cloud onboarding wizard overlays every authenticated page when the + * mocked user has no credentials. It would cover the surface under test (see + * .claude/rules/62), so mark it dismissed before the app boots. + */ +async function dismissOnboardingWizard(page: Page) { + await page.addInitScript((userId) => { + window.localStorage.setItem(`sam-onboarding-wizard-dismissed-${userId}`, 'true'); + }, ADMIN_USER.user.id); +} + +async function respondJson(route: Route, status: number, body: unknown) { + await route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); +} + +async function setupMocks( + page: Page, + options: { + breakers?: unknown; + telemetry?: unknown; + breakerStatus?: number; + telemetryStatus?: number; + } +) { + await dismissOnboardingWizard(page); + await setupAuditRoutes(page, (path, respond) => { + if (path === '/api/auth/get-session') return respond(200, ADMIN_USER); + if (path === '/api/dashboard/active-tasks') return respond(200, { tasks: [] }); + if (path === '/api/trial-status') return respond(200, { isTrial: false }); + if (path === '/api/projects') return respond(200, { projects: [], total: 0 }); + if (path === '/api/notifications/unread-count') return respond(200, { count: 0 }); + if (path === '/api/notifications') { + return respond(200, { notifications: [], unreadCount: 0, nextCursor: null }); + } + if (path.startsWith('/api/credentials')) return respond(200, []); + if (path === '/api/github/installations') return respond(200, []); + if (path === '/api/workspaces') return respond(200, []); + if (path.startsWith('/api/provider-catalog')) return respond(200, { catalogs: [] }); + if (path === '/api/admin/project-data/storage/archive-sharding/circuit-breakers') { + return respond(options.breakerStatus ?? 200, options.breakers ?? BREAKERS); + } + if (path === '/api/admin/project-data/storage') { + return respond(options.telemetryStatus ?? 200, options.telemetry ?? TELEMETRY); + } + return undefined; + }); +} + +async function openStoragePage(page: Page) { + await page.goto('/admin/storage'); + await page.waitForTimeout(700); + await expect(page.getByText('Something went wrong')).toHaveCount(0); + await expect(page.getByRole('tab', { name: 'Storage', selected: true })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Archive circuit breakers' })).toBeVisible(); +} + +test.describe('AdminStorage', () => { + test('open, frozen, deleted-project and closed breakers with long text and special characters', async ({ + page, + }) => { + await setupMocks(page, {}); + await openStoragePage(page); + // Only non-closed breakers expose the close control. + await expect(page.getByRole('button', { name: /close breaker/i })).toHaveCount(3); + await expect(page.getByText('Healthy project')).toBeVisible(); + await screenshot(page, 'admin-storage-breakers'); + await assertNoOverflow(page); + }); + + test('empty state', async ({ page }) => { + await setupMocks(page, { + breakers: { breakers: [], skippedRows: 0, limit: 25 }, + telemetry: { telemetry: [] }, + }); + await openStoragePage(page); + await expect(page.getByText('No archive circuit breakers recorded.')).toBeVisible(); + await expect(page.getByText('No storage telemetry recorded yet.')).toBeVisible(); + await screenshot(page, 'admin-storage-empty'); + await assertNoOverflow(page); + }); + + test('error state', async ({ page }) => { + await setupMocks(page, { + breakerStatus: 500, + breakers: { error: 'INTERNAL', message: 'D1 unavailable' }, + telemetryStatus: 500, + telemetry: { error: 'INTERNAL', message: 'D1 unavailable' }, + }); + await page.goto('/admin/storage'); + await page.waitForTimeout(700); + await expect(page.getByRole('alert').first()).toContainText('D1 unavailable'); + await screenshot(page, 'admin-storage-error'); + await assertNoOverflow(page); + }); + + test('closing a breaker posts state=closed with the reason and refreshes the list', async ({ + page, + }) => { + let breakers = structuredClone(BREAKERS); + const postBodies: Array<{ path: string; body: unknown }> = []; + await dismissOnboardingWizard(page); + await page.route('**/api/**', async (route) => { + const request = route.request(); + const path = new URL(request.url()).pathname; + if (path === '/api/auth/get-session') return respondJson(route, 200, ADMIN_USER); + if (path === '/api/dashboard/active-tasks') return respondJson(route, 200, { tasks: [] }); + if (path === '/api/trial-status') return respondJson(route, 200, { isTrial: false }); + if (path === '/api/projects') return respondJson(route, 200, { projects: [], total: 0 }); + if (path === '/api/notifications/unread-count') return respondJson(route, 200, { count: 0 }); + if (path === '/api/notifications') { + return respondJson(route, 200, { notifications: [], unreadCount: 0, nextCursor: null }); + } + if (path.startsWith('/api/credentials')) return respondJson(route, 200, []); + if (path === '/api/github/installations') return respondJson(route, 200, []); + if (path === '/api/workspaces') return respondJson(route, 200, []); + if (path.startsWith('/api/provider-catalog')) + return respondJson(route, 200, { catalogs: [] }); + if (path === '/api/admin/project-data/storage') return respondJson(route, 200, TELEMETRY); + if (path === '/api/admin/project-data/storage/archive-sharding/circuit-breakers') { + return respondJson(route, 200, breakers); + } + const match = path.match( + /^\/api\/admin\/project-data\/storage\/([^/]+)\/archive-sharding\/circuit-breaker$/ + ); + if (match && request.method() === 'POST') { + const body = request.postDataJSON() as { state: string; reason: string }; + postBodies.push({ path, body }); + breakers = { + ...breakers, + breakers: breakers.breakers.map((b) => + b.projectId === decodeURIComponent(match[1]) + ? { + ...b, + state: body.state, + reason: body.reason, + openedAt: null, + updatedAt: Date.now(), + } + : b + ), + }; + return respondJson(route, 200, { + result: { + projectId: match[1], + state: body.state, + reason: body.reason, + frozenMigrations: 0, + frozenLocations: 0, + updatedAt: Date.now(), + note: 'Circuit breaker closed for future archive work.', + }, + }); + } + return respondJson(route, 200, {}); + }); + + await openStoragePage(page); + const samCard = page.getByTestId('breaker-01KHRJGANBBWGDY1NZ0KVF0D4J'); + await samCard.getByRole('button', { name: /close breaker/i }).click(); + + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + await screenshot(page, 'admin-storage-close-dialog'); + await assertNoOverflow(page); + + await dialog.getByLabel('Reason').fill('Archive fix deployed; resume drain'); + await dialog.getByRole('button', { name: /^close breaker$/i }).click(); + + await expect(dialog).toHaveCount(0); + await expect(samCard.getByText('Closed', { exact: true })).toBeVisible(); + await expect(samCard.getByRole('button', { name: /close breaker/i })).toHaveCount(0); + expect(postBodies).toEqual([ + { + path: '/api/admin/project-data/storage/01KHRJGANBBWGDY1NZ0KVF0D4J/archive-sharding/circuit-breaker', + body: { state: 'closed', reason: 'Archive fix deployed; resume drain' }, + }, + ]); + await screenshot(page, 'admin-storage-after-close'); + await assertNoOverflow(page); + }); +}); diff --git a/apps/web/tests/unit/AdminStorage.test.tsx b/apps/web/tests/unit/AdminStorage.test.tsx new file mode 100644 index 0000000000..c692ca2295 --- /dev/null +++ b/apps/web/tests/unit/AdminStorage.test.tsx @@ -0,0 +1,181 @@ +import { fireEvent, screen, waitFor } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { ToastProvider } from '../../src/hooks/useToast'; +import { renderWithQuery } from '../test-utils/query-test-utils'; + +const mocks = vi.hoisted(() => ({ + fetchAdminProjectDataArchiveCircuitBreakers: vi.fn(), + fetchAdminProjectDataStorageTelemetry: vi.fn(), + closeAdminProjectDataArchiveCircuitBreaker: vi.fn(), +})); + +vi.mock('../../src/components/AuthProvider', () => ({ + useAuth: () => ({ + user: { id: 'admin-1', email: 'admin@example.com', name: 'Admin' }, + }), +})); + +vi.mock('../../src/lib/api', async (importOriginal) => ({ + ...(await importOriginal()), + fetchAdminProjectDataArchiveCircuitBreakers: mocks.fetchAdminProjectDataArchiveCircuitBreakers, + fetchAdminProjectDataStorageTelemetry: mocks.fetchAdminProjectDataStorageTelemetry, + closeAdminProjectDataArchiveCircuitBreaker: mocks.closeAdminProjectDataArchiveCircuitBreaker, +})); + +import { AdminStorage } from '../../src/pages/AdminStorage'; + +const OPEN_BREAKER = { + projectId: 'project-sam', + projectName: 'SAM', + repository: 'org/sam', + state: 'open' as const, + reason: 'attempts_exhausted:Error', + openedAt: 1_700_000_000_000, + updatedAt: 1_700_000_100_000, +}; + +const CLOSED_BREAKER = { + projectId: 'project-other', + projectName: 'Other', + repository: 'org/other', + state: 'closed' as const, + reason: 'operator reset', + openedAt: null, + updatedAt: 1_700_000_200_000, +}; + +function Wrapper({ children }: { children: ReactNode }) { + return {children}; +} + +function renderPage() { + return renderWithQuery(, { wrapper: Wrapper }); +} + +describe('AdminStorage', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.fetchAdminProjectDataArchiveCircuitBreakers.mockResolvedValue({ + breakers: [OPEN_BREAKER, CLOSED_BREAKER], + skippedRows: 0, + limit: 25, + }); + mocks.fetchAdminProjectDataStorageTelemetry.mockResolvedValue({ + telemetry: [ + { + project_id: 'project-sam', + project_name: 'SAM', + repository: 'org/sam', + measured_at: 1_700_000_100_000, + database_size_bytes: 10_097_864_704, + limit_bytes: 10_000_000_000, + usage_ratio: 1.0097864704, + status: 'degraded', + growth_rate_bytes_per_day: 30_925_756, + estimated_days_to_limit: 0, + cleanup_health: 'running', + last_error: null, + updated_at: 1_700_000_100_000, + }, + ], + }); + }); + + it('shows the close control only for breakers that are not closed', async () => { + renderPage(); + + const samCard = await screen.findByTestId('breaker-project-sam'); + expect(samCard).toHaveTextContent('attempts_exhausted:Error'); + expect(samCard).toHaveTextContent('Open'); + expect(samCard.querySelector('button')).toHaveTextContent(/close breaker/i); + + const otherCard = screen.getByTestId('breaker-project-other'); + expect(otherCard).toHaveTextContent('Closed'); + expect(otherCard.querySelector('button')).toBeNull(); + + // Telemetry section is live alongside the breaker list. + expect(await screen.findByText(/degraded/)).toBeInTheDocument(); + }); + + it('closes a breaker with the entered reason and refetches the list', async () => { + mocks.closeAdminProjectDataArchiveCircuitBreaker.mockResolvedValue({ + result: { + projectId: 'project-sam', + state: 'closed', + reason: 'fix deployed', + frozenMigrations: 0, + frozenLocations: 0, + updatedAt: 1_700_000_300_000, + note: null, + }, + }); + mocks.fetchAdminProjectDataArchiveCircuitBreakers + .mockResolvedValueOnce({ + breakers: [OPEN_BREAKER, CLOSED_BREAKER], + skippedRows: 0, + limit: 25, + }) + .mockResolvedValue({ + breakers: [{ ...OPEN_BREAKER, state: 'closed', reason: 'fix deployed' }, CLOSED_BREAKER], + skippedRows: 0, + limit: 25, + }); + + renderPage(); + + const samCard = await screen.findByTestId('breaker-project-sam'); + fireEvent.click(samCard.querySelector('button')!); + + const dialog = await screen.findByRole('dialog'); + expect(dialog).toHaveTextContent('Close archive circuit breaker'); + const reasonInput = screen.getByLabelText('Reason') as HTMLInputElement; + expect(reasonInput.value).toBe('Closed from admin UI'); + fireEvent.change(reasonInput, { target: { value: ' fix deployed ' } }); + fireEvent.click(screen.getByRole('button', { name: /^close breaker$/i })); + + await waitFor(() => { + expect(mocks.closeAdminProjectDataArchiveCircuitBreaker).toHaveBeenCalledWith( + 'project-sam', + 'fix deployed' + ); + }); + await waitFor(() => { + expect(screen.queryByRole('dialog')).toBeNull(); + }); + await waitFor(() => { + expect(screen.getByTestId('breaker-project-sam').querySelector('button')).toBeNull(); + }); + expect(screen.getByTestId('breaker-project-sam')).toHaveTextContent('fix deployed'); + expect(mocks.fetchAdminProjectDataArchiveCircuitBreakers).toHaveBeenCalledTimes(2); + }); + + it('keeps the dialog open and shows the error when closing fails', async () => { + mocks.closeAdminProjectDataArchiveCircuitBreaker.mockRejectedValue( + new Error('D1 write failed') + ); + + renderPage(); + + const samCard = await screen.findByTestId('breaker-project-sam'); + fireEvent.click(samCard.querySelector('button')!); + fireEvent.click(await screen.findByRole('button', { name: /^close breaker$/i })); + + expect( + await screen.findByText('D1 write failed', { selector: '[role="alert"] *' }) + ).toBeInTheDocument(); + expect(screen.getByRole('dialog')).toBeInTheDocument(); + expect(mocks.fetchAdminProjectDataArchiveCircuitBreakers).toHaveBeenCalledTimes(1); + }); + + it('shows an error when the breaker list fails to load', async () => { + mocks.fetchAdminProjectDataArchiveCircuitBreakers.mockRejectedValue(new Error('no d1 access')); + + renderPage(); + + expect(await screen.findByText('no d1 access')).toBeInTheDocument(); + // Telemetry still renders: one failed query must not blank the page. + expect(await screen.findByText(/degraded/)).toBeInTheDocument(); + }); +}); diff --git a/apps/www/src/content/docs/docs/guides/self-hosting.mdx b/apps/www/src/content/docs/docs/guides/self-hosting.mdx index 98b40f7a22..b1979a4b67 100644 --- a/apps/www/src/content/docs/docs/guides/self-hosting.mdx +++ b/apps/www/src/content/docs/docs/guides/self-hosting.mdx @@ -381,6 +381,10 @@ Notes: The sentinel user id defaults to `system_anonymous_trials`. If your deployment uses a different sentinel id, set `TRIAL_ANONYMOUS_USER_ID` to that value so the first-user checks exclude it correctly. This variable scopes the login-time self-heal hook and the first-user creation check; the deploy-time migration excludes the sentinel by its `status='system'` flag (not by id), so it stays correct regardless of this setting because the sentinel is always seeded with `status='system'`. Most deployments never need to set this. +### Storage and archive circuit breakers + +**Admin → Storage** (`/admin/storage`) shows per-project ProjectData storage telemetry and the archive-sharding circuit breakers. When a project's archive drain fails repeatedly, its breaker opens and the scheduled sweep stops archiving that project until a superadmin closes the breaker. Use the **Close breaker** button on that page (it works from a phone) once the underlying failure is fixed; it calls `POST /api/admin/project-data/storage/:projectId/archive-sharding/circuit-breaker` with `state: "closed"` and the reason you enter. Closing a breaker does not thaw migrations that were already frozen. + ## Teardown To remove all resources: Actions → Teardown → Run workflow → type `DELETE` to confirm. diff --git a/packages/shared/src/types/admin.ts b/packages/shared/src/types/admin.ts index fac09266df..001b5022f2 100644 --- a/packages/shared/src/types/admin.ts +++ b/packages/shared/src/types/admin.ts @@ -278,3 +278,62 @@ export interface LogStreamClientMessage { levels?: string[]; search?: string; } + +// ============================================================================= +// Admin ProjectData storage (archive circuit breakers + storage telemetry) +// ============================================================================= + +export type AdminProjectDataArchiveCircuitBreakerState = 'closed' | 'open' | 'frozen'; + +/** One row of `project_data_archive_circuit_breakers`, joined to the project name. */ +export interface AdminProjectDataArchiveCircuitBreaker { + projectId: string; + projectName: string | null; + repository: string | null; + state: AdminProjectDataArchiveCircuitBreakerState; + reason: string | null; + openedAt: number | null; + updatedAt: number; +} + +export interface AdminProjectDataArchiveCircuitBreakersResponse { + breakers: AdminProjectDataArchiveCircuitBreaker[]; + /** Rows skipped because they were malformed (rule 50: one bad row must not hide the list). */ + skippedRows: number; + limit: number; +} + +export interface AdminProjectDataArchiveCircuitBreakerControlResult { + projectId: string; + state: AdminProjectDataArchiveCircuitBreakerState; + reason: string; + frozenMigrations: number; + frozenLocations: number; + updatedAt: number; + note: string | null; +} + +export interface AdminProjectDataArchiveCircuitBreakerControlResponse { + result: AdminProjectDataArchiveCircuitBreakerControlResult; +} + +/** Raw D1 row shape returned by `GET /api/admin/project-data/storage`. */ +export interface AdminProjectDataStorageTelemetryRow { + project_id: string; + project_name: string | null; + repository: string | null; + measured_at: number; + database_size_bytes: number; + limit_bytes: number; + usage_ratio: number; + status: string; + growth_rate_bytes_per_day: number | null; + estimated_days_to_limit: number | null; + cleanup_health: string | null; + last_error: string | null; + updated_at: number; +} + +export interface AdminProjectDataStorageTelemetryResponse { + telemetry: AdminProjectDataStorageTelemetryRow[]; +} diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index c6fea3878c..fd92ab46bc 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -462,6 +462,13 @@ export type { AdminLogEntry, AdminNodesResponse, AdminNodeSummary, + AdminProjectDataArchiveCircuitBreaker, + AdminProjectDataArchiveCircuitBreakerControlResponse, + AdminProjectDataArchiveCircuitBreakerControlResult, + AdminProjectDataArchiveCircuitBreakersResponse, + AdminProjectDataArchiveCircuitBreakerState, + AdminProjectDataStorageTelemetryResponse, + AdminProjectDataStorageTelemetryRow, AdminProjectEventInspectorAdapterDecision, AdminProjectEventInspectorAttempt, AdminProjectEventInspectorBatch, diff --git a/tasks/backlog/2026-09-23-playwright-audit-shell-mocks-crash.md b/tasks/backlog/2026-09-23-playwright-audit-shell-mocks-crash.md new file mode 100644 index 0000000000..b5e3ab9da8 --- /dev/null +++ b/tasks/backlog/2026-09-23-playwright-audit-shell-mocks-crash.md @@ -0,0 +1,32 @@ +# Playwright audit specs crash the app shell when `/api/credentials` is unmocked + +## Problem + +`setupAuditRoutes` answers every unmatched `/api/**` request with `{}`. The app shell now +fetches `/api/credentials` (and `/api/github/installations`) on every authenticated page and +calls an array method on the result, so any audit spec that does not explicitly mock those +endpoints as `[]` renders the "Something went wrong" crash screen (`e.some is not a function`) +instead of the surface under test. `admin-trials-audit.spec.ts` fails this way today on `main` +(4/4 failing at Desktop 1280x800 on 2026-09-23); it only stays invisible because CI runs a +selected subset of audit specs. + +Separately, the first-run cloud onboarding wizard overlays every page for a mocked user without +credentials, so specs that do not seed `sam-onboarding-wizard-dismissed-` in +localStorage screenshot the wizard, not the page (rule 62 incident class). + +## Context + +Discovered while adding `admin-storage-audit.spec.ts` in PR #2135, which works around both by +mocking `/api/credentials*`, `/api/github/installations`, `/api/workspaces`, +`/api/provider-catalog*` and dismissing the wizard via `addInitScript`. + +## Acceptance Criteria + +- [ ] `setupAuditRoutes` (or a shared `setupAppShellMocks` helper) provides array-shaped defaults + for the shell fetches and dismisses the onboarding wizard by default +- [ ] `admin-trials-audit.spec.ts` and every other admin audit spec pass locally at + 375x667 and 1280x800 without per-spec shell mocks +- [ ] The full local audit corpus (`playwright.audit.config.ts`) is run once and any other + spec broken by the same cause is fixed +- [ ] The default `{}` fallback in `setupAuditRoutes` fails loudly (or logs) for unmatched + `/api/**` paths so the next new shell fetch is noticed