From b5c316794e4de388eb07657a60668eeb5e04a078 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:37:14 -0700 Subject: [PATCH 1/7] REFACTOR: Port files, tags, and purge to Postgres Files, tags, thumbnails, and purge run entirely on PgSql. Helper modules (thumbnail-storage, purge-sql, blob-compensation) become functions over the Postgres client. Search hydration reads Postgres because the dashboard column list is Postgres-only now. Three route tests fail on this commit by design: indexing and sites still write the files table in D1, so indexed search and site publishing are split across engines until the next two PRs port them. Co-Authored-By: Claude Fable 5.1 --- apps/web/src/lib/server/blob-compensation.ts | 23 +- apps/web/src/lib/server/file-rows.ts | 33 +-- apps/web/src/lib/server/purge-sql.ts | 86 +++--- apps/web/src/lib/server/services/files.ts | 8 +- .../lib/server/services/files/internals.ts | 154 +++++------ .../lib/server/services/files/mutations.ts | 155 +++++------ .../src/lib/server/services/files/purge.ts | 250 +++++++---------- .../src/lib/server/services/files/queries.ts | 120 ++++----- .../lib/server/services/files/thumbnails.ts | 80 +++--- .../src/lib/server/services/files/types.ts | 4 +- .../src/lib/server/services/files/upload.ts | 80 +++--- apps/web/src/lib/server/services/search.ts | 94 +++---- apps/web/src/lib/server/services/tags.ts | 252 ++++++++---------- apps/web/src/lib/server/thumbnail-storage.ts | 56 ++-- 14 files changed, 600 insertions(+), 795 deletions(-) diff --git a/apps/web/src/lib/server/blob-compensation.ts b/apps/web/src/lib/server/blob-compensation.ts index 175084d..04bba12 100644 --- a/apps/web/src/lib/server/blob-compensation.ts +++ b/apps/web/src/lib/server/blob-compensation.ts @@ -1,23 +1,22 @@ +import type { PgClient } from '@effect/sql-pg'; import { Cause, Effect } from 'effect'; -export interface BlobDeleteSqlCommand { - readonly sql: string; - readonly bindings: ReadonlyArray; -} - -export const deferredBlobDeleteCommand = ( +// R2 cleanup after a failed commit is compensatable: when the delete +// itself fails the key goes to the durable lifecycle queue and a later +// sweep retries it. Idempotent on r2_key. +export const queueDeferredBlobDelete = ( + sql: PgClient.PgClient, r2Key: string, fileId: string, version: number, queuedAt: string, lastError: string -): BlobDeleteSqlCommand => ({ - sql: `INSERT INTO pending_site_asset_deletes ( +) => + sql` + INSERT INTO pending_site_asset_deletes ( r2_key, file_id, version, queued_at, attempts, last_error - ) VALUES (?, ?, ?, ?, 1, ?) - ON CONFLICT(r2_key) DO NOTHING`, - bindings: [r2Key, fileId, version, queuedAt, lastError] -}); + ) VALUES (${r2Key}, ${fileId}, ${version}, ${queuedAt}, 1, ${lastError}) + ON CONFLICT (r2_key) DO NOTHING`.pipe(Effect.asVoid); export const compensateBlobFailure = < OriginalError, diff --git a/apps/web/src/lib/server/file-rows.ts b/apps/web/src/lib/server/file-rows.ts index 49fbb00..8b53bce 100644 --- a/apps/web/src/lib/server/file-rows.ts +++ b/apps/web/src/lib/server/file-rows.ts @@ -8,8 +8,8 @@ export const DashboardFileRow = Schema.Struct({ kind: Schema.Literals(['file', 'site']), current_version: Schema.Int, size_bytes: Schema.Int, - is_public: Schema.Int, - has_html: Schema.Int, + is_public: Schema.Boolean, + has_html: Schema.Boolean, created_at: Schema.String, updated_at: Schema.String, deleted_at: Schema.NullOr(Schema.String), @@ -26,9 +26,12 @@ export const DashboardFileRow = Schema.Struct({ indexed_version: Schema.NullOr(Schema.Int), index_attempts: Schema.Int, index_error: Schema.NullOr(Schema.String), - tags_json: Schema.String + // jsonb comes back from pg already parsed. + tags_json: Schema.Array(TagSchema) }); +// Column list for the `files f` alias. Interpolate as a raw string through +// `sql.unsafe` or wrap with `sql.literal`; it contains no parameters. export const dashboardFileColumns = ` f.id, f.display_name, @@ -53,19 +56,18 @@ export const dashboardFileColumns = ` f.index_attempts, f.index_error, COALESCE(( - SELECT json_group_array(json_object( + SELECT jsonb_agg(jsonb_build_object( 'id', t.id, 'name', t.name, 'normalizedName', t.normalized_name, 'color', t.color, 'fileCount', 0, 'createdAt', t.created_at - )) + ) ORDER BY t.normalized_name) FROM file_tags ft JOIN tags t ON t.id = ft.tag_id WHERE ft.file_id = f.id - ORDER BY t.normalized_name - ), '[]') AS tags_json + ), '[]'::jsonb) AS tags_json `; export const decodeDashboardRows = (rows: unknown) => { @@ -75,17 +77,6 @@ export const decodeDashboardRows = (rows: unknown) => { return decoded._tag === 'Some' ? decoded.value : []; }; -const decodeTags = (value: string) => { - try { - const decoded = Schema.decodeUnknownOption(Schema.Array(TagSchema))( - JSON.parse(value) - ); - return decoded._tag === 'Some' ? decoded.value : []; - } catch { - return []; - } -}; - export const toDashboardFile = ( row: typeof DashboardFileRow.Type ): DashboardFile => ({ @@ -95,8 +86,8 @@ export const toDashboardFile = ( kind: row.kind, version: row.current_version, sizeBytes: row.size_bytes, - public: row.is_public === 1, - htmlForcedPublic: row.has_html === 1, + public: row.is_public, + htmlForcedPublic: row.has_html, createdAt: row.created_at, updatedAt: row.updated_at, deletedAt: row.deleted_at, @@ -107,5 +98,5 @@ export const toDashboardFile = ( indexedVersion: row.indexed_version, indexAttempts: row.index_attempts, indexError: row.index_error, - tags: decodeTags(row.tags_json) + tags: row.tags_json }); diff --git a/apps/web/src/lib/server/purge-sql.ts b/apps/web/src/lib/server/purge-sql.ts index 0c639aa..a10273a 100644 --- a/apps/web/src/lib/server/purge-sql.ts +++ b/apps/web/src/lib/server/purge-sql.ts @@ -1,53 +1,35 @@ -export type PurgeSqlBinding = string | number | null; +import type { PgClient } from '@effect/sql-pg'; +import { Effect } from 'effect'; +import { StorageError } from './errors'; -export interface PurgeSqlCommand { - readonly sql: string; - readonly bindings: ReadonlyArray; -} - -export const purgeCompletionCommands = ( - fileId: string, - queuedAt: string -): ReadonlyArray => [ - { - sql: `INSERT INTO pending_vector_deletes (vector_id, queued_at) - SELECT vector_id, ? FROM file_chunks - WHERE file_id = ? - AND EXISTS ( - SELECT 1 FROM files - WHERE id = ? AND purge_state = 'pending' - ) - ON CONFLICT(vector_id) DO NOTHING`, - bindings: [queuedAt, fileId, fileId] - }, - { - sql: `DELETE FROM files_fts - WHERE file_id = ? - AND EXISTS ( - SELECT 1 FROM files - WHERE id = ? AND purge_state = 'pending' - )`, - bindings: [fileId, fileId] - }, - { - sql: `DELETE FROM files_trgm - WHERE file_id = ? - AND EXISTS ( - SELECT 1 FROM files - WHERE id = ? AND purge_state = 'pending' - )`, - bindings: [fileId, fileId] - }, - { - sql: `UPDATE files - SET purge_state = 'done', purge_error = NULL, - purge_next_run_at = NULL - WHERE id = ? AND purge_state = 'pending'`, - bindings: [fileId] - }, - { - sql: `DELETE FROM files - WHERE id = ? AND purge_state = 'done'`, - bindings: [fileId] - } -]; +// Runs after R2 confirmed the blobs are gone. Deleting the files row +// cascades to file_versions, file_tags, site_assets, file_chunks, and +// search_documents, so completion is only the ownership hand-off: mark +// the pending claim done, then drop the row. Both steps must hit exactly +// one row or the transaction rolls back and a later sweep retries. +export const completePurge = (sql: PgClient.PgClient, fileId: string) => + sql + .withTransaction( + Effect.gen(function* () { + const done = yield* sql<{ id: string }>` + UPDATE files + SET purge_state = 'done', purge_error = NULL, purge_next_run_at = NULL + WHERE id = ${fileId} AND purge_state = 'pending' + RETURNING id`; + const deleted = yield* sql<{ id: string }>` + DELETE FROM files + WHERE id = ${fileId} AND purge_state = 'done' + RETURNING id`; + if (done.length !== 1 || deleted.length !== 1) { + return yield* new StorageError({ + operation: 'finish file purge', + cause: 'File purge state changed before completion' + }); + } + }) + ) + .pipe( + Effect.catchTag('SqlError', (cause) => + Effect.fail(new StorageError({ operation: 'finish file purge', cause })) + ) + ); diff --git a/apps/web/src/lib/server/services/files.ts b/apps/web/src/lib/server/services/files.ts index d742248..fecefd9 100644 --- a/apps/web/src/lib/server/services/files.ts +++ b/apps/web/src/lib/server/services/files.ts @@ -1,8 +1,7 @@ import { Context, Effect, Layer } from 'effect'; -import { SqlClient } from 'effect/unstable/sql'; import { AppConfig } from '../config'; +import { PgSql } from '../pg'; import { Blobs } from './blobs'; -import { Db } from './bindings'; import { Tags } from './tags'; import { createInternals } from './files/internals'; import { mutationOps } from './files/mutations'; @@ -24,12 +23,11 @@ export type { export class Files extends Context.Service()('app/Files') {} const makeFiles = Effect.gen(function* () { - const db = yield* Db; + const sql = yield* PgSql; const blobs = yield* Blobs; - const sql = (yield* SqlClient.SqlClient).withoutTransforms(); const config = yield* AppConfig; const tags = yield* Tags; - const internals = createInternals({ db, blobs, sql, config, tags }); + const internals = createInternals({ sql, blobs, config, tags }); return Files.of({ ...uploadOps(internals), diff --git a/apps/web/src/lib/server/services/files/internals.ts b/apps/web/src/lib/server/services/files/internals.ts index cde19de..e7e2c3d 100644 --- a/apps/web/src/lib/server/services/files/internals.ts +++ b/apps/web/src/lib/server/services/files/internals.ts @@ -1,9 +1,9 @@ import type { DashboardFile } from '@adrive/shared'; +import type { PgClient } from '@effect/sql-pg'; import { Effect } from 'effect'; -import { SqlClient } from 'effect/unstable/sql'; import { compensateBlobFailure, - deferredBlobDeleteCommand + queueDeferredBlobDelete } from '../../blob-compensation'; import { NotFound, StorageError } from '../../errors'; import { @@ -12,27 +12,22 @@ import { toDashboardFile } from '../../file-rows'; import { visibilityForFile } from '../../file-policy'; -import type { PurgeSqlCommand } from '../../purge-sql'; -import { fileIndexStatements } from '../../search-index'; -import { ensureStorageQuota } from '../../storage-quota'; +import { refreshSearchDocument } from '../../search-index'; +import { ensureStoredBytesWithin } from '../../storage-quota'; import type { AppConfig } from '../../config'; import type { Blobs } from '../blobs'; -import type { Db } from '../bindings'; import type { Tags } from '../tags'; import type { MutationResult } from './types'; export interface CoreDeps { - readonly db: Db['Service']; + readonly sql: PgClient.PgClient; readonly blobs: Blobs['Service']; - readonly sql: (typeof SqlClient.SqlClient)['Service']; readonly config: AppConfig['Service']; readonly tags: Tags['Service']; } export const createInternals = (deps: CoreDeps) => { - const { db, blobs, sql, config, tags } = deps; - const preparePurgeCommand = (command: PurgeSqlCommand) => - db.prepare(command.sql).bind(...command.bindings); + const { sql, blobs, config, tags } = deps; const compensateStoredBlob = ( failure: OriginalError, fileId: string, @@ -43,27 +38,20 @@ export const createInternals = (deps: CoreDeps) => { compensateBlobFailure( failure, blobs.delete(r2Key), - (deleteCause) => { - const command = deferredBlobDeleteCommand( + (deleteCause) => + queueDeferredBlobDelete( + sql, r2Key, fileId, version, new Date().toISOString(), String(deleteCause) - ); - return Effect.tryPromise({ - try: () => - db - .prepare(command.sql) - .bind(...command.bindings) - .run(), - catch: (cause) => - new StorageError({ - operation: 'queue orphaned file blob', - cause - }) - }); - }, + ).pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'queue orphaned file blob', cause }) + ) + ), (deleteCause, queueCause) => { console.error( JSON.stringify({ @@ -79,25 +67,20 @@ export const createInternals = (deps: CoreDeps) => { // One aggregate query per upload; at personal scale this stays cheap // and cannot drift the way a maintained counter can. const checkStorageQuota = (incomingBytes: number) => - ensureStorageQuota(db, config.maxTotalBytes, incomingBytes); + ensureStoredBytesWithin(sql, config.maxTotalBytes, incomingBytes); const findDashboardFile = Effect.fn('Files.findDashboardFile')(function* ( id: string ) { - const rows = yield* sql - .unsafe( - `SELECT ${dashboardFileColumns} - FROM files f - WHERE f.id = ? - LIMIT 1`, - [id] + const rows = yield* sql` + SELECT ${sql.literal(dashboardFileColumns)} + FROM files f + WHERE f.id = ${id} + LIMIT 1`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'find dashboard file', cause }) ) - .pipe( - Effect.mapError( - (cause) => - new StorageError({ operation: 'find dashboard file', cause }) - ) - ); + ); const row = decodeDashboardRows(rows)[0]; if (!row) return yield* new NotFound({ id }); return toDashboardFile(row); @@ -116,60 +99,45 @@ export const createInternals = (deps: CoreDeps) => { current.htmlForcedPublic ? 'text/html' : contentType, current.public ); - const statements = [ - db - .prepare( - `UPDATE files - SET current_version = ?, size_bytes = ?, content_type = ?, - public = ?, updated_at = ?, index_state = 'pending', + // Optimistic concurrency on current_version: a concurrent upload + // that committed first makes this update match nothing. + yield* sql + .withTransaction( + Effect.gen(function* () { + const updated = yield* sql<{ id: string }>` + UPDATE files + SET current_version = ${version}, size_bytes = ${size}, + content_type = ${contentType}, public = ${visibility.public}, + updated_at = ${updatedAt}, index_state = 'pending', index_cursor = 0, index_attempts = 0, index_error = NULL, index_next_run_at = NULL, index_lease_token = NULL - WHERE id = ? AND current_version = ? AND deleted_at IS NULL` - ) - .bind( - version, - size, - contentType, - visibility.public ? 1 : 0, - updatedAt, - current.id, - current.version - ), - db - .prepare( - `INSERT INTO file_versions ( - file_id, version, r2_key, size_bytes, sha256, content_type, created_at, - text_content + WHERE id = ${current.id} AND current_version = ${current.version} + AND deleted_at IS NULL + RETURNING id`; + if (updated.length !== 1) { + return yield* new StorageError({ + operation: 'commit file version', + cause: 'File changed while the version was uploading' + }); + } + yield* sql` + INSERT INTO file_versions ( + file_id, version, r2_key, size_bytes, sha256, content_type, + created_at, text_content + ) VALUES ( + ${current.id}, ${version}, ${r2Key}, ${size}, NULL, ${contentType}, + ${updatedAt}, NULL + )`; + yield* refreshSearchDocument(sql, current.id); + }) + ) + .pipe( + Effect.catchTag('SqlError', (cause) => + Effect.fail( + new StorageError({ operation: 'commit file version', cause }) ) - SELECT ?, ?, ?, ?, NULL, ?, ?, ? - WHERE EXISTS ( - SELECT 1 FROM files - WHERE id = ? AND current_version = ? AND deleted_at IS NULL - )` ) - .bind( - current.id, - version, - r2Key, - size, - contentType, - updatedAt, - null, - current.id, - version - ), - ...fileIndexStatements(db, current.id) - ]; - yield* Effect.tryPromise({ - try: async () => { - const results = await db.batch(statements); - if (results[0]?.meta.changes !== 1 || results[1]?.meta.changes !== 1) { - throw new Error('File changed while the version was uploading'); - } - }, - catch: (cause) => - new StorageError({ operation: 'commit file version', cause }) - }); + ); return { file: { ...current, @@ -189,12 +157,10 @@ export const createInternals = (deps: CoreDeps) => { }); return { - db, - blobs, sql, + blobs, config, tags, - preparePurgeCommand, compensateStoredBlob, checkStorageQuota, findDashboardFile, diff --git a/apps/web/src/lib/server/services/files/mutations.ts b/apps/web/src/lib/server/services/files/mutations.ts index c846331..467944b 100644 --- a/apps/web/src/lib/server/services/files/mutations.ts +++ b/apps/web/src/lib/server/services/files/mutations.ts @@ -5,10 +5,12 @@ import { trashWindow, visibilityForFile } from '../../file-policy'; -import { fileIndexStatements } from '../../search-index'; +import { refreshSearchDocument } from '../../search-index'; import type { FileInternals } from './internals'; import type { FilesShape } from './types'; +const EPOCH = '1970-01-01T00:00:00.000Z'; + export const mutationOps = ( internals: FileInternals ): Pick< @@ -22,7 +24,7 @@ export const mutationOps = ( | 'scheduleAllPurgesNow' | 'recordDownload' > => { - const { db, sql } = internals; + const { sql } = internals; const { findDashboardFile } = internals; return { setVisibility: Effect.fn('Files.setVisibility')(function* (id, isPublic) { @@ -41,7 +43,7 @@ export const mutationOps = ( const updatedAt = new Date().toISOString(); yield* sql` UPDATE files - SET public = ${visibility.public ? 1 : 0}, updated_at = ${updatedAt} + SET public = ${visibility.public}, updated_at = ${updatedAt} WHERE id = ${id} `.pipe( Effect.mapError( @@ -61,20 +63,18 @@ export const mutationOps = ( trash: Effect.fn('Files.trash')(function* (id) { const current = yield* findDashboardFile(id); const { deletedAt, purgeAt } = trashWindow(current.deletedAt, new Date()); - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE files - SET deleted_at = ?, purge_at = ?, purge_state = 'none', - purge_error = NULL, purge_next_run_at = NULL, updated_at = ? - WHERE id = ? AND purge_state <> 'pending'` - ) - .bind(deletedAt, purgeAt, deletedAt, id) - .run(), - catch: (cause) => new StorageError({ operation: 'trash file', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql<{ id: string }>` + UPDATE files + SET deleted_at = ${deletedAt}, purge_at = ${purgeAt}, purge_state = 'none', + purge_error = NULL, purge_next_run_at = NULL, updated_at = ${deletedAt} + WHERE id = ${id} AND purge_state <> 'pending' + RETURNING id + `.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'trash file', cause }) + ) + ); + if (rows.length !== 1) { return yield* new NotFound({ id }); } return { @@ -85,21 +85,19 @@ export const mutationOps = ( restore: Effect.fn('Files.restore')(function* (id) { const current = yield* findDashboardFile(id); const updatedAt = new Date().toISOString(); - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE files - SET deleted_at = NULL, purge_at = NULL, purge_state = 'none', - purge_error = NULL, purge_next_run_at = NULL, - updated_at = ? - WHERE id = ? AND purge_state <> 'pending'` - ) - .bind(updatedAt, id) - .run(), - catch: (cause) => new StorageError({ operation: 'restore file', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql<{ id: string }>` + UPDATE files + SET deleted_at = NULL, purge_at = NULL, purge_state = 'none', + purge_error = NULL, purge_next_run_at = NULL, + updated_at = ${updatedAt} + WHERE id = ${id} AND purge_state <> 'pending' + RETURNING id + `.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'restore file', cause }) + ) + ); + if (rows.length !== 1) { return yield* new InvalidRequest({ status: 409, message: 'This file is already being purged' @@ -146,23 +144,22 @@ export const mutationOps = ( current.public ); const updatedAt = new Date().toISOString(); - yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `UPDATE files - SET display_name = ?, public = ?, updated_at = ?, - index_state = 'pending', index_cursor = 0, - index_attempts = 0, index_error = NULL, - index_next_run_at = NULL, index_lease_token = NULL - WHERE id = ?` - ) - .bind(displayName, visibility.public ? 1 : 0, updatedAt, id), - ...fileIndexStatements(db, id) - ]), - catch: (cause) => new StorageError({ operation: 'rename file', cause }) - }); + yield* sql + .withTransaction( + sql` + UPDATE files + SET display_name = ${displayName}, public = ${visibility.public}, + updated_at = ${updatedAt}, index_state = 'pending', + index_cursor = 0, index_attempts = 0, index_error = NULL, + index_next_run_at = NULL, index_lease_token = NULL + WHERE id = ${id} + `.pipe(Effect.andThen(refreshSearchDocument(sql, id))) + ) + .pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'rename file', cause }) + ) + ); return { file: { ...current, @@ -186,22 +183,20 @@ export const mutationOps = ( message: 'Move the file to trash before deleting it permanently' }); } - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE files - SET purge_at = ?, purge_state = 'none', purge_error = NULL, - purge_next_run_at = NULL - WHERE id = ? AND deleted_at IS NOT NULL - AND purge_state <> 'pending'` - ) - .bind('1970-01-01T00:00:00.000Z', id) - .run(), - catch: (cause) => - new StorageError({ operation: 'schedule immediate purge', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql<{ id: string }>` + UPDATE files + SET purge_at = ${EPOCH}, purge_state = 'none', purge_error = NULL, + purge_next_run_at = NULL + WHERE id = ${id} AND deleted_at IS NOT NULL + AND purge_state <> 'pending' + RETURNING id + `.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'schedule immediate purge', cause }) + ) + ); + if (rows.length !== 1) { return yield* new InvalidRequest({ status: 409, message: 'This file is already being purged' @@ -209,22 +204,20 @@ export const mutationOps = ( } return { file: current, forcedPublic: false }; }), - scheduleAllPurgesNow: Effect.tryPromise({ - try: async () => { - const result = await db - .prepare( - `UPDATE files - SET purge_at = ?, purge_state = 'none', purge_error = NULL, - purge_next_run_at = NULL - WHERE deleted_at IS NOT NULL AND purge_state <> 'pending'` - ) - .bind('1970-01-01T00:00:00.000Z') - .run(); - return result.meta.changes; - }, - catch: (cause) => - new StorageError({ operation: 'schedule empty trash', cause }) - }).pipe(Effect.withSpan('Files.scheduleAllPurgesNow')), + scheduleAllPurgesNow: sql<{ id: string }>` + UPDATE files + SET purge_at = ${EPOCH}, purge_state = 'none', purge_error = NULL, + purge_next_run_at = NULL + WHERE deleted_at IS NOT NULL AND purge_state <> 'pending' + RETURNING id + `.pipe( + Effect.map((rows) => rows.length), + Effect.mapError( + (cause) => + new StorageError({ operation: 'schedule empty trash', cause }) + ), + Effect.withSpan('Files.scheduleAllPurgesNow') + ), recordDownload: Effect.fn('Files.recordDownload')(function* (id) { const now = new Date().toISOString(); yield* sql` diff --git a/apps/web/src/lib/server/services/files/purge.ts b/apps/web/src/lib/server/services/files/purge.ts index ec02338..32c9ae7 100644 --- a/apps/web/src/lib/server/services/files/purge.ts +++ b/apps/web/src/lib/server/services/files/purge.ts @@ -1,6 +1,6 @@ import { dashboardThumbnailPrefix } from '../../../file-thumbnail'; import { StorageError } from '../../errors'; -import { purgeCompletionCommands } from '../../purge-sql'; +import { completePurge } from '../../purge-sql'; import { retryAt, safeIndexError } from '../../semantic-policy'; import { Effect } from 'effect'; import { forgetTagListCache } from '../tags'; @@ -10,172 +10,122 @@ import type { FilesShape } from './types'; export const purgeOps = ( internals: FileInternals ): Pick => { - const { db, blobs } = internals; - const { preparePurgeCommand } = internals; + const { sql, blobs } = internals; return { sweepPurges: Effect.fn('Files.sweepPurges')(function* (limit) { const bounded = Math.max(1, Math.min(limit, 10)); const now = new Date().toISOString(); - const due = yield* Effect.tryPromise({ - try: async () => { - const result = await db - .prepare( - `SELECT id - FROM files - WHERE ( - ( - (expires_at IS NOT NULL AND expires_at <= ?) - OR (deleted_at IS NOT NULL AND purge_at IS NOT NULL AND purge_at <= ?) - ) - AND purge_state IN ('none', 'failed') - AND (purge_next_run_at IS NULL OR purge_next_run_at <= ?) - ) OR ( - purge_state = 'pending' - AND (purge_next_run_at IS NULL OR purge_next_run_at <= ?) - ) - ORDER BY COALESCE(purge_at, expires_at), id - LIMIT ?` - ) - .bind(now, now, now, now, bounded) - .all<{ id: string }>(); - if (!result.success) - throw new Error(result.error ?? 'Purge listing failed'); - return result.results; - }, - catch: (cause) => - new StorageError({ operation: 'list files due for purge', cause }) - }); + const due = yield* sql<{ id: string }>` + SELECT id + FROM files + WHERE ( + ( + (expires_at IS NOT NULL AND expires_at <= ${now}) + OR (deleted_at IS NOT NULL AND purge_at IS NOT NULL AND purge_at <= ${now}) + ) + AND purge_state IN ('none', 'failed') + AND (purge_next_run_at IS NULL OR purge_next_run_at <= ${now}) + ) OR ( + purge_state = 'pending' + AND (purge_next_run_at IS NULL OR purge_next_run_at <= ${now}) + ) + ORDER BY COALESCE(purge_at, expires_at), id + LIMIT ${bounded} + `.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'list files due for purge', cause }) + ) + ); for (const row of due) { - const claimed = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE files - SET purge_state = 'pending', - deleted_at = COALESCE(deleted_at, ?), - purge_at = COALESCE(purge_at, ?), - purge_attempts = purge_attempts + 1, - purge_error = NULL, purge_next_run_at = ? - WHERE id = ? AND ( - ( - purge_state IN ('none', 'failed') - AND ( - (expires_at IS NOT NULL AND expires_at <= ?) - OR (deleted_at IS NOT NULL AND purge_at IS NOT NULL AND purge_at <= ?) - ) - ) - OR ( - purge_state = 'pending' - AND (purge_next_run_at IS NULL OR purge_next_run_at <= ?) - ) - )` - ) - .bind( - now, - now, - new Date(Date.now() + 5 * 60 * 1_000).toISOString(), - row.id, - now, - now, - now + const leaseUntil = new Date(Date.now() + 5 * 60 * 1_000).toISOString(); + const claimed = yield* sql<{ id: string }>` + UPDATE files + SET purge_state = 'pending', + deleted_at = COALESCE(deleted_at, ${now}), + purge_at = COALESCE(purge_at, ${now}), + purge_attempts = purge_attempts + 1, + purge_error = NULL, purge_next_run_at = ${leaseUntil} + WHERE id = ${row.id} AND ( + ( + purge_state IN ('none', 'failed') + AND ( + (expires_at IS NOT NULL AND expires_at <= ${now}) + OR (deleted_at IS NOT NULL AND purge_at IS NOT NULL AND purge_at <= ${now}) ) - .run(), - catch: (cause) => - new StorageError({ operation: 'claim file purge', cause }) - }); - if (claimed.meta.changes !== 1) continue; + ) + OR ( + purge_state = 'pending' + AND (purge_next_run_at IS NULL OR purge_next_run_at <= ${now}) + ) + ) + RETURNING id + `.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'claim file purge', cause }) + ) + ); + if (claimed.length !== 1) continue; - const deletion = yield* Effect.tryPromise({ - try: async () => { - const [versions, assets] = await Promise.all([ - db - .prepare( - `SELECT version, r2_key - FROM file_versions - WHERE file_id = ?` - ) - .bind(row.id) - .all<{ version: number; r2_key: string }>(), - db - .prepare('SELECT r2_key FROM site_assets WHERE file_id = ?') - .bind(row.id) - .all<{ r2_key: string }>() - ]); - if (!versions.success || !assets.success) { - throw new Error( - versions.error ?? assets.error ?? 'Purge key listing failed' - ); - } - return { - keys: [ - ...versions.results - .filter((item) => !item.r2_key.startsWith('site-version/')) - .map((item) => item.r2_key), - ...assets.results.map((item) => item.r2_key) - ], - thumbnailPrefixes: versions.results.map((item) => - dashboardThumbnailPrefix(row.id, item.version) - ) - }; - }, - catch: (cause) => - new StorageError({ operation: 'list file purge keys', cause }) - }); + const deletion = yield* Effect.all( + [ + sql<{ version: number; r2_key: string }>` + SELECT version, r2_key FROM file_versions WHERE file_id = ${row.id}`, + sql<{ r2_key: string }>` + SELECT r2_key FROM site_assets WHERE file_id = ${row.id}` + ], + { concurrency: 'unbounded' } + ).pipe( + Effect.map(([versions, assets]) => ({ + keys: [ + ...versions + .filter((item) => !item.r2_key.startsWith('site-version/')) + .map((item) => item.r2_key), + ...assets.map((item) => item.r2_key) + ], + thumbnailPrefixes: versions.map((item) => + dashboardThumbnailPrefix(row.id, item.version) + ) + })), + Effect.mapError( + (cause) => + new StorageError({ operation: 'list file purge keys', cause }) + ) + ); const deleted = yield* blobs.deleteMany(deletion.keys).pipe( Effect.andThen(blobs.deletePrefixes(deletion.thumbnailPrefixes)), Effect.as(true), Effect.catch((failure) => - Effect.tryPromise({ - try: async () => { - const attempts = - ( - await db - .prepare('SELECT purge_attempts FROM files WHERE id = ?') - .bind(row.id) - .first<{ purge_attempts: number }>() - )?.purge_attempts ?? 1; - await db - .prepare( - `UPDATE files - SET purge_state = 'failed', purge_error = ?, - purge_next_run_at = ? - WHERE id = ? AND purge_state = 'pending'` - ) - .bind( - safeIndexError(failure.cause), - retryAt(attempts), - row.id - ) - .run(); - }, - catch: (cause) => - new StorageError({ - operation: 'record file purge failure', - cause - }) - }).pipe(Effect.as(false)) + Effect.gen(function* () { + const attempts = yield* sql<{ purge_attempts: number }>` + SELECT purge_attempts FROM files WHERE id = ${row.id}`.pipe( + Effect.map((rows) => rows[0]?.purge_attempts ?? 1) + ); + yield* sql` + UPDATE files + SET purge_state = 'failed', + purge_error = ${safeIndexError(failure.cause)}, + purge_next_run_at = ${retryAt(attempts)} + WHERE id = ${row.id} AND purge_state = 'pending'`; + }).pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'record file purge failure', + cause + }) + ), + Effect.as(false) + ) ) ); if (!deleted) continue; - yield* Effect.tryPromise({ - try: async () => { - const results = await db.batch( - purgeCompletionCommands(row.id, now).map(preparePurgeCommand) - ); - if ( - results[3]?.meta.changes !== 1 || - results[4]?.meta.changes !== 1 - ) { - throw new Error('File purge state changed before completion'); - } - }, - catch: (cause) => - new StorageError({ operation: 'finish file purge', cause }) - }); - forgetTagListCache(db); + yield* completePurge(sql, row.id); + forgetTagListCache(); } return due.length; }) diff --git a/apps/web/src/lib/server/services/files/queries.ts b/apps/web/src/lib/server/services/files/queries.ts index 194b84f..d912311 100644 --- a/apps/web/src/lib/server/services/files/queries.ts +++ b/apps/web/src/lib/server/services/files/queries.ts @@ -29,31 +29,32 @@ export const queryOps = ( ? cause : new InvalidRequest({ status: 400, message: 'Cursor is invalid' }) }); - const sortColumn = trashed ? 'f.deleted_at' : 'f.updated_at'; + const sortColumn = sql.literal(trashed ? 'f.deleted_at' : 'f.updated_at'); // Keyset pagination over (sort column, id); one extra row tells us // whether another page exists without a second query. - const rows = yield* sql - .unsafe( - `SELECT ${dashboardFileColumns} - FROM files f - WHERE f.deleted_at IS ${trashed ? 'NOT NULL' : 'NULL'} - ${trashed ? 'AND (f.purge_at IS NULL OR f.purge_at > ?)' : 'AND (f.expires_at IS NULL OR f.expires_at > ?)'} - ${cursor ? `AND (${sortColumn} < ? OR (${sortColumn} = ? AND f.id > ?))` : ''} - ORDER BY ${sortColumn} DESC, f.id - LIMIT ?`, - cursor - ? [now, cursor.k, cursor.k, cursor.id, limit + 1] - : [now, limit + 1] + const rows = yield* sql` + SELECT ${sql.literal(dashboardFileColumns)} + FROM files f + WHERE ${sql.and([ + trashed + ? sql`f.deleted_at IS NOT NULL AND (f.purge_at IS NULL OR f.purge_at > ${now})` + : sql`f.deleted_at IS NULL AND (f.expires_at IS NULL OR f.expires_at > ${now})`, + ...(cursor + ? [ + sql`(${sortColumn} < ${cursor.k} OR (${sortColumn} = ${cursor.k} AND f.id > ${cursor.id}))` + ] + : []) + ])} + ORDER BY ${sortColumn} DESC, f.id + LIMIT ${limit + 1}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: trashed ? 'list trashed files' : 'list active files', + cause + }) ) - .pipe( - Effect.mapError( - (cause) => - new StorageError({ - operation: trashed ? 'list trashed files' : 'list active files', - cause - }) - ) - ); + ); const decoded = decodeDashboardRows(rows); const pageRows = decoded.slice(0, limit); const files = pageRows.map(toDashboardFile); @@ -84,23 +85,20 @@ export const queryOps = ( message: 'Cursor is invalid' }); } - const rows = yield* sql - .unsafe( - `SELECT version, size_bytes, content_type, created_at - FROM file_versions - WHERE file_id = ?${beforeVersion !== null ? ' AND version < ?' : ''} - ORDER BY version DESC - LIMIT ?`, - beforeVersion !== null - ? [id, beforeVersion, limit + 1] - : [id, limit + 1] + const rows = yield* sql` + SELECT version, size_bytes, content_type, created_at + FROM file_versions + WHERE ${sql.and([ + sql`file_id = ${id}`, + ...(beforeVersion !== null ? [sql`version < ${beforeVersion}`] : []) + ])} + ORDER BY version DESC + LIMIT ${limit + 1}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'list file versions', cause }) ) - .pipe( - Effect.mapError( - (cause) => - new StorageError({ operation: 'list file versions', cause }) - ) - ); + ); const decoded = decodeVersionRows(rows); const pageRows = decoded.slice(0, limit); const lastVersion = pageRows[pageRows.length - 1]; @@ -131,27 +129,26 @@ export const queryOps = ( message: 'Version is invalid' }); } + const now = new Date().toISOString(); + const available = sql`( + ${includeUnavailable}::boolean + OR ( + f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${now}) + ) + )`; + const siteFilter = sql`(${includeSites}::boolean OR f.is_site = false)`; const rows = version === undefined ? yield* sql` SELECT f.id, f.display_name, v.content_type, v.version, v.size_bytes, - f.public AS is_public, f.is_site, v.r2_key, v.thumbnail_r2_key, v.created_at + f.public AS is_public, f.is_site, v.r2_key, v.thumbnail_r2_key, + v.created_at FROM files f JOIN file_versions v ON v.file_id = f.id AND v.version = f.current_version - WHERE f.id = ${id} - AND ( - ${includeUnavailable ? 1 : 0} = 1 - OR ( - f.deleted_at IS NULL - AND ( - f.expires_at IS NULL - OR f.expires_at > ${new Date().toISOString()} - ) - ) - ) - AND (${includeSites ? 1 : 0} = 1 OR f.is_site = 0) + WHERE f.id = ${id} AND ${available} AND ${siteFilter} LIMIT 1 `.pipe( Effect.mapError( @@ -161,22 +158,13 @@ export const queryOps = ( : yield* sql` SELECT f.id, f.display_name, v.content_type, v.version, v.size_bytes, - f.public AS is_public, f.is_site, v.r2_key, v.thumbnail_r2_key, v.created_at + f.public AS is_public, f.is_site, v.r2_key, v.thumbnail_r2_key, + v.created_at FROM files f JOIN file_versions v ON v.file_id = f.id WHERE f.id = ${id} AND v.version = ${version} - AND ( - ${includeUnavailable ? 1 : 0} = 1 - OR ( - f.deleted_at IS NULL - AND ( - f.expires_at IS NULL - OR f.expires_at > ${new Date().toISOString()} - ) - ) - ) - AND (${includeSites ? 1 : 0} = 1 OR f.is_site = 0) - AND (f.is_site = 0 OR v.version = f.current_version) + AND ${available} AND ${siteFilter} + AND (f.is_site = false OR v.version = f.current_version) LIMIT 1 `.pipe( Effect.mapError( @@ -191,10 +179,10 @@ export const queryOps = ( id: row.id, displayName: row.display_name, contentType: row.content_type, - kind: row.is_site === 1 ? 'site' : 'file', + kind: row.is_site ? 'site' : 'file', version: row.version, sizeBytes: row.size_bytes, - public: row.is_public === 1, + public: row.is_public, createdAt: row.created_at, expiresAt: null, downloadCount: 0, diff --git a/apps/web/src/lib/server/services/files/thumbnails.ts b/apps/web/src/lib/server/services/files/thumbnails.ts index 328444b..e9337e6 100644 --- a/apps/web/src/lib/server/services/files/thumbnails.ts +++ b/apps/web/src/lib/server/services/files/thumbnails.ts @@ -1,9 +1,9 @@ import { dashboardThumbnailPrefix } from '../../../file-thumbnail'; -import { InvalidRequest, NotFound, StorageError } from '../../errors'; +import { NotFound, StorageError } from '../../errors'; import { - commitThumbnailStorageCommand, + commitThumbnailStorage, thumbnailQuotaDelta, - thumbnailStorageStateCommand + thumbnailStorageState } from '../../thumbnail-storage'; import { Effect } from 'effect'; import type { FileInternals } from './internals'; @@ -12,27 +12,20 @@ import type { FilesShape } from './types'; export const thumbnailOps = ( internals: FileInternals ): Pick => { - const { db, blobs } = internals; + const { sql, blobs } = internals; const { checkStorageQuota, compensateStoredBlob } = internals; return { storeDashboardThumbnail: Effect.fn('Files.storeDashboardThumbnail')( function* (id, version, body, size, expectedR2Key) { - const stateCommand = thumbnailStorageStateCommand(id, version); - const state = yield* Effect.tryPromise({ - try: () => - db - .prepare(stateCommand.sql) - .bind(...stateCommand.bindings) - .first<{ - thumbnail_r2_key: string | null; - thumbnail_size_bytes: number; - }>(), - catch: (cause) => - new StorageError({ - operation: 'find dashboard thumbnail state', - cause - }) - }); + const state = yield* thumbnailStorageState(sql, id, version).pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'find dashboard thumbnail state', + cause + }) + ) + ); if (state === null) return yield* new NotFound({ id }); if (state.thumbnail_r2_key !== expectedR2Key) { if (state.thumbnail_r2_key === null) { @@ -46,25 +39,22 @@ export const thumbnailOps = ( ); const r2Key = `${dashboardThumbnailPrefix(id, version)}${crypto.randomUUID()}.webp`; const stored = yield* blobs.put(r2Key, body, size, 'image/webp'); - const commitCommand = commitThumbnailStorageCommand( + const commit = commitThumbnailStorage( + sql, id, version, r2Key, stored.size, expectedR2Key + ).pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'record dashboard thumbnail', + cause + }) + ) ); - const commit = Effect.tryPromise({ - try: () => - db - .prepare(commitCommand.sql) - .bind(...commitCommand.bindings) - .run(), - catch: (cause) => - new StorageError({ - operation: 'record dashboard thumbnail', - cause - }) - }); const committed = yield* commit.pipe( Effect.catch((failure) => compensateStoredBlob( @@ -76,7 +66,7 @@ export const thumbnailOps = ( ) ) ); - if (committed.meta.changes !== 1) { + if (!committed) { yield* compensateStoredBlob( new NotFound({ id }), id, @@ -84,19 +74,15 @@ export const thumbnailOps = ( r2Key, 'dashboard thumbnail' ).pipe(Effect.catchTag('NotFound', () => Effect.void)); - const winnerCommand = thumbnailStorageStateCommand(id, version); - const winner = yield* Effect.tryPromise({ - try: () => - db - .prepare(winnerCommand.sql) - .bind(...winnerCommand.bindings) - .first<{ thumbnail_r2_key: string | null }>(), - catch: (cause) => - new StorageError({ - operation: 'find committed dashboard thumbnail', - cause - }) - }); + const winner = yield* thumbnailStorageState(sql, id, version).pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'find committed dashboard thumbnail', + cause + }) + ) + ); if (winner === null || winner.thumbnail_r2_key === null) { return yield* new NotFound({ id }); } diff --git a/apps/web/src/lib/server/services/files/types.ts b/apps/web/src/lib/server/services/files/types.ts index 37f5ed9..a35898d 100644 --- a/apps/web/src/lib/server/services/files/types.ts +++ b/apps/web/src/lib/server/services/files/types.ts @@ -10,8 +10,8 @@ const FileContentRow = Schema.Struct({ content_type: Schema.String, version: Schema.Int, size_bytes: Schema.Int, - is_public: Schema.Int, - is_site: Schema.Int, + is_public: Schema.Boolean, + is_site: Schema.Boolean, r2_key: Schema.String, thumbnail_r2_key: Schema.NullOr(Schema.String), created_at: Schema.String diff --git a/apps/web/src/lib/server/services/files/upload.ts b/apps/web/src/lib/server/services/files/upload.ts index fadd1e6..a7a2f2d 100644 --- a/apps/web/src/lib/server/services/files/upload.ts +++ b/apps/web/src/lib/server/services/files/upload.ts @@ -4,7 +4,7 @@ import { visibilityForFile } from '../../file-policy'; import { InvalidRequest, NotFound, StorageError } from '../../errors'; -import { fileIndexStatements } from '../../search-index'; +import { refreshSearchDocument } from '../../search-index'; import { validateUploadLength } from '../../upload-stream'; import { forgetTagListCache } from '../tags'; import { Effect } from 'effect'; @@ -15,7 +15,7 @@ import { decodeContentRows } from './types'; export const uploadOps = ( internals: FileInternals ): Pick => { - const { db, blobs, sql, config, tags } = internals; + const { blobs, sql, config, tags } = internals; const { checkStorageQuota, compensateStoredBlob, @@ -57,51 +57,47 @@ export const uploadOps = ( const r2Key = `v/${id}/${crypto.randomUUID()}`; const createdAt = new Date().toISOString(); const stored = yield* blobs.put(r2Key, input.body, size, contentType); - const statements = [ - db - .prepare( - `INSERT INTO files ( - id, display_name, content_type, kind, current_version, size_bytes, - public, is_site, created_at, updated_at, expires_at, index_state - ) VALUES (?, ?, ?, 'file', 1, ?, ?, 0, ?, ?, ?, 'pending')` - ) - .bind( - id, - displayName, - contentType, - stored.size, - visibility.public ? 1 : 0, - createdAt, - createdAt, - input.expiresAt - ), - db - .prepare( - `INSERT INTO file_versions ( - file_id, version, r2_key, size_bytes, sha256, content_type, created_at, - text_content - ) VALUES (?, 1, ?, ?, NULL, ?, ?, ?)` + const commit = sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO files ( + id, display_name, content_type, kind, current_version, size_bytes, + public, is_site, created_at, updated_at, expires_at, index_state + ) VALUES ( + ${id}, ${displayName}, ${contentType}, 'file', 1, ${stored.size}, + ${visibility.public}, false, ${createdAt}, ${createdAt}, + ${input.expiresAt}, 'pending' + )`; + yield* sql` + INSERT INTO file_versions ( + file_id, version, r2_key, size_bytes, sha256, content_type, + created_at, text_content + ) VALUES ( + ${id}, 1, ${r2Key}, ${stored.size}, NULL, ${contentType}, + ${createdAt}, NULL + )`; + if (resolvedTags.length > 0) { + yield* sql` + INSERT INTO file_tags ${sql.insert( + resolvedTags.map((tag) => ({ file_id: id, tag_id: tag.id })) + )}`; + } + yield* refreshSearchDocument(sql, id); + }) + ) + .pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'commit file metadata', cause }) ) - .bind(id, r2Key, stored.size, contentType, createdAt, null), - ...resolvedTags.map((tag) => - db - .prepare('INSERT INTO file_tags (file_id, tag_id) VALUES (?, ?)') - .bind(id, tag.id) - ), - ...fileIndexStatements(db, id) - ]; - - const commit = Effect.tryPromise({ - try: () => db.batch(statements), - catch: (cause) => - new StorageError({ operation: 'commit file metadata', cause }) - }); + ); yield* commit.pipe( Effect.catch((failure) => compensateStoredBlob(failure, id, 1, r2Key, 'upload') ) ); - forgetTagListCache(db); + forgetTagListCache(); return { file: { @@ -198,7 +194,7 @@ export const uploadOps = ( FROM files f JOIN file_versions v ON v.file_id = f.id WHERE f.id = ${id} AND v.version = ${version} - AND f.deleted_at IS NULL AND f.is_site = 0 + AND f.deleted_at IS NULL AND f.is_site = false LIMIT 1 `.pipe( Effect.mapError( diff --git a/apps/web/src/lib/server/services/search.ts b/apps/web/src/lib/server/services/search.ts index a2fd622..c243fee 100644 --- a/apps/web/src/lib/server/services/search.ts +++ b/apps/web/src/lib/server/services/search.ts @@ -19,6 +19,7 @@ import { SEARCH_CANDIDATE_LIMIT, type SearchCommand } from '../search-candidates'; +import { PgSql } from '../pg'; import { Db } from './bindings'; import { Embedder, VectorIndex } from './semantic'; @@ -64,8 +65,19 @@ const decodeCursor = (cursor: string | null | undefined) => { const makeSearch = Effect.gen(function* () { const db = yield* Db; + // Files rows live in Postgres now; the FTS and trigram indexes still + // live in D1 until the keyword search rewrite moves them over. + const sql = yield* PgSql; const embedder = yield* Embedder; const vectorIndex = yield* VectorIndex; + const selectedTagFilter = (tagIds: ReadonlyArray) => + tagIds.length === 0 + ? sql`` + : sql`AND EXISTS ( + SELECT 1 FROM file_tags selected + WHERE selected.file_id = f.id + AND ${sql.in('selected.tag_id', tagIds)} + )`; const runRanked = ( command: SearchCommand @@ -112,35 +124,18 @@ const makeSearch = Effect.gen(function* () { tagIds: ReadonlyArray ) { if (fileIds.length === 0) return []; - const idPlaceholders = fileIds.map(() => '?').join(', '); - const tagFilter = - tagIds.length === 0 - ? '' - : `AND EXISTS ( - SELECT 1 FROM file_tags selected - WHERE selected.file_id = f.id - AND selected.tag_id IN (${tagIds.map(() => '?').join(', ')}) - )`; - const result = yield* Effect.tryPromise({ - try: async () => { - const response = await db - .prepare( - `SELECT ${dashboardFileColumns} - FROM files f - WHERE f.id IN (${idPlaceholders}) - AND f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - ${tagFilter}` - ) - .bind(...fileIds, new Date().toISOString(), ...tagIds) - .all(); - if (!response.success) - throw new Error(response.error ?? 'Hydration failed'); - return response.results; - }, - catch: (cause) => - new StorageError({ operation: 'hydrate search results', cause }) - }); + const result = yield* sql` + SELECT ${sql.literal(dashboardFileColumns)} + FROM files f + WHERE ${sql.in('f.id', fileIds)} + AND f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) + ${selectedTagFilter(tagIds)}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'hydrate search results', cause }) + ) + ); const byId = new Map( decodeDashboardRows(result) .map(toDashboardFile) @@ -157,35 +152,18 @@ const makeSearch = Effect.gen(function* () { limit: number, offset: number ) { - const tagFilter = - tagIds.length === 0 - ? '' - : `AND EXISTS ( - SELECT 1 FROM file_tags selected - WHERE selected.file_id = f.id - AND selected.tag_id IN (${tagIds.map(() => '?').join(', ')}) - )`; - const rows = yield* Effect.tryPromise({ - try: async () => { - const response = await db - .prepare( - `SELECT ${dashboardFileColumns} - FROM files f - WHERE f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - ${tagFilter} - ORDER BY f.updated_at DESC, f.id - LIMIT ? OFFSET ?` - ) - .bind(new Date().toISOString(), ...tagIds, limit, offset) - .all(); - if (!response.success) - throw new Error(response.error ?? 'File listing failed'); - return response.results; - }, - catch: (cause) => - new StorageError({ operation: 'list filtered files', cause }) - }); + const rows = yield* sql` + SELECT ${sql.literal(dashboardFileColumns)} + FROM files f + WHERE f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) + ${selectedTagFilter(tagIds)} + ORDER BY f.updated_at DESC, f.id + LIMIT ${limit} OFFSET ${offset}`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'list filtered files', cause }) + ) + ); return decodeDashboardRows(rows).map(toDashboardFile); }); diff --git a/apps/web/src/lib/server/services/tags.ts b/apps/web/src/lib/server/services/tags.ts index 3a906fb..17e910d 100644 --- a/apps/web/src/lib/server/services/tags.ts +++ b/apps/web/src/lib/server/services/tags.ts @@ -1,26 +1,25 @@ import type { Tag, TagCreate, TagUpdate } from '@adrive/shared'; import { Context, Effect, Layer, Schema } from 'effect'; -import { SqlClient } from 'effect/unstable/sql'; import { InvalidRequest, NotFound, StorageError, validate } from '../errors'; -import { - fileIndexStatements, - refreshAllIndexedTagsStatement -} from '../search-index'; +import { refreshAllIndexedTags, refreshSearchDocument } from '../search-index'; import { normalizeTagColor, normalizeTagName, uniqueTagNames } from '../tag-policy'; import { createObjectTtlCache } from '../isolate-cache'; -import { Db } from './bindings'; +import { PgSql } from '../pg'; const TAG_LIST_CACHE_TTL_MS = 5_000; -const tagListCache = createObjectTtlCache>( +// One database per deployment, so the per-isolate cache needs a single +// stable key; the Postgres client is rebuilt per request and cannot be it. +const tagListCacheKey = {}; +const tagListCache = createObjectTtlCache>( TAG_LIST_CACHE_TTL_MS ); -export const forgetTagListCache = (db: D1Database) => { - tagListCache.delete(db); +export const forgetTagListCache = () => { + tagListCache.delete(tagListCacheKey); }; const TagRow = Schema.Struct({ @@ -52,7 +51,7 @@ const tagSelect = ` t.name, t.normalized_name, t.color, - COUNT(ft.file_id) AS file_count, + COUNT(ft.file_id)::integer AS file_count, t.created_at FROM tags t LEFT JOIN file_tags ft ON ft.tag_id = t.id @@ -80,42 +79,35 @@ export interface TagsShape { export class Tags extends Context.Service()('app/Tags') {} const makeTags = Effect.gen(function* () { - const db = yield* Db; - const sql = (yield* SqlClient.SqlClient).withoutTransforms(); + const sql = yield* PgSql; + const select = sql.literal(tagSelect); const list = Effect.gen(function* () { - const cached = tagListCache.get(db); + const cached = tagListCache.get(tagListCacheKey); if (cached) return cached; - const rows = yield* sql - .unsafe( - `${tagSelect} + const rows = yield* sql` + ${select} GROUP BY t.id - ORDER BY t.normalized_name, t.id` + ORDER BY t.normalized_name, t.id`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'list tags', cause }) ) - .pipe( - Effect.mapError( - (cause) => new StorageError({ operation: 'list tags', cause }) - ) - ); + ); const tags = decodeTagRows(rows).map(toTag); - tagListCache.set(db, tags); + tagListCache.set(tagListCacheKey, tags); return tags; }).pipe(Effect.withSpan('Tags.list')); const find = Effect.fn('Tags.find')(function* (id: string) { - const rows = yield* sql - .unsafe( - `${tagSelect} - WHERE t.id = ? - GROUP BY t.id - LIMIT 1`, - [id] + const rows = yield* sql` + ${select} + WHERE t.id = ${id} + GROUP BY t.id + LIMIT 1`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'find tag', cause }) ) - .pipe( - Effect.mapError( - (cause) => new StorageError({ operation: 'find tag', cause }) - ) - ); + ); const tag = decodeTagRows(rows)[0]; if (!tag) return yield* new NotFound({ id }); return toTag(tag); @@ -125,20 +117,15 @@ const makeTags = Effect.gen(function* () { normalizedNames: ReadonlyArray ) { if (normalizedNames.length === 0) return []; - const placeholders = normalizedNames.map(() => '?').join(', '); - const rows = yield* sql - .unsafe( - `${tagSelect} - WHERE t.normalized_name IN (${placeholders}) - GROUP BY t.id - ORDER BY t.normalized_name`, - normalizedNames + const rows = yield* sql` + ${select} + WHERE ${sql.in('t.normalized_name', normalizedNames)} + GROUP BY t.id + ORDER BY t.normalized_name`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'resolve tag names', cause }) ) - .pipe( - Effect.mapError( - (cause) => new StorageError({ operation: 'resolve tag names', cause }) - ) - ); + ); return decodeTagRows(rows).map(toTag); }); @@ -155,29 +142,22 @@ const makeTags = Effect.gen(function* () { (tag) => !existingNames.has(tag.normalizedName) ); if (missing.length > 0) { - yield* Effect.tryPromise({ - try: () => - db.batch( - missing.map((tag) => - db - .prepare( - `INSERT INTO tags ( - id, name, normalized_name, color, created_at - ) VALUES (?, ?, ?, NULL, ?) - ON CONFLICT(normalized_name) DO NOTHING` - ) - .bind( - crypto.randomUUID(), - tag.name, - tag.normalizedName, - createdAt - ) - ) - ), - catch: (cause) => - new StorageError({ operation: 'auto-create tags', cause }) - }); - forgetTagListCache(db); + yield* sql` + INSERT INTO tags ${sql.insert( + missing.map((tag) => ({ + id: crypto.randomUUID(), + name: tag.name, + normalized_name: tag.normalizedName, + color: null, + created_at: createdAt + })) + )} + ON CONFLICT (normalized_name) DO NOTHING`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'auto-create tags', cause }) + ) + ); + forgetTagListCache(); } const resolved = yield* findByNormalized(normalized); const byNormalized = new Map( @@ -196,24 +176,17 @@ const makeTags = Effect.gen(function* () { const tag = yield* validate(() => normalizeTagName(input.name)); const color = yield* validate(() => normalizeTagColor(input.color)); const createdAt = new Date().toISOString(); - yield* Effect.tryPromise({ - try: () => - db - .prepare( - `INSERT INTO tags (id, name, normalized_name, color, created_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(normalized_name) DO NOTHING` - ) - .bind( - crypto.randomUUID(), - tag.name, - tag.normalizedName, - color, - createdAt - ) - .run(), - catch: (cause) => new StorageError({ operation: 'create tag', cause }) - }); + yield* sql` + INSERT INTO tags (id, name, normalized_name, color, created_at) + VALUES ( + ${crypto.randomUUID()}, ${tag.name}, ${tag.normalizedName}, + ${color}, ${createdAt} + ) + ON CONFLICT (normalized_name) DO NOTHING`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'create tag', cause }) + ) + ); const resolved = yield* findByNormalized([tag.normalizedName]); const result = resolved[0]; if (!result) { @@ -222,7 +195,7 @@ const makeTags = Effect.gen(function* () { cause: 'Tag was not returned after creation' }); } - forgetTagListCache(db); + forgetTagListCache(); return result; }), update: Effect.fn('Tags.update')(function* (id, input) { @@ -250,65 +223,66 @@ const makeTags = Effect.gen(function* () { }); } - yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `UPDATE tags - SET name = ?, normalized_name = ?, color = ? - WHERE id = ?` - ) - .bind(name.name, name.normalizedName, color, id), - refreshAllIndexedTagsStatement(db) - ]), - catch: (cause) => new StorageError({ operation: 'update tag', cause }) - }); - forgetTagListCache(db); + yield* sql + .withTransaction( + sql` + UPDATE tags + SET name = ${name.name}, normalized_name = ${name.normalizedName}, + color = ${color} + WHERE id = ${id}`.pipe(Effect.andThen(refreshAllIndexedTags(sql))) + ) + .pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'update tag', cause }) + ) + ); + forgetTagListCache(); return yield* find(id); }), remove: Effect.fn('Tags.remove')(function* (id) { yield* find(id); - yield* Effect.tryPromise({ - try: () => - db.batch([ - db.prepare('DELETE FROM file_tags WHERE tag_id = ?').bind(id), - db.prepare('DELETE FROM tags WHERE id = ?').bind(id), - refreshAllIndexedTagsStatement(db) - ]), - catch: (cause) => new StorageError({ operation: 'delete tag', cause }) - }); - forgetTagListCache(db); - }), - setFileTags: Effect.fn('Tags.setFileTags')(function* (fileId, names) { - const file = yield* sql - .unsafe('SELECT id FROM files WHERE id = ? LIMIT 1', [fileId]) + // file_tags cascades from tags, so one delete drops the links. + yield* sql + .withTransaction( + sql`DELETE FROM tags WHERE id = ${id}`.pipe( + Effect.andThen(refreshAllIndexedTags(sql)) + ) + ) .pipe( Effect.mapError( - (cause) => - new StorageError({ operation: 'find tagged file', cause }) + (cause) => new StorageError({ operation: 'delete tag', cause }) ) ); + forgetTagListCache(); + }), + setFileTags: Effect.fn('Tags.setFileTags')(function* (fileId, names) { + const file = yield* sql` + SELECT id FROM files WHERE id = ${fileId} LIMIT 1`.pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'find tagged file', cause }) + ) + ); if (file.length === 0) return yield* new NotFound({ id: fileId }); const resolved = yield* resolveNames(names); - const statements = [ - db.prepare('DELETE FROM file_tags WHERE file_id = ?').bind(fileId), - ...resolved.map((tag) => - db - .prepare( - `INSERT INTO file_tags (file_id, tag_id) - VALUES (?, ?)` - ) - .bind(fileId, tag.id) - ), - ...fileIndexStatements(db, fileId) - ]; - yield* Effect.tryPromise({ - try: () => db.batch(statements), - catch: (cause) => - new StorageError({ operation: 'set file tags', cause }) - }); - forgetTagListCache(db); + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql`DELETE FROM file_tags WHERE file_id = ${fileId}`; + if (resolved.length > 0) { + yield* sql` + INSERT INTO file_tags ${sql.insert( + resolved.map((tag) => ({ file_id: fileId, tag_id: tag.id })) + )}`; + } + yield* refreshSearchDocument(sql, fileId); + }) + ) + .pipe( + Effect.mapError( + (cause) => new StorageError({ operation: 'set file tags', cause }) + ) + ); + forgetTagListCache(); }) }); }); diff --git a/apps/web/src/lib/server/thumbnail-storage.ts b/apps/web/src/lib/server/thumbnail-storage.ts index 34b22a9..d72f0c9 100644 --- a/apps/web/src/lib/server/thumbnail-storage.ts +++ b/apps/web/src/lib/server/thumbnail-storage.ts @@ -1,43 +1,47 @@ -export interface ThumbnailStorageCommand { - readonly sql: string; - readonly bindings: ReadonlyArray; +import type { PgClient } from '@effect/sql-pg'; +import { Effect } from 'effect'; + +export interface ThumbnailStorageState { + readonly thumbnail_r2_key: string | null; + readonly thumbnail_size_bytes: number; } -export const thumbnailStorageStateCommand = ( +// A thumbnail only attaches to a version whose file is not being purged; +// a purge that has claimed the file wins over any in-flight render. +export const thumbnailStorageState = ( + sql: PgClient.PgClient, fileId: string, version: number -): ThumbnailStorageCommand => ({ - sql: `SELECT v.thumbnail_r2_key, v.thumbnail_size_bytes +) => + sql` + SELECT v.thumbnail_r2_key, v.thumbnail_size_bytes FROM file_versions v JOIN files f ON f.id = v.file_id - WHERE v.file_id = ? AND v.version = ? + WHERE v.file_id = ${fileId} AND v.version = ${version} AND f.purge_state = 'none' - LIMIT 1`, - bindings: [fileId, version] -}); + LIMIT 1`.pipe(Effect.map((rows) => rows[0] ?? null)); -export const commitThumbnailStorageCommand = ( +// Compare-and-set on the stored key: only the writer whose expectation +// matches the current key commits, so concurrent renders keep one blob. +// Resolves to whether this writer won. +export const commitThumbnailStorage = ( + sql: PgClient.PgClient, fileId: string, version: number, r2Key: string, size: number, expectedR2Key: string | null -): ThumbnailStorageCommand => ({ - sql: `UPDATE file_versions - SET thumbnail_r2_key = ?, thumbnail_size_bytes = ? - WHERE file_id = ? AND version = ? - AND ( - (thumbnail_r2_key IS NULL AND ? IS NULL) - OR thumbnail_r2_key = ? - ) +) => + sql<{ file_id: string }>` + UPDATE file_versions + SET thumbnail_r2_key = ${r2Key}, thumbnail_size_bytes = ${size} + WHERE file_id = ${fileId} AND version = ${version} + AND thumbnail_r2_key IS NOT DISTINCT FROM ${expectedR2Key}::text AND EXISTS ( - SELECT 1 - FROM files f - WHERE f.id = file_versions.file_id - AND f.purge_state = 'none' - )`, - bindings: [r2Key, size, fileId, version, expectedR2Key, expectedR2Key] -}); + SELECT 1 FROM files f + WHERE f.id = file_versions.file_id AND f.purge_state = 'none' + ) + RETURNING file_id`.pipe(Effect.map((rows) => rows.length === 1)); export const thumbnailQuotaDelta = (storedSize: number, incomingSize: number) => Math.max(0, incomingSize - storedSize); From de190ac3e99092848e71c0484edf52b1f03aa89e Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:10:37 -0700 Subject: [PATCH 2/7] REFACTOR: Port auth and grant secrets to Postgres Auth (api_keys, dashboard_sessions, device_codes, credential_state) and the grant signing key now run on PgSql. D1 batches become transactions, meta.changes checks become RETURNING plus a length check, INSERT OR IGNORE becomes ON CONFLICT DO NOTHING, and the signing key cache is keyed by a module-scope sentinel since the Postgres client is per request. The sqlite-backed grant secrets test is replaced by a Postgres one, and the local key script inserts through pg. Co-Authored-By: Claude Fable 5.1 --- apps/web/src/lib/server/services/auth.ts | 360 ++++++++---------- .../src/lib/server/services/grant-secrets.ts | 93 ++--- scripts/create-local-key.mjs | 44 ++- 3 files changed, 226 insertions(+), 271 deletions(-) diff --git a/apps/web/src/lib/server/services/auth.ts b/apps/web/src/lib/server/services/auth.ts index b8f93ff..dcb1fce 100644 --- a/apps/web/src/lib/server/services/auth.ts +++ b/apps/web/src/lib/server/services/auth.ts @@ -1,7 +1,6 @@ import { API_KEY_PATTERN, type ApiKey, type ApiKeyScope } from '@adrive/shared'; import type { Cookies } from '@sveltejs/kit'; import { Context, Effect, Layer, Schema } from 'effect'; -import { SqlClient } from 'effect/unstable/sql'; import { DEVICE_CODE_TTL_SECONDS, DEVICE_POLL_INTERVAL_SECONDS, @@ -22,7 +21,7 @@ import { Unauthorized, validate } from '../errors'; -import { Db } from './bindings'; +import { PgSql } from '../pg'; const ApiKeyRow = Schema.Struct({ id: Schema.String, @@ -250,10 +249,12 @@ const toApiKey = (row: typeof ApiKeyRow.Type): ApiKey => ({ }); const makeAuth = Effect.gen(function* () { - const db = yield* Db; - const sql = (yield* SqlClient.SqlClient).withoutTransforms(); + const sql = yield* PgSql; const config = yield* AppConfig; + const storageError = (operation: string) => + Effect.mapError((cause: unknown) => new StorageError({ operation, cause })); + const makeApiKey = Effect.fn('Auth.makeApiKey')(function* (name: string) { const normalizedName = yield* validate(() => normalizeApiKeyName(name)); const prefix = randomHex(4); @@ -481,20 +482,13 @@ const makeAuth = Effect.gen(function* () { }; }), revokeApiKey: Effect.fn('Auth.revokeApiKey')(function* (id) { - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE api_keys - SET revoked_at = ? - WHERE id = ? AND revoked_at IS NULL` - ) - .bind(new Date().toISOString(), id) - .run(), - catch: (cause) => - new StorageError({ operation: 'revoke API key', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql` + UPDATE api_keys + SET revoked_at = ${new Date().toISOString()} + WHERE id = ${id} AND revoked_at IS NULL + RETURNING id + `.pipe(storageError('revoke API key')); + if (rows.length !== 1) { return yield* new InvalidRequest({ status: 404, message: 'API key was not found' @@ -512,32 +506,19 @@ const makeAuth = Effect.gen(function* () { ); let userCode = randomUserCode(); for (let attempt = 0; attempt < 5; attempt += 1) { - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `INSERT INTO device_codes ( - device_code_hash, user_code, status, interval_seconds, - expires_at, created_at, name - ) VALUES (?, ?, 'pending', ?, ?, ?, ?) - ON CONFLICT(user_code) DO NOTHING` - ) - .bind( - deviceCodeHash, - userCode, - DEVICE_POLL_INTERVAL_SECONDS, - expiresAt.toISOString(), - createdAt.toISOString(), - normalizedName - ) - .run(), - catch: (cause) => - new StorageError({ - operation: 'create device authorization', - cause - }) - }); - if (result.meta.changes === 1) { + const rows = yield* sql` + INSERT INTO device_codes ( + device_code_hash, user_code, status, interval_seconds, + expires_at, created_at, name + ) VALUES ( + ${deviceCodeHash}, ${userCode}, 'pending', + ${DEVICE_POLL_INTERVAL_SECONDS}, ${expiresAt.toISOString()}, + ${createdAt.toISOString()}, ${normalizedName} + ) + ON CONFLICT (user_code) DO NOTHING + RETURNING device_code_hash + `.pipe(storageError('create device authorization')); + if (rows.length === 1) { return { deviceCode, userCode, @@ -556,20 +537,13 @@ const makeAuth = Effect.gen(function* () { approveDevice: Effect.fn('Auth.approveDevice')(function* (userCode) { const code = yield* parseUserCode(userCode); const now = new Date().toISOString(); - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE device_codes - SET status = 'approved', approved_at = ? - WHERE user_code = ? AND status = 'pending' AND expires_at > ?` - ) - .bind(now, code, now) - .run(), - catch: (cause) => - new StorageError({ operation: 'approve device', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql` + UPDATE device_codes + SET status = 'approved', approved_at = ${now} + WHERE user_code = ${code} AND status = 'pending' AND expires_at > ${now} + RETURNING user_code + `.pipe(storageError('approve device')); + if (rows.length !== 1) { return yield* new InvalidRequest({ status: 400, message: 'Device approval code is invalid or expired' @@ -579,19 +553,13 @@ const makeAuth = Effect.gen(function* () { denyDevice: Effect.fn('Auth.denyDevice')(function* (userCode) { const code = yield* parseUserCode(userCode); const now = new Date().toISOString(); - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE device_codes - SET status = 'denied' - WHERE user_code = ? AND status = 'pending' AND expires_at > ?` - ) - .bind(code, now) - .run(), - catch: (cause) => new StorageError({ operation: 'deny device', cause }) - }); - if (result.meta.changes !== 1) { + const rows = yield* sql` + UPDATE device_codes + SET status = 'denied' + WHERE user_code = ${code} AND status = 'pending' AND expires_at > ${now} + RETURNING user_code + `.pipe(storageError('deny device')); + if (rows.length !== 1) { return yield* new InvalidRequest({ status: 400, message: 'Device approval code is invalid or expired' @@ -671,125 +639,108 @@ const makeAuth = Effect.gen(function* () { const generated = yield* makeApiKey(row.name); const consumedAt = now.toISOString(); - const results = yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `INSERT INTO api_keys ( - id, name, prefix, secret_hash, created_at - ) - SELECT ?, ?, ?, ?, ? - WHERE EXISTS ( - SELECT 1 FROM device_codes - WHERE device_code_hash = ? AND status = 'approved' - )` + // Guarded insert plus consume in one transaction: a concurrent poll + // that already consumed the code sees zero rows from both statements + // and the key it never minted is rolled back with it. + const completed = yield* sql + .withTransaction( + Effect.gen(function* () { + const inserted = yield* sql` + INSERT INTO api_keys ( + id, name, prefix, secret_hash, created_at ) - .bind( - generated.row.id, - generated.row.name, - generated.row.prefix, - generated.secretHash, - generated.row.createdAt, - hash - ), - db - .prepare( - `UPDATE device_codes - SET status = 'consumed', consumed_at = ?, api_key_id = ? - WHERE device_code_hash = ? AND status = 'approved'` + SELECT ${generated.row.id}, ${generated.row.name}, + ${generated.row.prefix}, ${generated.secretHash}, + ${generated.row.createdAt} + WHERE EXISTS ( + SELECT 1 FROM device_codes + WHERE device_code_hash = ${hash} AND status = 'approved' ) - .bind(consumedAt, generated.row.id, hash) - ]), - catch: (cause) => - new StorageError({ - operation: 'complete device authorization', - cause + RETURNING id + `; + const consumed = yield* sql` + UPDATE device_codes + SET status = 'consumed', consumed_at = ${consumedAt}, + api_key_id = ${generated.row.id} + WHERE device_code_hash = ${hash} AND status = 'approved' + RETURNING device_code_hash + `; + return inserted.length === 1 && consumed.length === 1; }) - }); - if (results[0]?.meta.changes !== 1 || results[1]?.meta.changes !== 1) { + ) + .pipe(storageError('complete device authorization')); + if (!completed) { return yield* new Unauthorized({ message: 'Device authorization was already consumed' }); } return { status: 'complete' as const, apiKey: generated.token }; }), - revokeAllSessions: Effect.gen(function* () { - const results = yield* Effect.tryPromise({ - try: () => - db.batch([ - db.prepare('DELETE FROM dashboard_sessions'), - db.prepare( - `UPDATE device_codes SET status = 'denied' - WHERE status IN ('pending', 'approved')` - ) - ]), - catch: (cause) => - new StorageError({ operation: 'revoke all sessions', cause }) - }); - return results.reduce( - (count, result) => count + (result.meta.changes ?? 0), - 0 - ); - }).pipe(Effect.withSpan('Auth.revokeAllSessions')), + revokeAllSessions: sql + .withTransaction( + Effect.gen(function* () { + const sessions = yield* sql` + DELETE FROM dashboard_sessions RETURNING token_hash + `; + const codes = yield* sql` + UPDATE device_codes SET status = 'denied' + WHERE status IN ('pending', 'approved') + RETURNING device_code_hash + `; + return sessions.length + codes.length; + }) + ) + .pipe( + storageError('revoke all sessions'), + Effect.withSpan('Auth.revokeAllSessions') + ), enforcePasscodeRotation: Effect.gen(function* () { const passcodeHash = yield* hashToken(config.passcode); const now = new Date().toISOString(); - // One transactional batch (D1 batches are transactions): seed the - // row on first boot without revoking anything, revoke while the - // stored hash still differs from the deployed one, then record the - // new hash. A failure anywhere rolls the whole claim back, so a - // rotation can never be marked recorded with revocation skipped. - const results = yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `INSERT INTO credential_state (id, passcode_hash, rotated_at) - VALUES (1, ?1, ?2) - ON CONFLICT(id) DO NOTHING` + // One transaction: seed the row on first boot without revoking + // anything, revoke while the stored hash still differs from the + // deployed one, then record the new hash. A failure anywhere rolls + // the whole claim back, so a rotation can never be marked recorded + // with revocation skipped. + return yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO credential_state (id, passcode_hash, rotated_at) + VALUES (1, ${passcodeHash}, ${now}) + ON CONFLICT (id) DO NOTHING + `; + const sessions = yield* sql` + DELETE FROM dashboard_sessions + WHERE EXISTS ( + SELECT 1 FROM credential_state + WHERE id = 1 AND passcode_hash <> ${passcodeHash} ) - .bind(passcodeHash, now), - db - .prepare( - `DELETE FROM dashboard_sessions - WHERE EXISTS ( + RETURNING token_hash + `; + const codes = yield* sql` + UPDATE device_codes SET status = 'denied' + WHERE status IN ('pending', 'approved') + AND EXISTS ( SELECT 1 FROM credential_state - WHERE id = 1 AND passcode_hash <> ?1 - )` - ) - .bind(passcodeHash), - db - .prepare( - `UPDATE device_codes SET status = 'denied' - WHERE status IN ('pending', 'approved') - AND EXISTS ( - SELECT 1 FROM credential_state - WHERE id = 1 AND passcode_hash <> ?1 - )` - ) - .bind(passcodeHash), - db - .prepare( - `UPDATE credential_state - SET passcode_hash = ?1, rotated_at = ?2 - WHERE id = 1 AND passcode_hash <> ?1` - ) - .bind(passcodeHash, now) - ]), - catch: (cause) => - new StorageError({ - operation: 'enforce passcode rotation', - cause + WHERE id = 1 AND passcode_hash <> ${passcodeHash} + ) + RETURNING device_code_hash + `; + const recorded = yield* sql` + UPDATE credential_state + SET passcode_hash = ${passcodeHash}, rotated_at = ${now} + WHERE id = 1 AND passcode_hash <> ${passcodeHash} + RETURNING id + `; + const rotated = recorded.length === 1; + return { + rotated, + revoked: rotated ? sessions.length + codes.length : 0 + }; }) - }); - const rotated = results[3]?.meta.changes === 1; - return { - rotated, - revoked: rotated - ? (results[1]?.meta.changes ?? 0) + (results[2]?.meta.changes ?? 0) - : 0 - }; + ) + .pipe(storageError('enforce passcode rotation')); }).pipe(Effect.withSpan('Auth.enforcePasscodeRotation')), sweepExpired: Effect.fn('Auth.sweepExpired')(function* (limit) { const bounded = Math.max(1, Math.min(limit, 100)); @@ -797,43 +748,34 @@ const makeAuth = Effect.gen(function* () { const cutoff = new Date( new Date(now).getTime() - 24 * 60 * 60 * 1_000 ).toISOString(); - const results = yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `DELETE FROM dashboard_sessions - WHERE token_hash IN ( - SELECT token_hash FROM dashboard_sessions - WHERE expires_at <= ? - ORDER BY expires_at - LIMIT ? - )` + return yield* sql + .withTransaction( + Effect.gen(function* () { + const sessions = yield* sql` + DELETE FROM dashboard_sessions + WHERE token_hash IN ( + SELECT token_hash FROM dashboard_sessions + WHERE expires_at <= ${now} + ORDER BY expires_at + LIMIT ${bounded} ) - .bind(now, bounded), - db - .prepare( - `DELETE FROM device_codes - WHERE device_code_hash IN ( - SELECT device_code_hash FROM device_codes - WHERE expires_at <= ? - OR (status = 'consumed' AND consumed_at <= ?) - ORDER BY expires_at - LIMIT ? - )` + RETURNING token_hash + `; + const codes = yield* sql` + DELETE FROM device_codes + WHERE device_code_hash IN ( + SELECT device_code_hash FROM device_codes + WHERE expires_at <= ${now} + OR (status = 'consumed' AND consumed_at <= ${cutoff}) + ORDER BY expires_at + LIMIT ${bounded} ) - .bind(now, cutoff, bounded) - ]), - catch: (cause) => - new StorageError({ - operation: 'sweep expired authentication state', - cause + RETURNING device_code_hash + `; + return sessions.length + codes.length; }) - }); - return results.reduce( - (count, result) => count + (result.meta.changes ?? 0), - 0 - ); + ) + .pipe(storageError('sweep expired authentication state')); }) }); }); diff --git a/apps/web/src/lib/server/services/grant-secrets.ts b/apps/web/src/lib/server/services/grant-secrets.ts index 5a6f972..c5ec55b 100644 --- a/apps/web/src/lib/server/services/grant-secrets.ts +++ b/apps/web/src/lib/server/services/grant-secrets.ts @@ -1,6 +1,7 @@ import { Context, Effect, Layer, Schema } from 'effect'; import { StorageError } from '../errors'; import { createObjectTtlCache } from '../isolate-cache'; +import { PgSql } from '../pg'; import { mintPrivateGrant, verifyPrivateGrant, @@ -8,7 +9,6 @@ import { type PrivateGrant, type VerifyPrivateGrantOptions } from '../private-grant'; -import { Db } from './bindings'; const PersistedSecretRow = Schema.Struct({ content_grant_signing_key: Schema.String @@ -17,13 +17,16 @@ const PersistedSecretRow = Schema.Struct({ const KEY_PATTERN = /^[A-Za-z0-9_-]{43}$/; // The grant signing key is seeded once per instance and never rewritten, so -// reading it from D1 on every request that mints or verifies a private link -// (each /link call plus each /t grant check) is wasted round trips. Cache it -// per isolate like the tag list and indexing status. If the database is ever -// reseeded, mint+verify stay consistent within an isolate, and a stale key -// on a handful of isolates self-heals on the next cache expiry. +// reading it from Postgres on every request that mints or verifies a private +// link (each /link call plus each /t grant check) is wasted round trips. +// Cache it per isolate like the tag list and indexing status. The Postgres +// client is built per request, so the cache is keyed by a module-scope +// sentinel: there is exactly one database per deployment. If the database +// is ever reseeded, mint+verify stay consistent within an isolate, and a +// stale key on a handful of isolates self-heals on the next cache expiry. const SIGNING_KEY_CACHE_TTL_MS = 5 * 60_000; -const signingKeyCache = createObjectTtlCache( +const signingKeyCacheKey = {}; +const signingKeyCache = createObjectTtlCache( SIGNING_KEY_CACHE_TTL_MS ); @@ -54,44 +57,46 @@ const randomSigningKey = () => { }; const makeGrantSecrets = Effect.gen(function* () { - const db = yield* Db; + const sql = yield* PgSql; - const signingKey = Effect.tryPromise({ - try: async () => { - const cached = signingKeyCache.get(db); - if (cached) return cached; - const candidate = randomSigningKey(); - await db - .prepare( - `INSERT OR IGNORE INTO instance_secrets ( - id, content_grant_signing_key, created_at - ) VALUES (1, ?, ?)` - ) - .bind(candidate, new Date().toISOString()) - .run(); - const value = await db - .prepare( - `SELECT content_grant_signing_key - FROM instance_secrets - WHERE id = 1` - ) - .first(); - const decoded = Schema.decodeUnknownOption(PersistedSecretRow)(value); - if ( - decoded._tag !== 'Some' || - !KEY_PATTERN.test(decoded.value.content_grant_signing_key) - ) { - throw new Error('The persisted content grant key is unavailable'); - } - signingKeyCache.set(db, decoded.value.content_grant_signing_key); - return decoded.value.content_grant_signing_key; - }, - catch: (cause) => - new StorageError({ - operation: 'initialize content grant signing key', - cause - }) - }); + const signingKey = Effect.gen(function* () { + const cached = signingKeyCache.get(signingKeyCacheKey); + if (cached) return cached; + const candidate = randomSigningKey(); + yield* sql` + INSERT INTO instance_secrets ( + id, content_grant_signing_key, created_at + ) VALUES (1, ${candidate}, ${new Date().toISOString()}) + ON CONFLICT (id) DO NOTHING + `; + const rows = yield* sql` + SELECT content_grant_signing_key + FROM instance_secrets + WHERE id = 1 + `; + const decoded = Schema.decodeUnknownOption(PersistedSecretRow)(rows[0]); + if ( + decoded._tag !== 'Some' || + !KEY_PATTERN.test(decoded.value.content_grant_signing_key) + ) { + return yield* Effect.fail( + 'The persisted content grant key is unavailable' + ); + } + signingKeyCache.set( + signingKeyCacheKey, + decoded.value.content_grant_signing_key + ); + return decoded.value.content_grant_signing_key; + }).pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'initialize content grant signing key', + cause + }) + ) + ); return GrantSecrets.of({ mint: Effect.fn('GrantSecrets.mint')(function* (input) { diff --git a/scripts/create-local-key.mjs b/scripts/create-local-key.mjs index e2e7da6..fec421a 100644 --- a/scripts/create-local-key.mjs +++ b/scripts/create-local-key.mjs @@ -1,30 +1,38 @@ import { createHash, randomBytes, randomUUID } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; import { join } from 'node:path'; +// pg is a dependency of apps/web, not the workspace root, so resolve it +// from there. +const require = createRequire( + join(import.meta.dirname, '..', 'apps', 'web', 'package.json') +); +/** @type {typeof import('pg')} */ +const Pg = require('pg'); + +const LOCAL_DATABASE_URL = 'postgres://adrive:adrive@127.0.0.1:5432/adrive'; + const prefix = randomBytes(4).toString('hex'); const secret = randomBytes(32).toString('base64url'); const token = `adr_${prefix}_${secret}`; const hash = createHash('sha256').update(token).digest('hex'); -const id = randomUUID(); -const createdAt = new Date().toISOString(); -const sql = [ - 'INSERT INTO api_keys (id, name, prefix, secret_hash, created_at)', - `VALUES ('${id}', 'local development', '${prefix}', '${hash}', '${createdAt}')` -].join(' '); -const result = spawnSync( - 'bun', - ['x', 'wrangler', 'd1', 'execute', 'DB', '--local', '--command', sql], - { - cwd: join(import.meta.dirname, '..', 'apps', 'web'), - stdio: ['ignore', 'inherit', 'inherit'] - } -); +const client = new Pg.Client({ + connectionString: process.env.DATABASE_URL ?? LOCAL_DATABASE_URL +}); -if (result.status !== 0) { - process.exitCode = result.status ?? 1; -} else { +try { + await client.connect(); + await client.query( + `INSERT INTO api_keys (id, name, prefix, secret_hash, created_at) + VALUES ($1, $2, $3, $4, $5)`, + [randomUUID(), 'local development', prefix, hash, new Date().toISOString()] + ); console.log('\nLocal API key (shown once):'); console.log(token); +} catch (cause) { + console.error(cause instanceof Error ? cause.message : String(cause)); + process.exitCode = 1; +} finally { + await client.end().catch(() => {}); } From 6c662b0dc27d66ebca400feaf7e5b92e6a80b73d Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:45:18 -0700 Subject: [PATCH 3/7] REFACTOR: Port sites to Postgres Move site sessions, asset staging, commit, cleanup, and asset lookup from D1 to the PgSql client. Batches become transactions, changes checks become RETURNING, and the site commit refreshes search_documents in the same transaction. Drop the D1-only exports that sites was the last consumer of: fileIndexStatements, refreshAllIndexedTagsStatement, ensureStorageQuota, and StorageQuotaDatabase. Co-Authored-By: Claude Fable 5.1 --- apps/web/src/lib/server/search-index.ts | 42 -- apps/web/src/lib/server/services/sites.ts | 6 +- .../lib/server/services/sites/internals.ts | 277 +++++-------- .../web/src/lib/server/services/sites/read.ts | 43 +- .../src/lib/server/services/sites/sessions.ts | 387 ++++++++---------- apps/web/src/lib/server/storage-quota.ts | 58 +-- 6 files changed, 307 insertions(+), 506 deletions(-) diff --git a/apps/web/src/lib/server/search-index.ts b/apps/web/src/lib/server/search-index.ts index b7379bf..4d7b515 100644 --- a/apps/web/src/lib/server/search-index.ts +++ b/apps/web/src/lib/server/search-index.ts @@ -1,48 +1,6 @@ import type { PgClient } from '@effect/sql-pg'; import { Effect } from 'effect'; -export const fileIndexStatements = (db: D1Database, fileId: string) => [ - db.prepare('DELETE FROM files_fts WHERE file_id = ?').bind(fileId), - db - .prepare( - `INSERT INTO files_fts (name, tags, body, file_id, chunk_no) - SELECT - f.display_name, - COALESCE(( - SELECT group_concat(t.name, ' ') - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = f.id - ), ''), - substr(COALESCE(v.text_content, ''), 1, 65536), - f.id, - 0 - FROM files f - JOIN file_versions v - ON v.file_id = f.id AND v.version = f.current_version - WHERE f.id = ?` - ) - .bind(fileId), - db.prepare('DELETE FROM files_trgm WHERE file_id = ?').bind(fileId), - db - .prepare( - `INSERT INTO files_trgm (name, file_id) - SELECT display_name, id FROM files WHERE id = ?` - ) - .bind(fileId) -]; - -export const refreshAllIndexedTagsStatement = (db: D1Database) => - db.prepare( - `UPDATE files_fts - SET tags = COALESCE(( - SELECT group_concat(t.name, ' ') - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = files_fts.file_id - ), '')` - ); - // Postgres keyword index. One search_documents row per file today, so a // refresh is delete then insert; chunk_no exists for splitting long bodies // later. Runs inside whatever transaction the caller holds. diff --git a/apps/web/src/lib/server/services/sites.ts b/apps/web/src/lib/server/services/sites.ts index 98856e7..b8ea582 100644 --- a/apps/web/src/lib/server/services/sites.ts +++ b/apps/web/src/lib/server/services/sites.ts @@ -1,7 +1,7 @@ import { Context, Effect, Layer } from 'effect'; import { AppConfig } from '../config'; +import { PgSql } from '../pg'; import { Blobs } from './blobs'; -import { Db } from './bindings'; import { cleanupOps } from './sites/cleanup'; import { createInternals } from './sites/internals'; import { readOps } from './sites/read'; @@ -18,11 +18,11 @@ export type { export class Sites extends Context.Service()('app/Sites') {} const makeSites = Effect.gen(function* () { - const db = yield* Db; + const sql = yield* PgSql; const blobs = yield* Blobs; const config = yield* AppConfig; - const internals = createInternals({ db, blobs, config }); + const internals = createInternals({ sql, blobs, config }); return Sites.of({ ...sessionOps(internals), diff --git a/apps/web/src/lib/server/services/sites/internals.ts b/apps/web/src/lib/server/services/sites/internals.ts index c675a2c..50ca185 100644 --- a/apps/web/src/lib/server/services/sites/internals.ts +++ b/apps/web/src/lib/server/services/sites/internals.ts @@ -1,9 +1,10 @@ +import type { PgClient } from '@effect/sql-pg'; import { Effect, Schema } from 'effect'; +import { queueDeferredBlobDelete } from '../../blob-compensation'; import { AppConfig } from '../../config'; import { NotFound, StorageError } from '../../errors'; import { siteCleanupDisposition } from '../../site-policy'; import { Blobs } from '../blobs'; -import { Db } from '../bindings'; import { PendingDeleteRow, SiteSessionRow, @@ -12,40 +13,32 @@ import { } from './types'; interface CoreDeps { - readonly db: Db['Service']; + readonly sql: PgClient.PgClient; readonly blobs: Blobs['Service']; readonly config: AppConfig['Service']; } -export const createInternals = ({ db, blobs, config }: CoreDeps) => { - const all = Effect.fn('Sites.all')(function* ( - statement: D1PreparedStatement, +export const createInternals = ({ sql, blobs, config }: CoreDeps) => { + const all = ( + statement: Effect.Effect, unknown>, schema: Schema.Codec, operation: string - ) { - const rows = yield* Effect.tryPromise({ - try: async () => { - const result = await statement.all(); - if (!result.success) throw new Error(result.error ?? operation); - return result.results; - }, - catch: (cause) => new StorageError({ operation, cause }) - }); - return decodeRows(schema, rows); - }); + ) => + statement.pipe( + Effect.mapError((cause) => new StorageError({ operation, cause })), + Effect.map((rows) => decodeRows(schema, rows)), + Effect.withSpan('Sites.all') + ); const findSession = Effect.fn('Sites.findSession')(function* ( sessionId: string ) { const rows = yield* all( - db - .prepare( - `SELECT id, file_id, display_name, version, status, created_at, expires_at - FROM site_upload_sessions - WHERE id = ? - LIMIT 1` - ) - .bind(sessionId), + sql` + SELECT id, file_id, display_name, version, status, created_at, expires_at + FROM site_upload_sessions + WHERE id = ${sessionId} + LIMIT 1`, SiteSessionRow, 'find site upload session' ); @@ -66,15 +59,12 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { sessionId: string ) { const rows = yield* all( - db - .prepare( - `SELECT path, expected_size_bytes, content_type, r2_key, - stored_size_bytes - FROM staged_site_assets - WHERE session_id = ? - ORDER BY path` - ) - .bind(sessionId), + sql` + SELECT path, expected_size_bytes, content_type, r2_key, + stored_size_bytes + FROM staged_site_assets + WHERE session_id = ${sessionId} + ORDER BY path`, StagedAssetRow, 'list staged site assets' ); @@ -90,18 +80,16 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { const pendingDeleteCount = Effect.fn('Sites.pendingDeleteCount')(function* ( fileId: string ) { - const result = yield* Effect.tryPromise({ - try: () => - db - .prepare( - 'SELECT COUNT(*) AS count FROM pending_site_asset_deletes WHERE file_id = ?' - ) - .bind(fileId) - .first<{ count: number }>(), - catch: (cause) => - new StorageError({ operation: 'count pending site cleanup', cause }) - }); - return (result?.count ?? 0) > 0; + const rows = yield* sql<{ count: number }>` + SELECT COUNT(*) AS count + FROM pending_site_asset_deletes + WHERE file_id = ${fileId}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'count pending site cleanup', cause }) + ) + ); + return (rows[0]?.count ?? 0) > 0; }); const compensateStagedBlob = Effect.fn('Sites.compensateStagedBlob')( @@ -114,29 +102,14 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { ) { yield* blobs.delete(r2Key).pipe( Effect.catchCause((deleteCause) => - Effect.tryPromise({ - try: () => - db - .prepare( - `INSERT INTO pending_site_asset_deletes ( - r2_key, file_id, version, queued_at, attempts, last_error - ) VALUES (?, ?, ?, ?, 1, ?) - ON CONFLICT(r2_key) DO NOTHING` - ) - .bind( - r2Key, - session.fileId, - session.version, - new Date().toISOString(), - String(deleteCause) - ) - .run(), - catch: (queueCause) => - new StorageError({ - operation: 'queue orphaned staged site asset', - cause: queueCause - }) - }).pipe( + queueDeferredBlobDelete( + sql, + r2Key, + session.fileId, + session.version, + new Date().toISOString(), + String(deleteCause) + ).pipe( Effect.catchCause((queueCause) => Effect.sync(() => { console.error( @@ -160,15 +133,12 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { fileId: string ) { const rows = yield* all( - db - .prepare( - `SELECT r2_key - FROM pending_site_asset_deletes - WHERE file_id = ? - ORDER BY queued_at - LIMIT 500` - ) - .bind(fileId), + sql` + SELECT r2_key + FROM pending_site_asset_deletes + WHERE file_id = ${fileId} + ORDER BY queued_at + LIMIT 500`, PendingDeleteRow, 'list pending site cleanup' ); @@ -178,47 +148,32 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { const deleted = yield* blobs.deleteMany(keys).pipe( Effect.as(true), Effect.catch((failure) => - Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE pending_site_asset_deletes - SET attempts = attempts + 1, last_error = ? - WHERE file_id = ?` - ) - .bind(String(failure.cause), fileId) - .run(), - catch: (cause) => - new StorageError({ - operation: 'record site cleanup failure', - cause - }) - }).pipe(Effect.as(false)) + sql` + UPDATE pending_site_asset_deletes + SET attempts = attempts + 1, last_error = ${String(failure.cause)} + WHERE file_id = ${fileId}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'record site cleanup failure', + cause + }) + ), + Effect.as(false) + ) ) ); const disposition = siteCleanupDisposition(keys, deleted); if (disposition.remainsPending) return true; - const statements = Array.from( - { length: Math.ceil(disposition.deleteFromQueue.length / 80) }, - (_, index) => { - const chunk = disposition.deleteFromQueue.slice( - index * 80, - index * 80 + 80 - ); - return db - .prepare( - `DELETE FROM pending_site_asset_deletes - WHERE r2_key IN (${chunk.map(() => '?').join(', ')})` - ) - .bind(...chunk); - } + yield* sql` + DELETE FROM pending_site_asset_deletes + WHERE r2_key = ANY(${disposition.deleteFromQueue}::text[])`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'finish site asset cleanup', cause }) + ) ); - yield* Effect.tryPromise({ - try: () => db.batch(statements), - catch: (cause) => - new StorageError({ operation: 'finish site asset cleanup', cause }) - }); return yield* pendingDeleteCount(fileId); }); @@ -231,33 +186,30 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { status: 'aborted' | 'complete' ) { const now = new Date().toISOString(); - yield* Effect.tryPromise({ - try: () => - db.batch([ - db - .prepare( - `INSERT INTO pending_site_asset_deletes ( - r2_key, file_id, version, queued_at - ) - SELECT r2_key, ?, ?, ? - FROM staged_site_assets - WHERE session_id = ? AND r2_key IS NOT NULL - ON CONFLICT(r2_key) DO NOTHING` - ) - .bind(session.fileId, session.version, now, session.id), - db - .prepare( - `UPDATE site_upload_sessions SET status = ? - WHERE id = ? AND status IN ('open', 'committing')` + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO pending_site_asset_deletes ( + r2_key, file_id, version, queued_at ) - .bind(status, session.id), - db - .prepare('DELETE FROM staged_site_assets WHERE session_id = ?') - .bind(session.id) - ]), - catch: (cause) => - new StorageError({ operation: 'clean staged site assets', cause }) - }); + SELECT r2_key, ${session.fileId}, ${session.version}, ${now} + FROM staged_site_assets + WHERE session_id = ${session.id} AND r2_key IS NOT NULL + ON CONFLICT (r2_key) DO NOTHING`; + yield* sql` + UPDATE site_upload_sessions SET status = ${status} + WHERE id = ${session.id} AND status IN ('open', 'committing')`; + yield* sql` + DELETE FROM staged_site_assets WHERE session_id = ${session.id}`; + }) + ) + .pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'clean staged site assets', cause }) + ) + ); yield* drainDeletes(session.fileId); }); @@ -266,16 +218,13 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { const now = new Date().toISOString(); const bounded = Math.max(1, Math.min(limit, 25)); const rows = yield* all( - db - .prepare( - `SELECT id, file_id, display_name, version, status, created_at, - expires_at - FROM site_upload_sessions - WHERE status = 'open' AND expires_at <= ? - ORDER BY expires_at - LIMIT ?` - ) - .bind(now, bounded), + sql` + SELECT id, file_id, display_name, version, status, created_at, + expires_at + FROM site_upload_sessions + WHERE status = 'open' AND expires_at <= ${now} + ORDER BY expires_at + LIMIT ${bounded}`, SiteSessionRow, 'list expired site upload sessions' ); @@ -297,25 +246,17 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { limit: number ) { const bounded = Math.max(1, Math.min(limit, 25)); - const rows = yield* Effect.tryPromise({ - try: async () => { - const result = await db - .prepare( - `SELECT DISTINCT file_id - FROM pending_site_asset_deletes - ORDER BY queued_at - LIMIT ?` - ) - .bind(bounded) - .all<{ file_id: string }>(); - if (!result.success) { - throw new Error(result.error ?? 'Site cleanup listing failed'); - } - return result.results; - }, - catch: (cause) => - new StorageError({ operation: 'list pending site cleanup', cause }) - }); + const rows = yield* sql<{ file_id: string }>` + SELECT file_id + FROM pending_site_asset_deletes + GROUP BY file_id + ORDER BY MIN(queued_at) + LIMIT ${bounded}`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'list pending site cleanup', cause }) + ) + ); for (const row of rows) yield* drainDeletes(row.file_id); return rows.length; }); @@ -330,7 +271,7 @@ export const createInternals = ({ db, blobs, config }: CoreDeps) => { cleanupStaged, sweepExpiredSessions, sweepPendingDeletes, - db, + sql, blobs, config }; diff --git a/apps/web/src/lib/server/services/sites/read.ts b/apps/web/src/lib/server/services/sites/read.ts index 9ac31e7..8739f1d 100644 --- a/apps/web/src/lib/server/services/sites/read.ts +++ b/apps/web/src/lib/server/services/sites/read.ts @@ -7,7 +7,7 @@ import { SiteAssetRow, type SitesShape } from './types'; export const readOps = ( internals: SiteInternals ): Pick => { - const { all, db } = internals; + const { all, sql } = internals; return { findAsset: Effect.fn('Sites.findAsset')(function* ( @@ -26,33 +26,24 @@ export const readOps = ( message: 'Site asset path is unsafe' }) }); + const includeUnavailable = options.includeUnavailable === true; + const pinVersion = options.version !== undefined; const rows = yield* all( - db - .prepare( - `SELECT a.path, a.r2_key, a.content_type, a.size_bytes - FROM files f - JOIN site_assets a - ON a.file_id = f.id AND a.version = f.current_version - WHERE f.id = ? AND f.is_site = 1 AND f.public = 1 - AND a.version = f.current_version - AND ( - ? = 1 - OR ( - f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - ) + sql` + SELECT a.path, a.r2_key, a.content_type, a.size_bytes + FROM files f + JOIN site_assets a + ON a.file_id = f.id AND a.version = f.current_version + WHERE f.id = ${fileId} AND f.is_site = true AND f.public = true + AND ( + ${includeUnavailable}::boolean + OR ( + f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) ) - AND (? = 0 OR a.version = ?) - AND a.path IN (${candidates.map(() => '?').join(', ')})` - ) - .bind( - fileId, - options.includeUnavailable ? 1 : 0, - new Date().toISOString(), - options.version === undefined ? 0 : 1, - options.version ?? 0, - ...candidates - ), + ) + AND (${pinVersion}::boolean = false OR a.version = ${options.version ?? 0}) + AND a.path = ANY(${candidates}::text[])`, SiteAssetRow, 'find site asset' ); diff --git a/apps/web/src/lib/server/services/sites/sessions.ts b/apps/web/src/lib/server/services/sites/sessions.ts index b3940e5..a47f8ed 100644 --- a/apps/web/src/lib/server/services/sites/sessions.ts +++ b/apps/web/src/lib/server/services/sites/sessions.ts @@ -1,8 +1,8 @@ import { normalizeSitePath } from '@adrive/shared'; import { Effect } from 'effect'; import { InvalidRequest, NotFound, StorageError } from '../../errors'; -import { fileIndexStatements } from '../../search-index'; -import { ensureStorageQuota } from '../../storage-quota'; +import { refreshSearchDocument } from '../../search-index'; +import { ensureStoredBytesWithin } from '../../storage-quota'; import { assertOpenSiteSession, prepareSiteManifest, @@ -24,7 +24,7 @@ export const sessionOps = ( cleanupStaged, drainDeletes, sweepExpiredSessions, - db, + sql, blobs, config } = internals; @@ -59,21 +59,18 @@ export const sessionOps = ( (total, asset) => total + asset.sizeBytes, 0 ); - yield* ensureStorageQuota(db, config.maxTotalBytes, declaredBytes); + yield* ensureStoredBytesWithin(sql, config.maxTotalBytes, declaredBytes); let fileId: string = crypto.randomUUID(); let version = 1; let displayName = prepared.displayName; if (input.fileId !== undefined) { const rows = yield* all( - db - .prepare( - `SELECT id, display_name, current_version - FROM files - WHERE id = ? AND is_site = 1 AND deleted_at IS NULL - LIMIT 1` - ) - .bind(input.fileId), + sql` + SELECT id, display_name, current_version + FROM files + WHERE id = ${input.fileId} AND is_site = true AND deleted_at IS NULL + LIMIT 1`, ExistingSiteRow, 'find site to republish' ); @@ -89,29 +86,35 @@ export const sessionOps = ( const expiresAt = new Date( new Date(createdAt).getTime() + SITE_SESSION_TTL_MS ).toISOString(); - const statements = [ - db - .prepare( - `INSERT INTO site_upload_sessions ( - id, file_id, display_name, version, status, created_at, expires_at - ) VALUES (?, ?, ?, ?, 'open', ?, ?)` - ) - .bind(id, fileId, displayName, version, createdAt, expiresAt), - ...prepared.assets.map((asset) => - db - .prepare( - `INSERT INTO staged_site_assets ( - session_id, path, expected_size_bytes, content_type - ) VALUES (?, ?, ?, ?)` - ) - .bind(id, asset.path, asset.sizeBytes, asset.contentType) + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO site_upload_sessions ( + id, file_id, display_name, version, status, created_at, expires_at + ) VALUES ( + ${id}, ${fileId}, ${displayName}, ${version}, 'open', + ${createdAt}, ${expiresAt} + )`; + for (const asset of prepared.assets) { + yield* sql` + INSERT INTO staged_site_assets ( + session_id, path, expected_size_bytes, content_type + ) VALUES ( + ${id}, ${asset.path}, ${asset.sizeBytes}, ${asset.contentType} + )`; + } + }) ) - ]; - yield* Effect.tryPromise({ - try: () => db.batch(statements), - catch: (cause) => - new StorageError({ operation: 'create site upload session', cause }) - }); + .pipe( + Effect.mapError( + (cause) => + new StorageError({ + operation: 'create site upload session', + cause + }) + ) + ); yield* drainDeletes(fileId).pipe( Effect.catchCause((cause) => Effect.sync(() => { @@ -182,38 +185,28 @@ export const sessionOps = ( asset.contentType ); const uploadedAt = new Date().toISOString(); - const update = yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE staged_site_assets - SET r2_key = ?, stored_size_bytes = ?, uploaded_at = ? - WHERE session_id = ? AND path = ? AND r2_key IS NULL - AND EXISTS ( - SELECT 1 FROM site_upload_sessions - WHERE id = ? AND status = 'open' AND expires_at > ? - )` - ) - .bind( - r2Key, - stored.size, - uploadedAt, - session.id, - path, - session.id, - uploadedAt - ) - .run(), - catch: (cause) => - new StorageError({ operation: 'record staged site asset', cause }) - }).pipe( + const updated = yield* sql<{ path: string }>` + UPDATE staged_site_assets + SET r2_key = ${r2Key}, stored_size_bytes = ${stored.size}, + uploaded_at = ${uploadedAt} + WHERE session_id = ${session.id} AND path = ${path} AND r2_key IS NULL + AND EXISTS ( + SELECT 1 FROM site_upload_sessions + WHERE id = ${session.id} AND status = 'open' + AND expires_at > ${uploadedAt} + ) + RETURNING path`.pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'record staged site asset', cause }) + ), Effect.catch((failure) => compensateStagedBlob(session, r2Key).pipe( Effect.andThen(Effect.fail(failure)) ) ) ); - if (update.meta.changes !== 1) { + if (updated.length !== 1) { yield* compensateStagedBlob(session, r2Key); return yield* new InvalidRequest({ status: 409, @@ -249,153 +242,129 @@ export const sessionOps = ( const versionKey = `site-version/${session.fileId}/${session.version}/${session.id}`; const guard = session.version === 1 - ? db - .prepare( - `UPDATE site_upload_sessions - SET status = 'committing' - WHERE id = ? AND status = 'open' AND expires_at > ? - AND NOT EXISTS (SELECT 1 FROM files WHERE id = ?)` + ? sql<{ id: string }>` + UPDATE site_upload_sessions + SET status = 'committing' + WHERE id = ${session.id} AND status = 'open' + AND expires_at > ${publishedAt} + AND NOT EXISTS (SELECT 1 FROM files WHERE id = ${session.fileId}) + RETURNING id` + : sql<{ id: string }>` + UPDATE site_upload_sessions + SET status = 'committing' + WHERE id = ${session.id} AND status = 'open' + AND expires_at > ${publishedAt} + AND EXISTS ( + SELECT 1 FROM files + WHERE id = ${session.fileId} AND is_site = true + AND deleted_at IS NULL + AND current_version = ${session.version - 1} + ) + RETURNING id`; + + const commit = sql + .withTransaction( + Effect.gen(function* () { + const guarded = yield* guard; + if (guarded.length !== 1) { + return yield* new StorageError({ + operation: 'commit site version', + cause: 'The site changed while it was publishing' + }); + } + yield* sql` + INSERT INTO files ( + id, display_name, content_type, kind, current_version, size_bytes, + public, is_site, created_at, updated_at, index_state ) - .bind(session.id, publishedAt, session.fileId) - : db - .prepare( - `UPDATE site_upload_sessions - SET status = 'committing' - WHERE id = ? AND status = 'open' AND expires_at > ? - AND EXISTS ( - SELECT 1 FROM files - WHERE id = ? AND is_site = 1 AND deleted_at IS NULL - AND current_version = ? - )` + SELECT file_id, display_name, 'text/html', 'site', 1, ${totalSize}, + true, true, ${publishedAt}, ${publishedAt}, 'pending' + FROM site_upload_sessions + WHERE id = ${session.id} AND status = 'committing' AND version = 1 + ON CONFLICT (id) DO NOTHING`; + yield* sql` + UPDATE files + SET current_version = ${session.version}, size_bytes = ${totalSize}, + content_type = 'text/html', public = true, + updated_at = ${publishedAt}, index_state = 'pending', + index_cursor = 0, index_attempts = 0, index_error = NULL, + index_next_run_at = NULL, index_lease_token = NULL + WHERE id = ${session.fileId} + AND current_version = ${session.version - 1} + AND is_site = true + AND EXISTS ( + SELECT 1 FROM site_upload_sessions + WHERE id = ${session.id} AND status = 'committing' + AND version > 1 + )`; + yield* sql` + INSERT INTO file_versions ( + file_id, version, r2_key, size_bytes, sha256, content_type, + created_at, text_content ) - .bind( - session.id, - publishedAt, - session.fileId, - session.version - 1 - ); - const statements = [ - guard, - db - .prepare( - `INSERT INTO files ( - id, display_name, content_type, kind, current_version, size_bytes, - public, is_site, created_at, updated_at, index_state - ) - SELECT file_id, display_name, 'text/html', 'site', 1, ?, 1, 1, - ?, ?, 'pending' - FROM site_upload_sessions - WHERE id = ? AND status = 'committing' AND version = 1 - ON CONFLICT(id) DO NOTHING` - ) - .bind(totalSize, publishedAt, publishedAt, session.id), - db - .prepare( - `UPDATE files - SET current_version = ?, size_bytes = ?, content_type = 'text/html', - public = 1, updated_at = ?, index_state = 'pending', - index_cursor = 0, index_attempts = 0, index_error = NULL, - index_next_run_at = NULL, index_lease_token = NULL - WHERE id = ? AND current_version = ? AND is_site = 1 - AND EXISTS ( - SELECT 1 FROM site_upload_sessions - WHERE id = ? AND status = 'committing' AND version > 1 - )` - ) - .bind( - session.version, - totalSize, - publishedAt, - session.fileId, - session.version - 1, - session.id - ), - db - .prepare( - `INSERT INTO file_versions ( - file_id, version, r2_key, size_bytes, sha256, content_type, - created_at, text_content - ) - SELECT s.file_id, s.version, ?, ?, NULL, 'text/html', ?, NULL - FROM site_upload_sessions s - JOIN files f ON f.id = s.file_id - WHERE s.id = ? AND s.status = 'committing' - AND f.current_version = s.version` - ) - .bind(versionKey, totalSize, publishedAt, session.id), - db - .prepare( - `INSERT INTO pending_site_asset_deletes ( - r2_key, file_id, version, queued_at - ) - SELECT old.r2_key, old.file_id, old.version, ? - FROM site_assets old - WHERE old.file_id = ? AND old.version <> ? - AND EXISTS ( - SELECT 1 FROM site_upload_sessions - WHERE id = ? AND status = 'committing' + SELECT s.file_id, s.version, ${versionKey}, ${totalSize}, NULL, + 'text/html', ${publishedAt}, NULL + FROM site_upload_sessions s + JOIN files f ON f.id = s.file_id + WHERE s.id = ${session.id} AND s.status = 'committing' + AND f.current_version = s.version`; + yield* sql` + INSERT INTO pending_site_asset_deletes ( + r2_key, file_id, version, queued_at ) - ON CONFLICT(r2_key) DO NOTHING` - ) - .bind(publishedAt, session.fileId, session.version, session.id), - db - .prepare( - `DELETE FROM site_assets - WHERE file_id = ? AND version <> ? - AND EXISTS ( + SELECT old.r2_key, old.file_id, old.version, ${publishedAt} + FROM site_assets old + WHERE old.file_id = ${session.fileId} + AND old.version <> ${session.version} + AND EXISTS ( + SELECT 1 FROM site_upload_sessions + WHERE id = ${session.id} AND status = 'committing' + ) + ON CONFLICT (r2_key) DO NOTHING`; + yield* sql` + DELETE FROM site_assets + WHERE file_id = ${session.fileId} AND version <> ${session.version} + AND EXISTS ( + SELECT 1 FROM site_upload_sessions + WHERE id = ${session.id} AND status = 'committing' + )`; + yield* sql` + INSERT INTO site_assets ( + file_id, version, path, r2_key, content_type, size_bytes + ) + SELECT s.file_id, s.version, a.path, a.r2_key, a.content_type, + a.stored_size_bytes + FROM site_upload_sessions s + JOIN staged_site_assets a ON a.session_id = s.id + WHERE s.id = ${session.id} AND s.status = 'committing' + AND a.r2_key IS NOT NULL AND a.stored_size_bytes IS NOT NULL`; + yield* refreshSearchDocument(sql, session.fileId); + yield* sql` + UPDATE site_upload_sessions SET status = 'complete' + WHERE id = ${session.id} AND status = 'committing' + AND ( + SELECT COUNT(*) FROM staged_site_assets + WHERE session_id = ${session.id} + ) = ( + SELECT COUNT(*) FROM site_assets + WHERE file_id = ${session.fileId} + AND version = ${session.version} + )`; + yield* sql` + DELETE FROM staged_site_assets + WHERE session_id = ${session.id} AND EXISTS ( SELECT 1 FROM site_upload_sessions - WHERE id = ? AND status = 'committing' - )` - ) - .bind(session.fileId, session.version, session.id), - db - .prepare( - `INSERT INTO site_assets ( - file_id, version, path, r2_key, content_type, size_bytes + WHERE id = ${session.id} AND status = 'complete' + )`; + }) + ) + .pipe( + Effect.catchTag('SqlError', (cause) => + Effect.fail( + new StorageError({ operation: 'commit site version', cause }) ) - SELECT s.file_id, s.version, a.path, a.r2_key, a.content_type, - a.stored_size_bytes - FROM site_upload_sessions s - JOIN staged_site_assets a ON a.session_id = s.id - WHERE s.id = ? AND s.status = 'committing' - AND a.r2_key IS NOT NULL AND a.stored_size_bytes IS NOT NULL` - ) - .bind(session.id), - ...fileIndexStatements(db, session.fileId), - db - .prepare( - `UPDATE site_upload_sessions SET status = 'complete' - WHERE id = ? AND status = 'committing' - AND ( - SELECT COUNT(*) FROM staged_site_assets - WHERE session_id = ? - ) = ( - SELECT COUNT(*) FROM site_assets - WHERE file_id = ? AND version = ? - )` ) - .bind(session.id, session.id, session.fileId, session.version), - db - .prepare( - `DELETE FROM staged_site_assets - WHERE session_id = ? AND EXISTS ( - SELECT 1 FROM site_upload_sessions - WHERE id = ? AND status = 'complete' - )` - ) - .bind(session.id, session.id) - ]; - - const commit = Effect.tryPromise({ - try: async () => { - const results = await db.batch(statements); - if (results[0]?.meta.changes !== 1) { - throw new Error('The site changed while it was publishing'); - } - }, - catch: (cause) => - new StorageError({ operation: 'commit site version', cause }) - }); + ); yield* commit.pipe( Effect.catch((failure) => cleanupStaged(session, 'aborted').pipe( @@ -430,15 +399,13 @@ export const sessionOps = ( ) ); const rows = yield* all( - db - .prepare( - `SELECT id, display_name, current_version, size_bytes, created_at, - expires_at, download_count, last_download_at - FROM files - WHERE id = ? AND is_site = 1 AND current_version = ? - LIMIT 1` - ) - .bind(session.fileId, session.version), + sql` + SELECT id, display_name, current_version, size_bytes, created_at, + expires_at, download_count, last_download_at + FROM files + WHERE id = ${session.fileId} AND is_site = true + AND current_version = ${session.version} + LIMIT 1`, SiteFileRow, 'read published site' ); diff --git a/apps/web/src/lib/server/storage-quota.ts b/apps/web/src/lib/server/storage-quota.ts index b8bb9de..44545af 100644 --- a/apps/web/src/lib/server/storage-quota.ts +++ b/apps/web/src/lib/server/storage-quota.ts @@ -2,12 +2,6 @@ import type { PgClient } from '@effect/sql-pg'; import { Effect } from 'effect'; import { InvalidRequest, StorageError } from './errors'; -export interface StorageQuotaDatabase { - readonly prepare: (query: string) => { - readonly first: () => Promise; - }; -} - // Bytes actually held in R2: every version of a regular file is stored, // a site keeps only its current version's assets (old assets are deleted // at commit), and assets staged into an uncommitted publish session are @@ -15,57 +9,7 @@ export interface StorageQuotaDatabase { // Concurrent uploads can each pass this check before the other commits — // for a single-user deployment that bounded overshoot (≤ one upload per // concurrent request) is accepted; the quota is a safety net, not billing. -const TOTAL_STORED_BYTES_SQL = ` - SELECT - COALESCE(( - SELECT SUM(v.size_bytes + v.thumbnail_size_bytes) - FROM file_versions v - JOIN files f ON f.id = v.file_id - WHERE f.is_site = 0 - ), 0) - + - COALESCE(( - SELECT SUM(size_bytes) FROM files WHERE is_site = 1 - ), 0) - + - COALESCE(( - SELECT SUM(a.stored_size_bytes) - FROM staged_site_assets a - JOIN site_upload_sessions s ON s.id = a.session_id - WHERE a.stored_size_bytes IS NOT NULL - AND s.status IN ('open', 'committing') - ), 0) - AS total`; - -export const ensureStorageQuota = ( - db: StorageQuotaDatabase, - maxTotalBytes: number, - incomingBytes: number -) => - Effect.gen(function* () { - const row = yield* Effect.tryPromise({ - try: () => db.prepare(TOTAL_STORED_BYTES_SQL).first<{ total: number }>(), - catch: (cause) => - new StorageError({ operation: 'measure stored bytes', cause }) - }); - // Fail closed: an unreadable total must block the upload, not wave - // it through as if the bucket were empty. - if (row === null || !Number.isFinite(row.total)) { - return yield* new StorageError({ - operation: 'measure stored bytes', - cause: 'Unexpected aggregate result' - }); - } - if (row.total + incomingBytes > maxTotalBytes) { - return yield* new InvalidRequest({ - status: 413, - message: 'The storage quota is exhausted' - }); - } - }).pipe(Effect.withSpan('StorageQuota.ensure')); - -// Postgres version of the same aggregate. Sites and files both call this -// before writing bytes; the D1 version above goes away with the cutover. +// Sites and files both call this before writing bytes. export const ensureStoredBytesWithin = ( sql: PgClient.PgClient, maxTotalBytes: number, From 6343c214d6fd090b8be70549c58b464bc6101b6b Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 02:36:14 -0700 Subject: [PATCH 4/7] REFACTOR: Move keyword and semantic search into Postgres Keyword search now runs on search_documents (tsvector + pg_trgm) and semantic search on file_chunks.embedding (pgvector), replacing the D1 FTS5/trigram tables and the Vectorize index. - search-candidates.ts: fullTextCandidates (websearch_to_tsquery parsed with both the simple and english dictionaries, ts_rank_cd) and trigramCandidates (word_similarity on the name, threshold in code) - services/search.ts: drop the FTS5 sanitizers and the post-hoc semantic eligibility filter; the vector query filters visibility and tags itself - services/semantic.ts: VectorIndex is built from PgSql; Embedder still binds Workers AI; hasSemanticBindings only checks AI - indexing-sql.ts: lease-guarded transactions with FOR UPDATE, chunks and embeddings committed together, no pending_vector_deletes - services/indexing.ts: PgSql port, retryVectorDeletes removed - services/lifecycle.ts: vectors task removed - wrangler.jsonc / worker-configuration.d.ts: VECTORIZE binding removed - scripts/pg-rebuild-search.mjs replaces rebuild-search-index.sql - docs, skills, and README updated for the pgvector layout Co-Authored-By: Claude Fable 5.1 --- .agents/skills/deploy-fresh-instance/SKILL.md | 32 +- .agents/skills/verify-deployment/SKILL.md | 4 +- README.md | 42 +- apps/web/package.json | 2 +- apps/web/scripts/pg-rebuild-search.mjs | 46 ++ apps/web/scripts/rebuild-search-index.sql | 24 - apps/web/src/lib/server/indexing-sql.ts | 386 +++++++--------- apps/web/src/lib/server/layer.ts | 6 +- apps/web/src/lib/server/search-candidates.ts | 113 ++--- apps/web/src/lib/server/search-ranking.ts | 38 +- apps/web/src/lib/server/semantic-policy.ts | 30 -- apps/web/src/lib/server/services/indexing.ts | 413 +++++------------- apps/web/src/lib/server/services/lifecycle.ts | 8 +- apps/web/src/lib/server/services/search.ts | 138 ++---- apps/web/src/lib/server/services/semantic.ts | 164 ++++--- apps/web/worker-configuration.d.ts | 4 +- apps/web/wrangler.jsonc | 27 +- docs/release.md | 28 +- 18 files changed, 539 insertions(+), 966 deletions(-) create mode 100644 apps/web/scripts/pg-rebuild-search.mjs delete mode 100644 apps/web/scripts/rebuild-search-index.sql diff --git a/.agents/skills/deploy-fresh-instance/SKILL.md b/.agents/skills/deploy-fresh-instance/SKILL.md index 776be91..53d12b5 100644 --- a/.agents/skills/deploy-fresh-instance/SKILL.md +++ b/.agents/skills/deploy-fresh-instance/SKILL.md @@ -1,6 +1,6 @@ --- name: deploy-fresh-instance -description: Deploy a fresh a-drive instance to Cloudflare Workers — provision D1, R2, KV, and Vectorize plus Workers AI, set the PASSCODE secret, attach the two custom domains, and cut the first release. Use for the first-ever deploy to a new Cloudflare account or a new operator domain (not for routine redeploys, which are just `bun release`). +description: Deploy a fresh a-drive instance to Cloudflare Workers — provision D1, R2, KV, and Postgres plus Workers AI, set the PASSCODE secret, attach the two custom domains, and cut the first release. Use for the first-ever deploy to a new Cloudflare account or a new operator domain (not for routine redeploys, which are just `bun release`). --- # Deploy a fresh a-drive instance to Cloudflare @@ -41,29 +41,15 @@ into `env.production` by hand. (no id to paste — the bucket is bound by name.) 3. `wrangler kv namespace create AUTH_GUARD --env production` → paste the printed id into `env.production.kv_namespaces[0].id`. -4. Create the Vectorize index and its three metadata indexes: - - ``` - wrangler vectorize create adrive-production --dimensions=384 --metric=cosine - wrangler vectorize create-metadata-index adrive-production --property-name=deleted --type=boolean - wrangler vectorize create-metadata-index adrive-production --property-name=kind --type=string - wrangler vectorize create-metadata-index adrive-production --property-name=visibility --type=string - ``` - - **Gotcha (expected, safe):** metadata-index creation is async — each - command returns "enqueued", and `wrangler vectorize list-metadata-index -adrive-production` shows them appearing one at a time over a minute or - two. That is propagation lag, not a failure. It is harmless because no - vectors are written until after the first deploy, so you don't need to - wait for all three before continuing. - - **Workers AI needs no provisioning** — it activates from the `ai` - binding already declared in `wrangler.jsonc` `env.production`. +4. Semantic search needs no extra provisioning. Embeddings come from + the Workers AI `ai` binding already declared in `wrangler.jsonc` + `env.production`, and the vectors are stored in Postgres + (`file_chunks.embedding`, pgvector), which the `HYPERDRIVE` binding + already reaches. Note: `env.production` sets `SEMANTIC_SEARCH=required` (not `auto`), so a - missing `AI` or `VECTORIZE` binding fails the deploy **loudly** instead - of silently degrading to keyword search. That is intended — if the - Vectorize/AI setup above is incomplete, the deploy in step 5 will stop. + missing `AI` binding fails the deploy **loudly** instead of silently + degrading to keyword search. That is intended. 5. In the Cloudflare dashboard, open **Images → Transformations**, select the zone that owns `CONTENT_ORIGIN` (`davis7.space` for @@ -140,7 +126,7 @@ Expected live checks once both origins are up: ## 7. Backfill note (semantic search) -Any files uploaded before the Vectorize index existed sit in +Any files uploaded while the `AI` binding was absent sit in `index_state = 'disabled'` and are backfilled by the maintenance cron at about **5 files per 5 minutes**. The settings page's indexed-chunk count shows progress. On a truly fresh instance there's usually nothing to diff --git a/.agents/skills/verify-deployment/SKILL.md b/.agents/skills/verify-deployment/SKILL.md index 7ae84fb..69f682f 100644 --- a/.agents/skills/verify-deployment/SKILL.md +++ b/.agents/skills/verify-deployment/SKILL.md @@ -57,8 +57,8 @@ key-only run can conclude at best INCONCLUSIVE — never PASS. repo, or `wrangler deployments list --env production` when wrangler is authenticated) plus the current UTC time. - **[M]** If wrangler is available: confirm the production env lists DB - (D1), BUCKET (R2), AUTH_GUARD (KV), AI (Workers AI), and VECTORIZE - bindings (`wrangler deploy --dry-run --env production` output). Without + (D1), BUCKET (R2), AUTH_GUARD (KV), HYPERDRIVE (Postgres), and AI + (Workers AI) bindings (`wrangler deploy --dry-run --env production` output). Without wrangler this mandatory check is SKIPPED and the run cannot PASS. ## 2. Authentication diff --git a/README.md b/README.md index 58b9f1a..a2785a4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ scheduled storage lifecycle management. Uploads stream directly to R2, metadata and append-only version history live in D1, and file/site bytes are served from a separate cookie-less content origin. Search combines weighted FTS5 BM25 results, a filename trigram index, and an -optional Workers AI + Vectorize semantic source with reciprocal rank fusion. +optional Workers AI + pgvector semantic source with reciprocal rank fusion. Canonical D1 hydration still applies deletion, expiry, visibility, and tag filters. The CLI supports file transfer and safe, staged directory publishing. @@ -173,9 +173,8 @@ and `scheduled` exports. Its signed internal maintenance request is authenticate with a short-lived HMAC derived from `PASSCODE`; the endpoint cannot be invoked with a static or public header. -The FTS5 tables are derived state so canonical tables can be backed up one table -at a time even though D1 whole-database export does not support virtual tables. -After restoring the canonical tables, rebuild local search state with: +The `search_documents` table is derived keyword-search state. After restoring +the canonical tables, rebuild local search state with: ```bash bun search:rebuild:local @@ -201,7 +200,7 @@ No remote resource is created or modified by the local setup above. File expiration is enforced immediately by API, search, file, and site reads. The scheduled Worker runs every five minutes. It re-drives interrupted indexing, physically deletes expired/trash bytes before removing canonical D1 rows, expires -dashboard/device/site-upload sessions, and retries deferred R2/site/vector +dashboard/device/site-upload sessions, and retries deferred R2/site deletes. Work is bounded per invocation and every retryable transition is stored in D1. Download counts increment for full downloads and the initial `bytes=0-…` request only, so follow-up range requests do not inflate the count. @@ -209,29 +208,17 @@ in D1. Download counts increment for full downloads and the initial ### Optional semantic search Keyword, typo-tolerant, tag, and extracted-text search work without AI bindings. -`SEMANTIC_SEARCH` defaults to `auto`: both `AI` and `VECTORIZE` must be present -before the semantic layer activates. Set it to `off` to force the null-object -layer, or `required` to make a missing binding a startup error. +`SEMANTIC_SEARCH` defaults to `auto`: the `AI` binding must be present before +the semantic layer activates. Set it to `off` to force the null-object layer, +or `required` to make a missing binding a startup error. Embeddings are stored +in Postgres (`file_chunks.embedding`, pgvector), so no separate vector service +is provisioned. -Provision the optional resources once, before the first vector insert: - -```bash -cd apps/web -bun x wrangler vectorize create adrive \ - --dimensions=384 --metric=cosine -bun x wrangler vectorize create-metadata-index adrive \ - --propertyName=deleted --type=boolean -bun x wrangler vectorize create-metadata-index adrive \ - --propertyName=kind --type=string -bun x wrangler vectorize create-metadata-index adrive \ - --propertyName=visibility --type=string -``` - -Then add these bindings to `apps/web/wrangler.jsonc` and regenerate types: +To enable semantic search locally, add the binding to `apps/web/wrangler.jsonc` +and regenerate types: ```jsonc -"ai": { "binding": "AI" }, -"vectorize": [{ "binding": "VECTORIZE", "index_name": "adrive" }] +"ai": { "binding": "AI" } ``` ```bash @@ -240,7 +227,6 @@ bun --filter @adrive/web types:worker The embedding contract is pinned in Wrangler config to `@cf/baai/bge-small-en-v1.5`, `pooling: "cls"`, and 384 dimensions. Changing any -of those values requires a new Vectorize index and a full reindex. The dashboard -shows the indexed chunk count and warns that Vectorize queries are billed against -stored vectors multiplied by dimensions. Failed files retry with exponential +of those values requires a full reindex. The dashboard shows the indexed chunk +count. Failed files retry with exponential backoff up to five attempts and can be queued again with **Reindex**. diff --git a/apps/web/package.json b/apps/web/package.json index 0af697d..b1144a6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,7 +11,7 @@ "dev": "concurrently --kill-others --names dashboard,content \"bun run dev:dashboard\" \"bun run dev:content\"", "dev:content": "node scripts/content-proxy.mjs", "dev:dashboard": "vite dev --host 0.0.0.0 --port 5173", - "search:rebuild:local": "wrangler d1 execute DB --local --file scripts/rebuild-search-index.sql", + "search:rebuild:local": "bun scripts/pg-rebuild-search.mjs", "types:worker": "wrangler types" }, "dependencies": { diff --git a/apps/web/scripts/pg-rebuild-search.mjs b/apps/web/scripts/pg-rebuild-search.mjs new file mode 100644 index 0000000..98b7c40 --- /dev/null +++ b/apps/web/scripts/pg-rebuild-search.mjs @@ -0,0 +1,46 @@ +// Rebuilds search_documents (derived keyword-search state) from the +// canonical files, file_versions, tags, and file_tags tables. Mirrors +// refreshSearchDocument in src/lib/server/search-index.ts; keep the two +// SELECTs in sync. +// +// bun scripts/pg-rebuild-search.mjs # DATABASE_URL or local default +// bun scripts/pg-rebuild-search.mjs --url postgres://... +import Pg from 'pg'; +import { LOCAL_DATABASE_URL } from './pg-migrate.mjs'; + +const urlFlag = process.argv.indexOf('--url'); +const url = + urlFlag >= 0 + ? process.argv[urlFlag + 1] + : (process.env.DATABASE_URL ?? LOCAL_DATABASE_URL); + +const client = new Pg.Client({ connectionString: url }); +await client.connect(); +try { + await client.query('BEGIN'); + await client.query('DELETE FROM search_documents'); + const inserted = await client.query( + `INSERT INTO search_documents (file_id, chunk_no, name, tags, body) + SELECT + f.id, + 0, + f.display_name, + COALESCE(( + SELECT string_agg(t.name, ' ' ORDER BY t.normalized_name) + FROM file_tags ft + JOIN tags t ON t.id = ft.tag_id + WHERE ft.file_id = f.id + ), ''), + left(COALESCE(v.text_content, ''), 65536) + FROM files f + JOIN file_versions v + ON v.file_id = f.id AND v.version = f.current_version` + ); + await client.query('COMMIT'); + console.log(`rebuilt ${inserted.rowCount ?? 0} search documents`); +} catch (cause) { + await client.query('ROLLBACK'); + throw cause; +} finally { + await client.end(); +} diff --git a/apps/web/scripts/rebuild-search-index.sql b/apps/web/scripts/rebuild-search-index.sql deleted file mode 100644 index f552248..0000000 --- a/apps/web/scripts/rebuild-search-index.sql +++ /dev/null @@ -1,24 +0,0 @@ --- files_fts and files_trgm are derived state. This script deliberately reads --- only canonical tables and never invokes FTS maintenance commands. -DELETE FROM files_fts; -DELETE FROM files_trgm; - -INSERT INTO files_fts (name, tags, body, file_id, chunk_no) -SELECT - f.display_name, - COALESCE(( - SELECT group_concat(t.name, ' ') - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = f.id - ), ''), - substr(COALESCE(v.text_content, ''), 1, 65536), - f.id, - 0 -FROM files f -JOIN file_versions v - ON v.file_id = f.id AND v.version = f.current_version; - -INSERT INTO files_trgm (name, file_id) -SELECT display_name, id -FROM files; diff --git a/apps/web/src/lib/server/indexing-sql.ts b/apps/web/src/lib/server/indexing-sql.ts index 66f5ad2..d306fe0 100644 --- a/apps/web/src/lib/server/indexing-sql.ts +++ b/apps/web/src/lib/server/indexing-sql.ts @@ -1,11 +1,6 @@ -import type { TextChunk } from './semantic-policy'; - -export type SqlBinding = string | number | null; - -export interface SqlCommand { - readonly sql: string; - readonly bindings: ReadonlyArray; -} +import type { PgClient } from '@effect/sql-pg'; +import { Data, Effect } from 'effect'; +import { refreshSearchDocument } from './search-index'; export interface IndexLease { readonly fileId: string; @@ -14,257 +9,172 @@ export interface IndexLease { readonly token: string; } -interface IndexFailure { +export interface IndexFailure { readonly state: 'pending' | 'failed'; readonly error: string; readonly nextRunAt: string | null; } -const leaseExists = `SELECT 1 FROM files - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`; +export interface VectorChunk { + readonly fileId: string; + readonly version: number; + readonly ordinal: number; + readonly charStart: number; + readonly charEnd: number; + readonly values: ReadonlyArray; +} -const leaseBindings = (lease: IndexLease) => [ - lease.fileId, - lease.version, - lease.token -]; +// Raised inside a transaction when the claimed lease no longer holds; the +// transaction rolls back and the caller sees `false` instead of an error. +class StaleIndexLease extends Data.TaggedError('StaleIndexLease')<{}> {} -const vectorIdsJson = (vectorIds: ReadonlyArray) => - JSON.stringify(vectorIds); +const staleAsFalse = ( + effect: Effect.Effect +) => + effect.pipe( + Effect.as(true), + Effect.catchTag('StaleIndexLease', () => Effect.succeed(false)) + ); -export const claimIndexCommand = ( +// Locks the files row for the rest of the transaction so a concurrent +// claim for a newer version waits for this commit instead of interleaving. +const holdLease = (sql: PgClient.PgClient, lease: IndexLease) => + sql<{ held: number }>` + SELECT 1 AS held FROM files + WHERE id = ${lease.fileId} AND current_version = ${lease.version} + AND index_state = 'running' AND index_lease_token = ${lease.token} + FOR UPDATE`.pipe( + Effect.flatMap((rows) => + rows.length === 1 ? Effect.void : new StaleIndexLease() + ) + ); + +const leaseUpdateFilter = (sql: PgClient.PgClient, lease: IndexLease) => + sql`id = ${lease.fileId} AND current_version = ${lease.version} + AND index_state = 'running' AND index_lease_token = ${lease.token}`; + +export const claimIndex = ( + sql: PgClient.PgClient, lease: IndexLease, now: string, leaseUntil: string, maxAttempts: number -): SqlCommand => ({ - sql: `UPDATE files +) => + sql<{ id: string }>` + UPDATE files SET index_state = 'running', index_attempts = index_attempts + 1, - index_error = NULL, index_next_run_at = ?, index_lease_token = ? - WHERE id = ? AND current_version = ? AND deleted_at IS NULL - AND (expires_at IS NULL OR expires_at > ?) - AND index_attempts = ? AND index_attempts < ? + index_error = NULL, index_next_run_at = ${leaseUntil}, + index_lease_token = ${lease.token} + WHERE id = ${lease.fileId} AND current_version = ${lease.version} + AND deleted_at IS NULL + AND (expires_at IS NULL OR expires_at > ${now}) + AND index_attempts = ${lease.attempt - 1} + AND index_attempts < ${maxAttempts} AND ( index_state IN ('pending', 'disabled') - OR (index_state = 'running' AND index_next_run_at <= ?) - )`, - bindings: [ - leaseUntil, - lease.token, - lease.fileId, - lease.version, - now, - lease.attempt - 1, - maxAttempts, - now - ] -}); + OR (index_state = 'running' AND index_next_run_at <= ${now}) + ) + RETURNING id`.pipe(Effect.map((rows) => rows.length === 1)); -export const extractedTextCommands = ( +// Stores the extracted text for the leased version and refreshes the +// keyword document from it. index_cursor = 1 marks the text as stored so a +// retried attempt skips the R2 read. +export const storeExtractedText = ( + sql: PgClient.PgClient, lease: IndexLease, text: string -): ReadonlyArray => [ - { - sql: `UPDATE file_versions SET text_content = ? - WHERE file_id = ? AND version = ? - AND EXISTS (${leaseExists})`, - bindings: [text, lease.fileId, lease.version, ...leaseBindings(lease)] - }, - { - sql: `UPDATE files SET index_cursor = 1 - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`, - bindings: leaseBindings(lease) - }, - { - sql: `DELETE FROM files_fts - WHERE file_id = ? AND EXISTS (${leaseExists})`, - bindings: [lease.fileId, ...leaseBindings(lease)] - }, - { - sql: `INSERT INTO files_fts (name, tags, body, file_id, chunk_no) - SELECT - f.display_name, - COALESCE(( - SELECT group_concat(t.name, ' ') - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = f.id - ), ''), - substr(COALESCE(v.text_content, ''), 1, 65536), - f.id, - 0 - FROM files f - JOIN file_versions v - ON v.file_id = f.id AND v.version = f.current_version - WHERE f.id = ? AND f.current_version = ? - AND f.index_state = 'running' AND f.index_lease_token = ?`, - bindings: leaseBindings(lease) - }, - { - sql: `DELETE FROM files_trgm - WHERE file_id = ? AND EXISTS (${leaseExists})`, - bindings: [lease.fileId, ...leaseBindings(lease)] - }, - { - sql: `INSERT INTO files_trgm (name, file_id) - SELECT display_name, id - FROM files - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`, - bindings: leaseBindings(lease) - } -]; - -export const finishKeywordOnlyCommand = (lease: IndexLease): SqlCommand => ({ - sql: `UPDATE files +) => + sql + .withTransaction( + Effect.gen(function* () { + yield* holdLease(sql, lease); + yield* sql` + UPDATE file_versions SET text_content = ${text} + WHERE file_id = ${lease.fileId} AND version = ${lease.version}`; + yield* sql` + UPDATE files SET index_cursor = 1 + WHERE ${leaseUpdateFilter(sql, lease)}`; + yield* refreshSearchDocument(sql, lease.fileId); + }) + ) + .pipe(staleAsFalse); + +export const finishKeywordOnly = (sql: PgClient.PgClient, lease: IndexLease) => + sql<{ id: string }>` + UPDATE files SET index_state = 'disabled', indexed_version = NULL, index_attempts = 0, index_error = NULL, index_next_run_at = NULL, index_lease_token = NULL - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`, - bindings: leaseBindings(lease) -}); - -const chunkJson = ( - chunks: ReadonlyArray, - vectorIds: ReadonlyArray + WHERE ${leaseUpdateFilter(sql, lease)} + RETURNING id`.pipe(Effect.map((rows) => rows.length === 1)); + +// One multi-row statement: the columns arrive as parallel arrays and are +// zipped by unnest, and the embeddings travel in pgvector's text form. +export const upsertFileChunks = ( + sql: PgClient.PgClient, + rows: ReadonlyArray ) => - JSON.stringify( - chunks.map((chunk, index) => ({ - vectorId: vectorIds[index], - ordinal: chunk.ordinal, - charStart: chunk.charStart, - charEnd: chunk.charEnd - })) - ); - -const queueStaleVectorsCommand = ( - lease: IndexLease, - vectorIds: ReadonlyArray, - queuedAt: string -): SqlCommand => ({ - sql: `INSERT INTO pending_vector_deletes (vector_id, queued_at) - SELECT CAST(value AS TEXT), ? - FROM json_each(?) - WHERE NOT EXISTS (${leaseExists}) - ON CONFLICT(vector_id) DO NOTHING`, - bindings: [queuedAt, vectorIdsJson(vectorIds), ...leaseBindings(lease)] -}); - -const queueAttemptVectorsCommand = ( - vectorIds: ReadonlyArray, - queuedAt: string -): SqlCommand => ({ - sql: `INSERT INTO pending_vector_deletes (vector_id, queued_at) - SELECT CAST(value AS TEXT), ? FROM json_each(?) WHERE 1 - ON CONFLICT(vector_id) DO NOTHING`, - bindings: [queuedAt, vectorIdsJson(vectorIds)] -}); - -export const semanticCommitCommands = ( - lease: IndexLease, - chunks: ReadonlyArray, - vectorIds: ReadonlyArray, - indexedAt: string -): ReadonlyArray => [ - queueStaleVectorsCommand(lease, vectorIds, indexedAt), - { - sql: `INSERT INTO pending_vector_deletes (vector_id, queued_at) - SELECT chunks.vector_id, ? - FROM file_chunks chunks - WHERE chunks.file_id = ? - AND EXISTS (${leaseExists}) - ON CONFLICT(vector_id) DO NOTHING`, - bindings: [indexedAt, lease.fileId, ...leaseBindings(lease)] - }, - { - sql: `DELETE FROM file_chunks - WHERE file_id = ? AND EXISTS (${leaseExists})`, - bindings: [lease.fileId, ...leaseBindings(lease)] - }, - { - sql: `INSERT INTO file_chunks ( - vector_id, file_id, version, ordinal, char_start, char_end + rows.length === 0 + ? Effect.void + : sql` + INSERT INTO file_chunks ( + file_id, version, ordinal, char_start, char_end, embedding ) - SELECT - CAST(json_extract(value, '$.vectorId') AS TEXT), - ?, - ?, - CAST(json_extract(value, '$.ordinal') AS INTEGER), - CAST(json_extract(value, '$.charStart') AS INTEGER), - CAST(json_extract(value, '$.charEnd') AS INTEGER) - FROM json_each(?) - WHERE EXISTS (${leaseExists})`, - bindings: [ - lease.fileId, - lease.version, - chunkJson(chunks, vectorIds), - ...leaseBindings(lease) - ] - }, - { - sql: `UPDATE files - SET index_state = 'ready', indexed_version = ?, index_cursor = ?, - index_attempts = 0, index_error = NULL, index_next_run_at = NULL, - index_lease_token = NULL - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`, - bindings: [lease.version, chunks.length, ...leaseBindings(lease)] - } -]; - -export const indexFailureCommands = ( - lease: IndexLease, - vectorIds: ReadonlyArray, - failure: IndexFailure, - failedAt: string -): ReadonlyArray => [ - ...(vectorIds.length === 0 - ? [] - : [queueAttemptVectorsCommand(vectorIds, failedAt)]), - { - sql: `UPDATE files - SET index_state = ?, index_error = ?, index_next_run_at = ?, - index_lease_token = NULL - WHERE id = ? AND current_version = ? - AND index_state = 'running' AND index_lease_token = ?`, - bindings: [ - failure.state, - failure.error, - failure.nextRunAt, - ...leaseBindings(lease) - ] - } -]; - -export const vectorDeleteFailureCommand = ( - rows: ReadonlyArray<{ - readonly vectorId: string; - readonly nextRunAt: string; - }>, - error: string -): SqlCommand => ({ - sql: `UPDATE pending_vector_deletes - SET attempts = attempts + 1, - last_error = ?, - next_run_at = ( - SELECT json_extract(value, '$.nextRunAt') - FROM json_each(?) - WHERE json_extract(value, '$.vectorId') = - pending_vector_deletes.vector_id + SELECT * + FROM unnest( + ${rows.map((row) => row.fileId)}::text[], + ${rows.map((row) => row.version)}::integer[], + ${rows.map((row) => row.ordinal)}::integer[], + ${rows.map((row) => row.charStart)}::integer[], + ${rows.map((row) => row.charEnd)}::integer[], + ${rows.map((row) => `[${row.values.join(',')}]`)}::vector[] ) - WHERE vector_id IN ( - SELECT json_extract(value, '$.vectorId') FROM json_each(?) - )`, - bindings: [error, JSON.stringify(rows), JSON.stringify(rows)] -}); - -export const vectorDeleteSuccessCommand = ( - vectorIds: ReadonlyArray -): SqlCommand => ({ - sql: `DELETE FROM pending_vector_deletes - WHERE vector_id IN (SELECT CAST(value AS TEXT) FROM json_each(?))`, - bindings: [vectorIdsJson(vectorIds)] -}); + ON CONFLICT (file_id, version, ordinal) DO UPDATE + SET char_start = EXCLUDED.char_start, + char_end = EXCLUDED.char_end, + embedding = EXCLUDED.embedding`.pipe(Effect.asVoid); + +// Commits the embedded chunks and the ready state together. Chunks from +// older versions (and any same-version ordinals past the new count) go in +// the same transaction, so a stale lease rolls everything back and the +// newer version's rows are never touched. +export const semanticCommit = ( + sql: PgClient.PgClient, + lease: IndexLease, + chunks: ReadonlyArray +) => + sql + .withTransaction( + Effect.gen(function* () { + yield* holdLease(sql, lease); + yield* sql` + DELETE FROM file_chunks + WHERE file_id = ${lease.fileId} + AND (version <> ${lease.version} OR ordinal >= ${chunks.length})`; + yield* upsertFileChunks(sql, chunks); + const updated = yield* sql<{ id: string }>` + UPDATE files + SET index_state = 'ready', indexed_version = ${lease.version}, + index_cursor = ${chunks.length}, index_attempts = 0, + index_error = NULL, index_next_run_at = NULL, + index_lease_token = NULL + WHERE ${leaseUpdateFilter(sql, lease)} + RETURNING id`; + if (updated.length !== 1) return yield* new StaleIndexLease(); + }) + ) + .pipe(staleAsFalse); + +// Chunks are only written inside semanticCommit, so a failed attempt has +// nothing to clean up; recording the disposition releases the lease. +export const recordIndexFailure = ( + sql: PgClient.PgClient, + lease: IndexLease, + failure: IndexFailure +) => + sql<{ id: string }>` + UPDATE files + SET index_state = ${failure.state}, index_error = ${failure.error}, + index_next_run_at = ${failure.nextRunAt}, index_lease_token = NULL + WHERE ${leaseUpdateFilter(sql, lease)} + RETURNING id`.pipe(Effect.map((rows) => rows.length === 1)); diff --git a/apps/web/src/lib/server/layer.ts b/apps/web/src/lib/server/layer.ts index 800e176..7dfb6cf 100644 --- a/apps/web/src/lib/server/layer.ts +++ b/apps/web/src/lib/server/layer.ts @@ -34,7 +34,11 @@ export const requestLayer = (env: Env) => { const pg = PgLive.pipe(Layer.provide(bindings)); const blobs = BlobsLive.pipe(Layer.provide(bindings)); const infrastructure = Layer.mergeAll(bindings, sql, pg, blobs); - const semantic = SemanticBindingsLive(env); + // The vector index reads and writes file_chunks, so it sits on Postgres + // like every other service; only the embedder still binds Workers AI. + const semantic = SemanticBindingsLive(env).pipe( + Layer.provide(infrastructure) + ); const auth = AuthLive.pipe(Layer.provide(infrastructure)); const authGuard = AuthGuardLive().pipe(Layer.provide(bindings)); const grantSecrets = GrantSecretsLive.pipe(Layer.provide(infrastructure)); diff --git a/apps/web/src/lib/server/search-candidates.ts b/apps/web/src/lib/server/search-candidates.ts index 9b9d9c6..c0cb3e1 100644 --- a/apps/web/src/lib/server/search-candidates.ts +++ b/apps/web/src/lib/server/search-candidates.ts @@ -1,64 +1,75 @@ -export type SearchIndex = 'files_fts' | 'files_trgm'; +import type { PgClient } from '@effect/sql-pg'; // Candidate pool for one ranking pass. Search pages 50 at a time out of // this list; keep it large enough for a few pages without scanning the // whole index. export const SEARCH_CANDIDATE_LIMIT = 200; -export interface SearchCommand { - readonly sql: string; - readonly bindings: ReadonlyArray; +// word_similarity scores how well the query matches any substring of the +// name, so a typo ("reprot") still finds "Quarterly report.pdf". Kept in +// code rather than pg_trgm's session threshold so the cutoff is explicit. +export const TRIGRAM_THRESHOLD = 0.3; + +export interface RankedRow { + readonly file_id: string; + readonly score: number; +} + +export interface CandidateFilter { + readonly now: string; + readonly tagIds: ReadonlyArray; } -const selectedTagFilter = (fileAlias: string, tagIds: ReadonlyArray) => +export const selectedTagFilter = ( + sql: PgClient.PgClient, + tagIds: ReadonlyArray +) => tagIds.length === 0 - ? '' - : `AND EXISTS ( + ? sql`` + : sql`AND EXISTS ( SELECT 1 FROM file_tags selected - WHERE selected.file_id = ${fileAlias}.id - AND selected.tag_id IN (${tagIds.map(() => '?').join(', ')}) + WHERE selected.file_id = f.id + AND selected.tag_id = ANY(${tagIds}::text[]) )`; -export const rankedSearchCommand = ( - index: SearchIndex, - match: string, - now: string, - tagIds: ReadonlyArray -): SearchCommand => { - const score = - index === 'files_fts' - ? 'bm25(files_fts, 10.0, 5.0, 1.0)' - : 'bm25(files_trgm)'; - return { - sql: `SELECT ${index}.file_id, ${score} AS score - FROM ${index} - JOIN files f ON f.id = ${index}.file_id - WHERE ${index} MATCH ? - AND f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - ${selectedTagFilter('f', tagIds)} - ORDER BY score ASC, ${index}.file_id - LIMIT ${SEARCH_CANDIDATE_LIMIT}`, - bindings: [match, now, ...tagIds] - }; -}; +const visibleFile = (sql: PgClient.PgClient, filter: CandidateFilter) => + sql`f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${filter.now}) + ${selectedTagFilter(sql, filter.tagIds)}`; -export const eligibleSemanticCommand = ( - fileIds: ReadonlyArray, - now: string, - tagIds: ReadonlyArray -): SearchCommand => ({ - sql: `WITH requested AS ( - SELECT CAST(key AS INTEGER) AS ordinal, CAST(value AS TEXT) AS file_id - FROM json_each(?) - ) - SELECT requested.file_id, requested.ordinal - FROM requested - JOIN files f ON f.id = requested.file_id - WHERE f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - ${selectedTagFilter('f', tagIds)} - ORDER BY requested.ordinal - LIMIT ${SEARCH_CANDIDATE_LIMIT}`, - bindings: [JSON.stringify(fileIds), now, ...tagIds] -}); +// Names and tags are indexed with the `simple` dictionary and bodies with +// `english`, so the query is parsed both ways and OR-ed: "Quarterly" must +// match the name token `quarterly` and the body stem `quarter`. +export const fullTextCandidates = ( + sql: PgClient.PgClient, + query: string, + filter: CandidateFilter +) => + sql` + SELECT d.file_id, MAX(ts_rank_cd('{0.1, 0.2, 0.4, 1.0}', d.tsv, q.query)) AS score + FROM search_documents d + JOIN files f ON f.id = d.file_id + CROSS JOIN ( + SELECT websearch_to_tsquery('simple', ${query}) + || websearch_to_tsquery('english', ${query}) AS query + ) q + WHERE d.tsv @@ q.query + AND ${visibleFile(sql, filter)} + GROUP BY d.file_id + ORDER BY score DESC, d.file_id + LIMIT ${SEARCH_CANDIDATE_LIMIT}`; + +export const trigramCandidates = ( + sql: PgClient.PgClient, + query: string, + filter: CandidateFilter +) => + sql` + SELECT d.file_id, word_similarity(${query}, d.name) AS score + FROM search_documents d + JOIN files f ON f.id = d.file_id + WHERE d.chunk_no = 0 + AND word_similarity(${query}, d.name) > ${TRIGRAM_THRESHOLD}::real + AND ${visibleFile(sql, filter)} + ORDER BY score DESC, d.file_id + LIMIT ${SEARCH_CANDIDATE_LIMIT}`; diff --git a/apps/web/src/lib/server/search-ranking.ts b/apps/web/src/lib/server/search-ranking.ts index 695c1bd..2ecd27d 100644 --- a/apps/web/src/lib/server/search-ranking.ts +++ b/apps/web/src/lib/server/search-ranking.ts @@ -12,19 +12,6 @@ export interface FusedFile extends RankedFile { readonly ranks: Readonly>; } -export const sanitizeMatchQuery = (value: string) => { - const terms = value - .normalize('NFKC') - .match(/[\p{L}\p{N}]+/gu) - ?.map((term) => term.slice(0, 64)) - .filter(Boolean) - .slice(0, 20); - if (!terms?.length) return null; - return terms - .map((term, index) => `"${term}"${index === terms.length - 1 ? '*' : ''}`) - .join(' '); -}; - export const normalizedSearchText = (value: string) => value .normalize('NFKC') @@ -32,23 +19,20 @@ export const normalizedSearchText = (value: string) => .replace(/[^\p{L}\p{N}]+/gu, ' ') .trim(); +// Embeddings and trigram matching both need a few real characters before +// they say anything useful; shorter queries run keyword search only. +const MIN_FUZZY_QUERY_LENGTH = 3; + export const shouldEmbedSearchQuery = (value: string) => - normalizedSearchText(value).length >= 3; + normalizedSearchText(value).length >= MIN_FUZZY_QUERY_LENGTH; -export const sanitizeTrigramQuery = (value: string) => { - const normalized = normalizedSearchText(value); - if (normalized.length < 3) return null; +export const shouldFuzzyMatchQuery = shouldEmbedSearchQuery; - const trigrams = new Set(); - for (let index = 0; index <= normalized.length - 3; index += 1) { - const trigram = normalized.slice(index, index + 3); - if (trigram.trim().length === 3) trigrams.add(trigram); - if (trigrams.size === 32) break; - } - return trigrams.size - ? [...trigrams].map((trigram) => `"${trigram}"`).join(' OR ') - : null; -}; +// Postgres websearch_to_tsquery handles quoting and operators itself, so +// the only guard left is "is there anything to search for": a query of pure +// punctuation falls back to the recent-files listing. +export const hasSearchableQuery = (value: string) => + normalizedSearchText(value).length > 0; export const reciprocalRankFusion = ( sources: Readonly>, diff --git a/apps/web/src/lib/server/semantic-policy.ts b/apps/web/src/lib/server/semantic-policy.ts index 8e30a41..d2d0403 100644 --- a/apps/web/src/lib/server/semantic-policy.ts +++ b/apps/web/src/lib/server/semantic-policy.ts @@ -50,36 +50,6 @@ export const newIndexLeaseToken = () => { .replaceAll('=', ''); }; -export const vectorIdForChunk = ( - fileId: string, - leaseToken: string, - ordinal: number -) => `${fileId}:${leaseToken}:${ordinal.toString(36)}`; - -export const fileIdFromVectorId = (vectorId: string) => { - const last = vectorId.lastIndexOf(':'); - if (last < 0) return null; - const version = vectorId.lastIndexOf(':', last - 1); - return version < 0 ? null : vectorId.slice(0, version); -}; - -export const collapseVectorMatches = ( - matches: ReadonlyArray<{ readonly id: string; readonly score: number }> -) => { - const best = new Map(); - for (const match of matches) { - const fileId = fileIdFromVectorId(match.id); - if (!fileId) continue; - best.set(fileId, Math.max(best.get(fileId) ?? -Infinity, match.score)); - } - return [...best.entries()] - .sort( - ([leftId, leftScore], [rightId, rightScore]) => - rightScore - leftScore || leftId.localeCompare(rightId) - ) - .map(([fileId]) => ({ fileId })); -}; - export const retryAt = (attempt: number, now = new Date()) => new Date( now.getTime() + Math.min(60 * 2 ** Math.max(0, attempt - 1), 3_600) * 1_000 diff --git a/apps/web/src/lib/server/services/indexing.ts b/apps/web/src/lib/server/services/indexing.ts index 126193e..e7bf51c 100644 --- a/apps/web/src/lib/server/services/indexing.ts +++ b/apps/web/src/lib/server/services/indexing.ts @@ -1,35 +1,33 @@ import { Cause, Context, Effect, Layer, Schema } from 'effect'; import { StorageError } from '../errors'; import { - claimIndexCommand, - extractedTextCommands, - finishKeywordOnlyCommand, - indexFailureCommands, - semanticCommitCommands, - type IndexLease, - type SqlCommand, - vectorDeleteFailureCommand, - vectorDeleteSuccessCommand + claimIndex, + finishKeywordOnly, + recordIndexFailure, + semanticCommit, + storeExtractedText, + type IndexLease } from '../indexing-sql'; import { MAX_INDEX_ATTEMPTS, chunkSearchText, indexFailureDisposition, newIndexLeaseToken, - retryAt, - safeIndexError, - vectorIdForChunk + safeIndexError } from '../semantic-policy'; import { isSearchableText, searchTextLimit } from '../search-text'; import { createObjectTtlCache } from '../isolate-cache'; +import { PgSql } from '../pg'; import { Blobs } from './blobs'; -import { Db } from './bindings'; import { Embedder, VectorIndex } from './semantic'; const INDEX_LEASE_MS = 5 * 60 * 1_000; const EMBEDDING_MODEL = '@cf/baai/bge-small-en-v1.5'; const SEMANTIC_STATUS_CACHE_TTL_MS = 10_000; -const semanticStatusCache = createObjectTtlCache( +// The Postgres client is built per request, so the cache is keyed by a +// module-scope sentinel: there is exactly one database per deployment. +const semanticStatusKey = {}; +const semanticStatusCache = createObjectTtlCache( SEMANTIC_STATUS_CACHE_TTL_MS ); @@ -39,7 +37,6 @@ const IndexJobRow = Schema.Struct({ content_type: Schema.String, kind: Schema.String, current_version: Schema.Int, - is_public: Schema.Int, r2_key: Schema.String, text_content: Schema.NullOr(Schema.String), index_state: Schema.String, @@ -48,15 +45,6 @@ const IndexJobRow = Schema.Struct({ index_next_run_at: Schema.NullOr(Schema.String) }); -const PendingVectorRow = Schema.Struct({ - vector_id: Schema.String, - attempts: Schema.Int -}); - -const CountRow = Schema.Struct({ - count: Schema.Int -}); - const decodeRows = (schema: Schema.Codec, rows: unknown) => { const decoded = Schema.decodeUnknownOption(Schema.Array(schema))(rows); return decoded._tag === 'Some' ? decoded.value : []; @@ -74,9 +62,6 @@ export interface IndexingShape { readonly enqueue: (fileId: string) => Effect.Effect; readonly process: (fileId: string) => Effect.Effect; readonly runDue: (limit: number) => Effect.Effect; - readonly retryVectorDeletes: ( - limit: number - ) => Effect.Effect; readonly status: Effect.Effect; } @@ -85,74 +70,39 @@ export class Indexing extends Context.Service()( ) {} const makeIndexing = Effect.gen(function* () { - const db = yield* Db; + const sql = yield* PgSql; const blobs = yield* Blobs; const embedder = yield* Embedder; const vectors = yield* VectorIndex; - const prepareCommand = (command: SqlCommand) => - db.prepare(command.sql).bind(...command.bindings); - - const all = Effect.fn('Indexing.all')(function* ( - statement: D1PreparedStatement, - schema: Schema.Codec, - operation: string - ) { - const rows = yield* Effect.tryPromise({ - try: async () => { - const result = await statement.all(); - if (!result.success) throw new Error(result.error ?? operation); - return result.results; - }, - catch: (cause) => new StorageError({ operation, cause }) - }); - return decodeRows(schema, rows); - }); + const storage = (operation: string) => (cause: unknown) => + new StorageError({ operation, cause }); const findJob = Effect.fn('Indexing.findJob')(function* (fileId: string) { - const rows = yield* all( - db - .prepare( - `SELECT - f.id, f.display_name, f.content_type, f.kind, f.current_version, - f.public AS is_public, v.r2_key, v.text_content, f.index_state, - f.index_cursor, f.index_attempts, f.index_next_run_at - FROM files f - JOIN file_versions v - ON v.file_id = f.id AND v.version = f.current_version - WHERE f.id = ? AND f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ?) - LIMIT 1` - ) - .bind(fileId, new Date().toISOString()), - IndexJobRow, - 'find indexing job' - ); - return rows[0] ?? null; + const rows = yield* sql` + SELECT + f.id, f.display_name, f.content_type, f.kind, f.current_version, + v.r2_key, v.text_content, f.index_state, + f.index_cursor, f.index_attempts, f.index_next_run_at + FROM files f + JOIN file_versions v + ON v.file_id = f.id AND v.version = f.current_version + WHERE f.id = ${fileId} AND f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) + LIMIT 1`.pipe(Effect.mapError(storage('find indexing job'))); + return decodeRows(IndexJobRow, rows)[0] ?? null; }); const markFailure = Effect.fn('Indexing.markFailure')(function* ( lease: IndexLease, - vectorIds: ReadonlyArray, cause: unknown ) { const disposition = indexFailureDisposition(lease.attempt); const error = safeIndexError(cause); - const commands = indexFailureCommands( - lease, - vectorIds, - { ...disposition, error }, - new Date().toISOString() - ); - const results = yield* Effect.tryPromise({ - try: () => db.batch(commands.map(prepareCommand)), - catch: (failure) => - new StorageError({ - operation: 'record indexing failure', - cause: failure - }) - }); - const stateChanged = results.at(-1)?.meta.changes === 1; + const stateChanged = yield* recordIndexFailure(sql, lease, { + ...disposition, + error + }).pipe(Effect.mapError(storage('record indexing failure'))); console.error( JSON.stringify({ message: !stateChanged @@ -168,61 +118,6 @@ const makeIndexing = Effect.gen(function* () { ); }); - const retryVectorDeletes = Effect.fn('Indexing.retryVectorDeletes')( - function* (limit: number) { - if (!vectors.enabled) return 0; - const bounded = Math.max(1, Math.min(limit, 100)); - const now = new Date().toISOString(); - const rows = yield* all( - db - .prepare( - `SELECT vector_id, attempts - FROM pending_vector_deletes - WHERE next_run_at IS NULL OR next_run_at <= ? - ORDER BY queued_at, vector_id - LIMIT ?` - ) - .bind(now, bounded), - PendingVectorRow, - 'list pending vector deletes' - ); - if (rows.length === 0) return 0; - const ids = rows.map((row) => row.vector_id); - const removed = yield* vectors.remove(ids).pipe( - Effect.as(true), - Effect.catch((failure) => - Effect.tryPromise({ - try: () => - prepareCommand( - vectorDeleteFailureCommand( - rows.map((row) => ({ - vectorId: row.vector_id, - nextRunAt: retryAt(row.attempts + 1) - })), - safeIndexError(failure.cause) - ) - ).run(), - catch: (cause) => - new StorageError({ - operation: 'record vector cleanup failure', - cause - }) - }).pipe(Effect.as(false)) - ) - ); - if (!removed) return 0; - yield* Effect.tryPromise({ - try: () => prepareCommand(vectorDeleteSuccessCommand(ids)).run(), - catch: (cause) => - new StorageError({ - operation: 'finish vector cleanup', - cause - }) - }); - return ids.length; - } - ); - const perform = Effect.fn('Indexing.perform')(function* (fileId: string) { const initial = yield* findJob(fileId); if (!initial) return; @@ -234,22 +129,27 @@ const makeIndexing = Effect.gen(function* () { attempt: initial.index_attempts + 1, token: newIndexLeaseToken() } satisfies IndexLease; - const claim = yield* Effect.tryPromise({ - try: () => - prepareCommand( - claimIndexCommand( - lease, - now.toISOString(), - leaseUntil, - MAX_INDEX_ATTEMPTS - ) - ).run(), - catch: (cause) => - new StorageError({ operation: 'claim indexing job', cause }) - }); - if (claim.meta.changes !== 1) return; + const claimed = yield* claimIndex( + sql, + lease, + now.toISOString(), + leaseUntil, + MAX_INDEX_ATTEMPTS + ).pipe(Effect.mapError(storage('claim indexing job'))); + if (!claimed) return; + + const stale = (message: string) => + Effect.sync(() => { + console.log( + JSON.stringify({ + message, + fileId, + version: lease.version, + attempt: lease.attempt + }) + ); + }); - let attemptVectorIds: ReadonlyArray = []; yield* Effect.gen(function* () { const text = initial.index_cursor >= 1 && initial.text_content !== null @@ -259,45 +159,19 @@ const makeIndexing = Effect.gen(function* () { ? yield* blobs.readTextPrefix(initial.r2_key, searchTextLimit) : ''; - const extracted = yield* Effect.tryPromise({ - try: () => - db.batch(extractedTextCommands(lease, text).map(prepareCommand)), - catch: (cause) => - new StorageError({ - operation: 'store extracted search text', - cause - }) - }); - if (extracted[1]?.meta.changes !== 1) { - console.log( - JSON.stringify({ - message: 'stale semantic extraction ignored', - fileId, - version: lease.version, - attempt: lease.attempt - }) - ); - return; + const extracted = yield* storeExtractedText(sql, lease, text).pipe( + Effect.mapError(storage('store extracted search text')) + ); + if (!extracted) { + return yield* stale('stale semantic extraction ignored'); } if (!embedder.enabled || !vectors.enabled) { - const finished = yield* Effect.tryPromise({ - try: () => prepareCommand(finishKeywordOnlyCommand(lease)).run(), - catch: (cause) => - new StorageError({ - operation: 'finish keyword-only indexing', - cause - }) - }); - if (finished.meta.changes !== 1) { - console.log( - JSON.stringify({ - message: 'stale keyword-only indexing completion ignored', - fileId, - version: lease.version, - attempt: lease.attempt - }) - ); + const finished = yield* finishKeywordOnly(sql, lease).pipe( + Effect.mapError(storage('finish keyword-only indexing')) + ); + if (!finished) { + yield* stale('stale keyword-only indexing completion ignored'); } return; } @@ -313,63 +187,24 @@ const makeIndexing = Effect.gen(function* () { }); } - const vectorRecords = chunks.map((chunk, index) => ({ - id: vectorIdForChunk(fileId, lease.token, chunk.ordinal), - values: embeddings[index] ?? [], - metadata: { + const committed = yield* semanticCommit( + sql, + lease, + chunks.map((chunk, index) => ({ fileId, version: lease.version, - deleted: false, - kind: initial.kind, - visibility: initial.is_public === 1 ? 'public' : 'private' - } - })); - attemptVectorIds = vectorRecords.map((record) => record.id); - yield* vectors.upsert(vectorRecords); - - const results = yield* Effect.tryPromise({ - try: () => - db.batch( - semanticCommitCommands( - lease, - chunks, - attemptVectorIds, - new Date().toISOString() - ).map(prepareCommand) - ), - catch: (cause) => - new StorageError({ - operation: 'commit semantic index state', - cause - }) - }); - if (results.at(-1)?.meta.changes !== 1) { - console.log( - JSON.stringify({ - message: 'stale semantic indexing completion cleaned up', - fileId, - version: lease.version, - attempt: lease.attempt - }) - ); + ordinal: chunk.ordinal, + charStart: chunk.charStart, + charEnd: chunk.charEnd, + values: embeddings[index] ?? [] + })) + ).pipe(Effect.mapError(storage('commit semantic index state'))); + if (!committed) { + yield* stale('stale semantic indexing completion rolled back'); } - - yield* retryVectorDeletes(100).pipe( - Effect.catchCause((cause) => - Effect.sync(() => { - console.error( - JSON.stringify({ - message: 'vector cleanup remains pending', - fileId, - cause: String(cause) - }) - ); - }) - ) - ); }).pipe( Effect.catchCause((cause) => - markFailure(lease, attemptVectorIds, Cause.pretty(cause)).pipe( + markFailure(lease, Cause.pretty(cause)).pipe( Effect.catchCause((recordCause) => Effect.sync(() => { console.error( @@ -407,89 +242,55 @@ const makeIndexing = Effect.gen(function* () { const runDue = Effect.fn('Indexing.runDue')(function* (limit: number) { const bounded = Math.max(1, Math.min(limit, 10)); const now = new Date().toISOString(); - const includeDisabled = embedder.enabled && vectors.enabled ? 1 : 0; - const rows = yield* Effect.tryPromise({ - try: async () => { - const result = await db - .prepare( - `SELECT id - FROM files - WHERE deleted_at IS NULL - AND (expires_at IS NULL OR expires_at > ?) - AND index_attempts < ? - AND ( - (index_state = 'pending' - AND (index_next_run_at IS NULL OR index_next_run_at <= ?)) - OR (index_state = 'running' AND index_next_run_at <= ?) - OR (index_state = 'disabled' AND ? = 1) - ) - ORDER BY COALESCE(index_next_run_at, updated_at), id - LIMIT ?` - ) - .bind(now, MAX_INDEX_ATTEMPTS, now, now, includeDisabled, bounded) - .all<{ id: string }>(); - if (!result.success) - throw new Error(result.error ?? 'Index job listing failed'); - return result.results; - }, - catch: (cause) => - new StorageError({ operation: 'list due indexing jobs', cause }) - }); + const includeDisabled = embedder.enabled && vectors.enabled; + const rows = yield* sql<{ id: string }>` + SELECT id + FROM files + WHERE deleted_at IS NULL + AND (expires_at IS NULL OR expires_at > ${now}) + AND index_attempts < ${MAX_INDEX_ATTEMPTS} + AND ( + (index_state = 'pending' + AND (index_next_run_at IS NULL OR index_next_run_at <= ${now})) + OR (index_state = 'running' AND index_next_run_at <= ${now}) + OR (index_state = 'disabled' AND ${includeDisabled}::boolean) + ) + ORDER BY COALESCE(index_next_run_at, updated_at), id + LIMIT ${bounded}`.pipe(Effect.mapError(storage('list due indexing jobs'))); for (const row of rows) yield* process(row.id); return rows.length; }); const enqueue = Effect.fn('Indexing.enqueue')(function* (fileId: string) { - yield* Effect.tryPromise({ - try: () => - db - .prepare( - `UPDATE files - SET index_state = 'pending', index_cursor = 0, index_attempts = 0, - index_error = NULL, index_next_run_at = NULL, - index_lease_token = NULL - WHERE id = ? AND deleted_at IS NULL - AND (expires_at IS NULL OR expires_at > ?)` - ) - .bind(fileId, new Date().toISOString()) - .run(), - catch: (cause) => - new StorageError({ operation: 'enqueue semantic indexing', cause }) - }); + yield* sql` + UPDATE files + SET index_state = 'pending', index_cursor = 0, index_attempts = 0, + index_error = NULL, index_next_run_at = NULL, + index_lease_token = NULL + WHERE id = ${fileId} AND deleted_at IS NULL + AND (expires_at IS NULL OR expires_at > ${new Date().toISOString()})`.pipe( + Effect.mapError(storage('enqueue semantic indexing')) + ); }); const status = Effect.gen(function* () { - const cached = semanticStatusCache.get(db); - if (cached) { - return { - ...cached, - enabled: embedder.enabled && vectors.enabled - }; - } - const rows = yield* all( - db.prepare('SELECT COUNT(*) AS count FROM file_chunks'), - CountRow, - 'count indexed chunks' - ); + const enabled = embedder.enabled && vectors.enabled; + const cached = semanticStatusCache.get(semanticStatusKey); + if (cached) return { ...cached, enabled }; + const indexedChunks = yield* vectors.count; const result = { - enabled: embedder.enabled && vectors.enabled, - indexedChunks: rows[0]?.count ?? 0, + enabled, + indexedChunks, dimensions: 384, model: EMBEDDING_MODEL, costNotice: - 'Vectorize bills each query against stored vectors × 384 dimensions; keyword search stays available when semantic search is off.' + 'Embeddings are stored in Postgres (pgvector) beside the file rows; each indexed chunk costs 384 floats plus its HNSW index entry, and keyword search stays available when semantic search is off.' }; - semanticStatusCache.set(db, result); + semanticStatusCache.set(semanticStatusKey, result); return result; }).pipe(Effect.withSpan('Indexing.status')); - return Indexing.of({ - enqueue, - process, - runDue, - retryVectorDeletes, - status - }); + return Indexing.of({ enqueue, process, runDue, status }); }); export const IndexingLive = Layer.effect(Indexing, makeIndexing); diff --git a/apps/web/src/lib/server/services/lifecycle.ts b/apps/web/src/lib/server/services/lifecycle.ts index 2f4ea11..5e6b326 100644 --- a/apps/web/src/lib/server/services/lifecycle.ts +++ b/apps/web/src/lib/server/services/lifecycle.ts @@ -9,7 +9,6 @@ export interface LifecycleSummary { readonly sites: number; readonly indexing: number; readonly files: number; - readonly vectors: number; } export interface LifecycleShape { @@ -25,7 +24,6 @@ export interface LifecycleTasks { readonly sites: Effect.Effect; readonly indexing: Effect.Effect; readonly files: Effect.Effect; - readonly vectors: Effect.Effect; } const recover = ( @@ -58,8 +56,7 @@ export const runLifecycleTasks = (tasks: LifecycleTasks) => const sites = yield* recover('sites', tasks.sites, 0); const indexing = yield* recover('indexing', tasks.indexing, 0); const files = yield* recover('files', tasks.files, 0); - const vectors = yield* recover('vectors', tasks.vectors, 0); - return { authentication, sites, indexing, files, vectors }; + return { authentication, sites, indexing, files }; }); const makeLifecycle = Effect.gen(function* () { @@ -90,8 +87,7 @@ const makeLifecycle = Effect.gen(function* () { ).pipe(Effect.map(([revoked, swept]) => revoked + swept)), sites: sites.sweepLifecycle(10), indexing: indexing.runDue(5), - files: files.sweepPurges(5), - vectors: indexing.retryVectorDeletes(100) + files: files.sweepPurges(5) }).pipe(Effect.withSpan('Lifecycle.run')); return Lifecycle.of({ run }); diff --git a/apps/web/src/lib/server/services/search.ts b/apps/web/src/lib/server/services/search.ts index c243fee..5bddd50 100644 --- a/apps/web/src/lib/server/services/search.ts +++ b/apps/web/src/lib/server/services/search.ts @@ -1,5 +1,6 @@ import type { DashboardFile } from '@adrive/shared'; import { Context, Effect, Layer } from 'effect'; +import type { SqlError } from 'effect/unstable/sql/SqlError'; import { InvalidRequest, StorageError } from '../errors'; import { dashboardFileColumns, @@ -7,27 +8,22 @@ import { toDashboardFile } from '../file-rows'; import { + hasSearchableQuery, pinExactName, reciprocalRankFusion, - sanitizeMatchQuery, - sanitizeTrigramQuery, - shouldEmbedSearchQuery + shouldEmbedSearchQuery, + shouldFuzzyMatchQuery } from '../search-ranking'; import { - eligibleSemanticCommand, - rankedSearchCommand, + fullTextCandidates, SEARCH_CANDIDATE_LIMIT, - type SearchCommand + selectedTagFilter, + trigramCandidates, + type RankedRow } from '../search-candidates'; import { PgSql } from '../pg'; -import { Db } from './bindings'; import { Embedder, VectorIndex } from './semantic'; -interface RankedRow { - readonly file_id: string; - readonly score: number; -} - export interface SearchInput { readonly query: string; readonly tagIds: ReadonlyArray; @@ -63,61 +59,20 @@ const decodeCursor = (cursor: string | null | undefined) => { : null; }; +const noCandidates = Effect.succeed>([]); + const makeSearch = Effect.gen(function* () { - const db = yield* Db; - // Files rows live in Postgres now; the FTS and trigram indexes still - // live in D1 until the keyword search rewrite moves them over. const sql = yield* PgSql; const embedder = yield* Embedder; const vectorIndex = yield* VectorIndex; - const selectedTagFilter = (tagIds: ReadonlyArray) => - tagIds.length === 0 - ? sql`` - : sql`AND EXISTS ( - SELECT 1 FROM file_tags selected - WHERE selected.file_id = f.id - AND ${sql.in('selected.tag_id', tagIds)} - )`; - const runRanked = ( - command: SearchCommand - ): Effect.Effect, StorageError> => - Effect.tryPromise({ - try: async () => { - const result = await db - .prepare(command.sql) - .bind(...command.bindings) - .all(); - if (!result.success) throw new Error(result.error ?? 'Search failed'); - return result.results; - }, - catch: (cause) => new StorageError({ operation: 'search index', cause }) - }); - - const filterSemantic = Effect.fn('Search.filterSemantic')(function* ( - fileIds: ReadonlyArray, - tagIds: ReadonlyArray, - now: string - ) { - if (fileIds.length === 0) return []; - const command = eligibleSemanticCommand(fileIds, now, tagIds); - return yield* Effect.tryPromise({ - try: async () => { - const result = await db - .prepare(command.sql) - .bind(...command.bindings) - .all<{ file_id: string; ordinal: number }>(); - if (!result.success) - throw new Error(result.error ?? 'Semantic filtering failed'); - return result.results.map((row) => ({ fileId: row.file_id })); - }, - catch: (cause) => - new StorageError({ - operation: 'filter semantic search candidates', - cause - }) - }); - }); + const ranked = ( + operation: string, + candidates: Effect.Effect, SqlError> + ) => + candidates.pipe( + Effect.mapError((cause) => new StorageError({ operation, cause })) + ); const hydrate = Effect.fn('Search.hydrate')(function* ( fileIds: ReadonlyArray, @@ -130,7 +85,7 @@ const makeSearch = Effect.gen(function* () { WHERE ${sql.in('f.id', fileIds)} AND f.deleted_at IS NULL AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) - ${selectedTagFilter(tagIds)}`.pipe( + ${selectedTagFilter(sql, tagIds)}`.pipe( Effect.mapError( (cause) => new StorageError({ operation: 'hydrate search results', cause }) @@ -157,7 +112,7 @@ const makeSearch = Effect.gen(function* () { FROM files f WHERE f.deleted_at IS NULL AND (f.expires_at IS NULL OR f.expires_at > ${new Date().toISOString()}) - ${selectedTagFilter(tagIds)} + ${selectedTagFilter(sql, tagIds)} ORDER BY f.updated_at DESC, f.id LIMIT ${limit} OFFSET ${offset}`.pipe( Effect.mapError( @@ -179,8 +134,7 @@ const makeSearch = Effect.gen(function* () { const offset = page * PAGE_SIZE; const selectedTagIds = [...new Set(tagIds)].slice(0, 20); const trimmedQuery = query.trim().slice(0, 256); - const keywordMatch = sanitizeMatchQuery(trimmedQuery); - if (!keywordMatch) { + if (!hasSearchableQuery(trimmedQuery)) { const recent = yield* filteredRecent( selectedTagIds, PAGE_SIZE + 1, @@ -195,30 +149,30 @@ const makeSearch = Effect.gen(function* () { }; } - const trigramMatch = sanitizeTrigramQuery(trimmedQuery); - const now = new Date().toISOString(); - // The index reads (FTS, trigram) and the optional embedding + + const filter = { now: new Date().toISOString(), tagIds: selectedTagIds }; + // The index reads (full text, trigram) and the optional embedding + // vector query are independent; run them concurrently so search // latency is the slowest source, not their sum. Workers AI - // embeddings are usually the slowest leg, so they start now. - const [keyword, trigram, semanticCandidates] = yield* Effect.all( + // embeddings are usually the slowest leg, so they start now. The + // vector query filters visibility and tags itself, so its rows + // go straight into fusion. + const [keyword, trigram, semantic] = yield* Effect.all( [ - runRanked( - rankedSearchCommand('files_fts', keywordMatch, now, selectedTagIds) + ranked( + 'keyword search', + fullTextCandidates(sql, trimmedQuery, filter) ), - trigramMatch - ? runRanked( - rankedSearchCommand( - 'files_trgm', - trigramMatch, - now, - selectedTagIds - ) + shouldFuzzyMatchQuery(trimmedQuery) + ? ranked( + 'trigram search', + trigramCandidates(sql, trimmedQuery, filter) ) - : Effect.succeed>([]), + : noCandidates, shouldEmbedSearchQuery(trimmedQuery) ? embedder.query(trimmedQuery).pipe( - Effect.flatMap((embedding) => vectorIndex.search(embedding)), + Effect.flatMap((embedding) => + vectorIndex.search(embedding, filter) + ), Effect.catch((failure) => Effect.sync(() => { console.error( @@ -235,24 +189,6 @@ const makeSearch = Effect.gen(function* () { ], { concurrency: 'unbounded' } ); - const semantic = yield* filterSemantic( - semanticCandidates.map((candidate) => candidate.fileId), - selectedTagIds, - now - ).pipe( - Effect.catch((failure) => - Effect.sync(() => { - console.error( - JSON.stringify({ - message: - 'semantic candidate filtering degraded to keyword search', - operation: failure.operation - }) - ); - return []; - }) - ) - ); // Fuse the full candidate pool on every page so page 0 and // page 1 slice the same ranking instead of two different // prefixes (offset+51 vs offset+101). diff --git a/apps/web/src/lib/server/services/semantic.ts b/apps/web/src/lib/server/services/semantic.ts index 5194cd3..b6e111d 100644 --- a/apps/web/src/lib/server/services/semantic.ts +++ b/apps/web/src/lib/server/services/semantic.ts @@ -1,9 +1,14 @@ import { Context, Effect, Layer } from 'effect'; import { StorageError } from '../errors'; +import { upsertFileChunks, type VectorChunk } from '../indexing-sql'; +import { PgSql } from '../pg'; import { createQueryEmbeddingCache } from '../query-embedding-cache'; -import { collapseVectorMatches } from '../semantic-policy'; +import { selectedTagFilter, type CandidateFilter } from '../search-candidates'; const EMBEDDING_MODEL = '@cf/baai/bge-small-en-v1.5'; +// Vector candidates feed reciprocal rank fusion beside the keyword pool; +// one hundred nearest files is plenty for a few result pages. +const SEMANTIC_CANDIDATE_LIMIT = 100; export interface EmbedderShape { readonly enabled: boolean; @@ -15,28 +20,14 @@ export interface EmbedderShape { ) => Effect.Effect | null, StorageError>; } -export interface VectorRecord { - readonly id: string; - readonly values: ReadonlyArray; - readonly metadata: { - readonly fileId: string; - readonly version: number; - readonly deleted: boolean; - readonly kind: string; - readonly visibility: string; - }; -} - export interface VectorIndexShape { readonly enabled: boolean; readonly upsert: ( - vectors: ReadonlyArray - ) => Effect.Effect; - readonly remove: ( - vectorIds: ReadonlyArray + rows: ReadonlyArray ) => Effect.Effect; readonly search: ( - vector: ReadonlyArray | null + vector: ReadonlyArray | null, + filter: CandidateFilter ) => Effect.Effect, StorageError>; readonly count: Effect.Effect; } @@ -52,11 +43,12 @@ export class VectorIndex extends Context.Service< type SemanticBoundEnv = Env & { readonly AI: Ai; - readonly VECTORIZE: Vectorize; }; +// Vectors live in Postgres, which every request has, so the only optional +// binding left is Workers AI for the embeddings themselves. export const hasSemanticBindings = (env: Env): env is SemanticBoundEnv => - 'AI' in env && 'VECTORIZE' in env; + 'AI' in env && env.AI !== undefined; const embeddingData = (response: Ai_Cf_Baai_Bge_Small_En_V1_5_Output) => { if ( @@ -91,13 +83,67 @@ export const VectorIndexNull = Layer.succeed( VectorIndex.of({ enabled: false, upsert: () => Effect.void, - remove: () => Effect.void, search: () => Effect.succeed([]), count: Effect.succeed(0) }) ); -const liveLayers = (env: SemanticBoundEnv) => { +// pgvector reads the same bracketed text form it prints, so the query +// vector travels as one bound parameter cast on the server. +export const vectorLiteral = (vector: ReadonlyArray) => + `[${vector.join(',')}]`; + +export const makeVectorIndex = ( + sql: PgSql['Service'], + enabled: boolean +): VectorIndexShape => ({ + enabled, + upsert: (rows) => + upsertFileChunks(sql, rows).pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'upsert semantic vectors', cause }) + ) + ), + // The nearest chunk stands in for its file, and visibility and tags + // are filtered here so the caller never sees a file it cannot open. + search: (vector, filter) => + vector === null + ? Effect.succeed([]) + : sql<{ file_id: string }>` + SELECT c.file_id + FROM file_chunks c + JOIN files f ON f.id = c.file_id AND f.current_version = c.version + WHERE c.embedding IS NOT NULL + AND f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${filter.now}) + ${selectedTagFilter(sql, filter.tagIds)} + GROUP BY c.file_id + ORDER BY MIN(c.embedding <=> ${vectorLiteral(vector)}::vector), c.file_id + LIMIT ${SEMANTIC_CANDIDATE_LIMIT}`.pipe( + Effect.map((rows) => rows.map((row) => ({ fileId: row.file_id }))), + Effect.mapError( + (cause) => + new StorageError({ operation: 'query semantic vectors', cause }) + ) + ), + count: sql<{ count: number }>` + SELECT COUNT(*)::integer AS count FROM file_chunks WHERE embedding IS NOT NULL`.pipe( + Effect.map((rows) => rows[0]?.count ?? 0), + Effect.mapError( + (cause) => + new StorageError({ operation: 'count semantic vectors', cause }) + ) + ) +}); + +export const VectorIndexLive = (enabled: boolean) => + Layer.effect( + VectorIndex, + Effect.map(PgSql, (sql) => makeVectorIndex(sql, enabled)) + ); + +const embedderLive = (env: SemanticBoundEnv) => { const embeddings = createQueryEmbeddingCache(env.AUTH_GUARD); const embed = (values: ReadonlyArray) => Effect.tryPromise({ @@ -112,7 +158,7 @@ const liveLayers = (env: SemanticBoundEnv) => { new StorageError({ operation: 'generate embeddings', cause }) }); - const embedder = Layer.succeed( + return Layer.succeed( Embedder, Embedder.of({ enabled: true, @@ -132,78 +178,18 @@ const liveLayers = (env: SemanticBoundEnv) => { }) }) ); - - const vectorIndex = Layer.succeed( - VectorIndex, - VectorIndex.of({ - enabled: true, - upsert: (vectors) => - vectors.length === 0 - ? Effect.void - : Effect.tryPromise({ - try: () => - env.VECTORIZE.upsert( - vectors.map((vector) => ({ - id: vector.id, - values: [...vector.values], - metadata: vector.metadata - })) - ), - catch: (cause) => - new StorageError({ - operation: 'upsert semantic vectors', - cause - }) - }).pipe(Effect.asVoid), - remove: (vectorIds) => - vectorIds.length === 0 - ? Effect.void - : Effect.tryPromise({ - try: () => env.VECTORIZE.deleteByIds([...vectorIds]), - catch: (cause) => - new StorageError({ - operation: 'delete semantic vectors', - cause - }) - }).pipe(Effect.asVoid), - search: (vector) => - vector === null - ? Effect.succeed([]) - : Effect.tryPromise({ - try: () => - env.VECTORIZE.query([...vector], { - topK: 100, - returnMetadata: false - }), - catch: (cause) => - new StorageError({ - operation: 'query semantic vectors', - cause - }) - }).pipe( - Effect.map((result) => collapseVectorMatches(result.matches)) - ), - count: Effect.tryPromise({ - try: async () => (await env.VECTORIZE.describe()).vectorsCount, - catch: (cause) => - new StorageError({ operation: 'describe semantic index', cause }) - }) - }) - ); - - return Layer.merge(embedder, vectorIndex); }; export const SemanticBindingsLive = (env: Env) => { const mode = String(env.SEMANTIC_SEARCH); if (mode === 'off') { - return Layer.merge(EmbedderNull, VectorIndexNull); + return Layer.merge(EmbedderNull, VectorIndexLive(false)); + } + if (hasSemanticBindings(env)) { + return Layer.merge(embedderLive(env), VectorIndexLive(true)); } - if (hasSemanticBindings(env)) return liveLayers(env); if (mode === 'required') { - throw new Error( - 'SEMANTIC_SEARCH=required needs both AI and VECTORIZE bindings' - ); + throw new Error('SEMANTIC_SEARCH=required needs the AI binding'); } - return Layer.merge(EmbedderNull, VectorIndexNull); + return Layer.merge(EmbedderNull, VectorIndexLive(false)); }; diff --git a/apps/web/worker-configuration.d.ts b/apps/web/worker-configuration.d.ts index 54c6dd2..2429469 100644 --- a/apps/web/worker-configuration.d.ts +++ b/apps/web/worker-configuration.d.ts @@ -1,11 +1,10 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 53f8559271301eb2083d9952fb869d11) +// Generated by Wrangler by running `wrangler types` (hash: a28431120a2ba6281544be2aa23c9aa1) // Runtime types generated with workerd@1.20260722.1 2026-07-27 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { AUTH_GUARD: KVNamespace; BUCKET: R2Bucket; DB: D1Database; - VECTORIZE?: VectorizeIndex; HYPERDRIVE: Hyperdrive; BROWSER: BrowserRun; AI?: Ai; @@ -26,7 +25,6 @@ declare namespace Cloudflare { AUTH_GUARD: KVNamespace; BUCKET: R2Bucket; DB: D1Database; - VECTORIZE: VectorizeIndex; HYPERDRIVE: Hyperdrive; BROWSER: BrowserRun; AI: Ai; diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index aa58625..21b042e 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -70,10 +70,9 @@ // 1b. Create a Hyperdrive config pointing at PlanetScale Postgres // (direct port 5432, caching disabled) and paste its id below. // 2. Create the production R2 bucket and KV namespace; paste the KV id. - // 3. Create the Vectorize index and its metadata indexes (docs/release.md). - // 4. Point DNS for both hostnames at the Worker (routes below). - // 5. `wrangler secret put PASSCODE --env production` - // 6. `wrangler d1 migrations apply DB --env production --remote` + // 3. Point DNS for both hostnames at the Worker (routes below). + // 4. `wrangler secret put PASSCODE --env production` + // 5. `wrangler d1 migrations apply DB --env production --remote` "env": { "production": { // Place the Worker near D1 for dashboard/search queries. Content @@ -89,8 +88,8 @@ "CONTENT_ORIGIN": "https://files.davis7.space", "MAX_UPLOAD_BYTES": "99614720", "MAX_TOTAL_BYTES": "107374182400", - // required (not auto) so a missing AI/Vectorize binding fails - // the deploy loudly instead of silently degrading to keyword. + // required (not auto) so a missing AI binding fails the deploy + // loudly instead of silently degrading to keyword search. "SEMANTIC_SEARCH": "required", "EMBEDDING_MODEL": "@cf/baai/bge-small-en-v1.5", "EMBEDDING_POOLING": "cls", @@ -99,12 +98,6 @@ "ai": { "binding": "AI" }, - "vectorize": [ - { - "binding": "VECTORIZE", - "index_name": "adrive-production" - } - ], "triggers": { "crons": ["*/5 * * * *"] }, @@ -149,9 +142,9 @@ } } } - // Local dev runs keyword-only (SEMANTIC_SEARCH=auto, no AI/Vectorize - // bindings above). Production declares both bindings and uses - // SEMANTIC_SEARCH=required. To enable semantic search locally, create a - // 384-dimension cosine index and add the same two bindings to the top - // level, then run `bun --filter @adrive/web types:worker`. See README.md. + // Local dev runs keyword-only (SEMANTIC_SEARCH=auto, no AI binding + // above). Vectors live in Postgres (pgvector), so production only adds + // the AI binding and uses SEMANTIC_SEARCH=required. To enable semantic + // search locally, add the same "ai" binding to the top level, then run + // `bun --filter @adrive/web types:worker`. See README.md. } diff --git a/docs/release.md b/docs/release.md index 791a419..1f8b718 100644 --- a/docs/release.md +++ b/docs/release.md @@ -11,21 +11,12 @@ the repository root. 2. From `apps/web`: `wrangler r2 bucket create adrive-production` 3. From `apps/web`: `wrangler kv namespace create AUTH_GUARD --env production` — paste the id into `env.production.kv_namespaces[0].id`. -4. From `apps/web`: create the semantic-search index (the production env - sets `SEMANTIC_SEARCH=required`, so the deploy fails without it): - - ``` - wrangler vectorize create adrive-production --dimensions=384 --metric=cosine - wrangler vectorize create-metadata-index adrive-production --property-name=deleted --type=boolean - wrangler vectorize create-metadata-index adrive-production --property-name=kind --type=string - wrangler vectorize create-metadata-index adrive-production --property-name=visibility --type=string - ``` - - The Workers AI binding needs no provisioning — it activates with the - `ai` binding already declared in `wrangler.jsonc`. Both services sit - inside the Workers Paid plan's included allocation at personal scale - (50M queried + 10M stored vector dimensions per month ≈ 26k chunks at - 384 dims; embeddings run within the 10k neurons/day allocation). +4. Semantic search needs no extra provisioning: embeddings come from the + Workers AI `ai` binding already declared in `wrangler.jsonc`, and the + vectors live in the Postgres `file_chunks` table (pgvector). The + production env sets `SEMANTIC_SEARCH=required`, so the deploy fails + loudly if the `AI` binding is missing. Embeddings run within the + Workers Paid plan's included neuron allocation at personal scale. 5. In the Cloudflare dashboard, open **Images → Transformations**, select the zone that owns `CONTENT_ORIGIN` (`davis7.space` for @@ -50,10 +41,9 @@ Semantic search notes for the first deploy: sit in `index_state = 'disabled'` and are backfilled by the maintenance cron at 5 files per 5 minutes. A large pre-existing corpus takes hours; the settings page's "indexed chunks" count shows progress. -- Vectorize contents are derived state (like the FTS tables): after a D1 - restore, vectors for purged files are orphaned but harmless, and - missing vectors regenerate on reindex. They are deliberately not part - of the backup set. +- Embeddings live in Postgres beside the file rows, so they are restored + with the database. Files whose embeddings are missing after a partial + restore regenerate on reindex. ## Releasing From 345d464c6953e86a0b3b012a6bae71708709ddc3 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 02:41:05 -0700 Subject: [PATCH 5/7] REFACTOR: Remove D1 and cut over to Postgres Deletes the D1 binding, client, and migrations, wires the release script and backup host to Postgres (pg-migrate and pg_dump), and adds a one-off d1-to-postgres script that replays a D1 export into the new schema. Docs and the deploy skill describe the Hyperdrive setup instead of D1. Co-Authored-By: Claude Fable 5.1 --- .agents/skills/deploy-fresh-instance/SKILL.md | 14 +- apps/web/migrations/0001_init.sql | 142 ------------ apps/web/migrations/0002_tags_search.sql | 37 --- apps/web/migrations/0003_sites.sql | 44 ---- .../migrations/0004_auth_ttl_downloads.sql | 21 -- .../migrations/0005_semantic_lifecycle.sql | 18 -- apps/web/migrations/0006_index_leases.sql | 1 - apps/web/migrations/0007_instance_secrets.sql | 8 - apps/web/migrations/0008_api_key_scopes.sql | 5 - apps/web/migrations/0009_credential_state.sql | 7 - .../web/migrations/0010_thumbnail_storage.sql | 6 - apps/web/package.json | 1 - apps/web/scripts/d1-to-postgres.mjs | 213 ++++++++++++++++++ apps/web/src/lib/server/edge.ts | 1 - apps/web/src/lib/server/layer.ts | 10 +- apps/web/src/lib/server/services/bindings.ts | 2 - apps/web/worker-configuration.d.ts | 4 +- apps/web/wrangler.jsonc | 18 +- bun.lock | 9 - docs/backup-restore.md | 67 +++--- docs/release.md | 46 +++- package.json | 3 +- scripts/backup/backup.env.example | 9 +- scripts/backup/backup.sh | 98 +++----- scripts/backup/install-backup-host.sh | 2 +- scripts/check-wrangler-drift.mjs | 14 +- scripts/release.sh | 10 +- 27 files changed, 342 insertions(+), 468 deletions(-) delete mode 100644 apps/web/migrations/0001_init.sql delete mode 100644 apps/web/migrations/0002_tags_search.sql delete mode 100644 apps/web/migrations/0003_sites.sql delete mode 100644 apps/web/migrations/0004_auth_ttl_downloads.sql delete mode 100644 apps/web/migrations/0005_semantic_lifecycle.sql delete mode 100644 apps/web/migrations/0006_index_leases.sql delete mode 100644 apps/web/migrations/0007_instance_secrets.sql delete mode 100644 apps/web/migrations/0008_api_key_scopes.sql delete mode 100644 apps/web/migrations/0009_credential_state.sql delete mode 100644 apps/web/migrations/0010_thumbnail_storage.sql create mode 100644 apps/web/scripts/d1-to-postgres.mjs diff --git a/.agents/skills/deploy-fresh-instance/SKILL.md b/.agents/skills/deploy-fresh-instance/SKILL.md index 53d12b5..1fe3dd0 100644 --- a/.agents/skills/deploy-fresh-instance/SKILL.md +++ b/.agents/skills/deploy-fresh-instance/SKILL.md @@ -1,6 +1,6 @@ --- name: deploy-fresh-instance -description: Deploy a fresh a-drive instance to Cloudflare Workers — provision D1, R2, KV, and Postgres plus Workers AI, set the PASSCODE secret, attach the two custom domains, and cut the first release. Use for the first-ever deploy to a new Cloudflare account or a new operator domain (not for routine redeploys, which are just `bun release`). +description: Deploy a fresh a-drive instance to Cloudflare Workers — provision Postgres (PlanetScale + Hyperdrive), R2, and KV plus Workers AI, set the PASSCODE secret, attach the two custom domains, and cut the first release. Use for the first-ever deploy to a new Cloudflare account or a new operator domain (not for routine redeploys, which are just `bun release`). --- # Deploy a fresh a-drive instance to Cloudflare @@ -35,8 +35,10 @@ Wrangler runs **non-interactively** here: each create prints an id and exits. It does **not** edit `wrangler.jsonc` for you — you paste each id into `env.production` by hand. -1. `wrangler d1 create adrive-production` - → paste the printed id into `env.production.d1_databases[0].database_id`. +1. Create the PlanetScale Postgres database and a Hyperdrive config for it + (`wrangler hyperdrive create adrive-production --connection-string=... --caching-disabled`) + → paste the printed id into `env.production.hyperdrive[0].id`. Keep the + connection string; `bun release` needs it as `DATABASE_URL`. 2. `wrangler r2 bucket create adrive-production` (no id to paste — the bucket is bound by name.) 3. `wrangler kv namespace create AUTH_GUARD --env production` @@ -73,7 +75,7 @@ before the first deploy. ## 4. Commit the provisioned ids -The D1 `database_id` and KV `id` are **configuration, not secrets** — +The Hyperdrive `id` and KV `id` are **configuration, not secrets** — commit the edited `wrangler.jsonc`. Two `bun release` preflight gates depend on it: @@ -92,7 +94,7 @@ bun release ``` `scripts/release.sh` runs the full gate → format check → type/lint → -tests → audit → build → deploy dry-run → D1 migrations → deploy → append +tests → audit → build → deploy dry-run → Postgres migrations → deploy → append the commit to `.release-history`. The two custom domains attach from the `routes` in `env.production` automatically, provided the zone from step 1 exists. @@ -100,7 +102,7 @@ exists. ## 6. Post-deploy verification gotchas (both hit on the davis7.space run) - **Transient 500s right after deploy.** The very first requests can 500 - for a few seconds while D1 migrations settle against the first live + for a few seconds while the Hyperdrive pool warms against the first live request. Recheck once things settle — it clears on its own. - **The second (content) custom domain lags the first.** The content origin's DNS record and edge certificate propagate minutes behind the diff --git a/apps/web/migrations/0001_init.sql b/apps/web/migrations/0001_init.sql deleted file mode 100644 index 06d3322..0000000 --- a/apps/web/migrations/0001_init.sql +++ /dev/null @@ -1,142 +0,0 @@ -PRAGMA foreign_keys = ON; - -CREATE TABLE files ( - id TEXT PRIMARY KEY, - display_name TEXT NOT NULL, - content_type TEXT NOT NULL, - kind TEXT NOT NULL DEFAULT 'file' CHECK (kind IN ('file', 'site')), - current_version INTEGER NOT NULL DEFAULT 1 CHECK (current_version > 0), - size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), - public INTEGER NOT NULL DEFAULT 1 CHECK (public IN (0, 1)), - is_site INTEGER NOT NULL DEFAULT 0 CHECK (is_site IN (0, 1)), - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - deleted_at TEXT, - purge_at TEXT, - purge_state TEXT NOT NULL DEFAULT 'none' CHECK ( - purge_state IN ('none', 'pending', 'done', 'failed') - ), - expires_at TEXT, - download_count INTEGER NOT NULL DEFAULT 0 CHECK (download_count >= 0), - last_download_at TEXT, - index_state TEXT NOT NULL DEFAULT 'pending' CHECK ( - index_state IN ('pending', 'running', 'ready', 'failed', 'disabled') - ), - indexed_version INTEGER, - index_cursor INTEGER NOT NULL DEFAULT 0 CHECK (index_cursor >= 0), - index_attempts INTEGER NOT NULL DEFAULT 0 CHECK (index_attempts >= 0), - index_error TEXT, - index_next_run_at TEXT -); - -CREATE TABLE file_versions ( - file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE, - version INTEGER NOT NULL CHECK (version > 0), - r2_key TEXT NOT NULL UNIQUE, - size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), - sha256 TEXT, - content_type TEXT NOT NULL, - created_at TEXT NOT NULL, - text_content TEXT, - PRIMARY KEY (file_id, version) -); - -CREATE TABLE tags ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - normalized_name TEXT NOT NULL UNIQUE, - color TEXT, - created_at TEXT NOT NULL -); - -CREATE TABLE file_tags ( - file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE, - tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, - PRIMARY KEY (file_id, tag_id) -); - -CREATE TABLE site_assets ( - file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE, - version INTEGER NOT NULL, - path TEXT NOT NULL, - r2_key TEXT NOT NULL UNIQUE, - content_type TEXT NOT NULL, - size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), - PRIMARY KEY (file_id, version, path) -); - -CREATE TABLE file_chunks ( - vector_id TEXT PRIMARY KEY, - file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE, - version INTEGER NOT NULL, - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - char_start INTEGER NOT NULL CHECK (char_start >= 0), - char_end INTEGER NOT NULL CHECK (char_end >= char_start) -); - -CREATE TABLE pending_vector_deletes ( - vector_id TEXT PRIMARY KEY, - queued_at TEXT NOT NULL -); - -CREATE TABLE api_keys ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - prefix TEXT NOT NULL UNIQUE, - secret_hash TEXT NOT NULL, - created_at TEXT NOT NULL, - last_used_at TEXT, - revoked_at TEXT -); - -CREATE TABLE device_codes ( - device_code_hash TEXT PRIMARY KEY, - user_code TEXT NOT NULL UNIQUE, - status TEXT NOT NULL, - interval_seconds INTEGER NOT NULL, - expires_at TEXT NOT NULL, - created_at TEXT NOT NULL -); - -CREATE TABLE device_tokens ( - token_hash TEXT PRIMARY KEY, - device_id TEXT NOT NULL, - name TEXT NOT NULL, - created_at TEXT NOT NULL, - last_used_at TEXT, - revoked_at TEXT -); - --- Reserved now so multipart can be added without changing existing file rows. -CREATE TABLE upload_sessions ( - id TEXT PRIMARY KEY, - file_id TEXT NOT NULL, - r2_key TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - content_type TEXT NOT NULL, - public INTEGER NOT NULL CHECK (public IN (0, 1)), - expected_size_bytes INTEGER NOT NULL CHECK (expected_size_bytes >= 0), - part_size_bytes INTEGER NOT NULL CHECK (part_size_bytes > 0), - r2_upload_id TEXT NOT NULL, - status TEXT NOT NULL CHECK (status IN ('open', 'committing', 'complete', 'aborted')), - created_at TEXT NOT NULL, - expires_at TEXT NOT NULL -); - -CREATE TABLE upload_parts ( - session_id TEXT NOT NULL REFERENCES upload_sessions(id) ON DELETE CASCADE, - part_number INTEGER NOT NULL CHECK (part_number > 0), - etag TEXT NOT NULL, - size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), - PRIMARY KEY (session_id, part_number) -); - -CREATE INDEX files_active_updated_idx - ON files(deleted_at, updated_at DESC); -CREATE INDEX file_versions_r2_key_idx - ON file_versions(r2_key); -CREATE INDEX api_keys_prefix_active_idx - ON api_keys(prefix, revoked_at); -CREATE INDEX upload_sessions_expiry_idx - ON upload_sessions(status, expires_at); - diff --git a/apps/web/migrations/0002_tags_search.sql b/apps/web/migrations/0002_tags_search.sql deleted file mode 100644 index 8711ca8..0000000 --- a/apps/web/migrations/0002_tags_search.sql +++ /dev/null @@ -1,37 +0,0 @@ -CREATE INDEX file_tags_tag_idx - ON file_tags(tag_id, file_id); - -CREATE VIRTUAL TABLE files_fts USING fts5( - name, - tags, - body, - file_id UNINDEXED, - chunk_no UNINDEXED, - tokenize = "unicode61 remove_diacritics 2" -); - -CREATE VIRTUAL TABLE files_trgm USING fts5( - name, - file_id UNINDEXED, - tokenize = "trigram" -); - -INSERT INTO files_fts (name, tags, body, file_id, chunk_no) -SELECT - f.display_name, - COALESCE(( - SELECT group_concat(t.name, ' ') - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = f.id - ), ''), - substr(COALESCE(v.text_content, ''), 1, 65536), - f.id, - 0 -FROM files f -JOIN file_versions v - ON v.file_id = f.id AND v.version = f.current_version; - -INSERT INTO files_trgm (name, file_id) -SELECT display_name, id -FROM files; diff --git a/apps/web/migrations/0003_sites.sql b/apps/web/migrations/0003_sites.sql deleted file mode 100644 index 54d4077..0000000 --- a/apps/web/migrations/0003_sites.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE TABLE site_upload_sessions ( - id TEXT PRIMARY KEY, - file_id TEXT NOT NULL, - display_name TEXT NOT NULL, - version INTEGER NOT NULL CHECK (version > 0), - status TEXT NOT NULL CHECK ( - status IN ('open', 'committing', 'complete', 'aborted') - ), - created_at TEXT NOT NULL, - expires_at TEXT NOT NULL -); - -CREATE TABLE staged_site_assets ( - session_id TEXT NOT NULL REFERENCES site_upload_sessions(id) ON DELETE CASCADE, - path TEXT NOT NULL, - expected_size_bytes INTEGER NOT NULL CHECK (expected_size_bytes >= 0), - content_type TEXT NOT NULL, - r2_key TEXT UNIQUE, - stored_size_bytes INTEGER CHECK (stored_size_bytes >= 0), - uploaded_at TEXT, - PRIMARY KEY (session_id, path), - CHECK ( - (r2_key IS NULL AND stored_size_bytes IS NULL AND uploaded_at IS NULL) - OR - (r2_key IS NOT NULL AND stored_size_bytes IS NOT NULL AND uploaded_at IS NOT NULL) - ) -); - --- R2 cleanup is compensatable, while D1 is the serving authority. Rows remain --- here after a failed delete and are retried by later site operations. -CREATE TABLE pending_site_asset_deletes ( - r2_key TEXT PRIMARY KEY, - file_id TEXT NOT NULL, - version INTEGER NOT NULL CHECK (version > 0), - queued_at TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), - last_error TEXT -); - -CREATE INDEX site_upload_sessions_expiry_idx - ON site_upload_sessions(status, expires_at); - -CREATE INDEX pending_site_asset_deletes_file_idx - ON pending_site_asset_deletes(file_id, queued_at); diff --git a/apps/web/migrations/0004_auth_ttl_downloads.sql b/apps/web/migrations/0004_auth_ttl_downloads.sql deleted file mode 100644 index 82d619b..0000000 --- a/apps/web/migrations/0004_auth_ttl_downloads.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE dashboard_sessions ( - token_hash TEXT PRIMARY KEY, - created_at TEXT NOT NULL, - expires_at TEXT NOT NULL, - last_used_at TEXT NOT NULL -); - -CREATE INDEX dashboard_sessions_expiry_idx - ON dashboard_sessions(expires_at); - -ALTER TABLE device_codes ADD COLUMN name TEXT NOT NULL DEFAULT 'adrive CLI'; -ALTER TABLE device_codes ADD COLUMN last_polled_at TEXT; -ALTER TABLE device_codes ADD COLUMN approved_at TEXT; -ALTER TABLE device_codes ADD COLUMN consumed_at TEXT; -ALTER TABLE device_codes ADD COLUMN api_key_id TEXT REFERENCES api_keys(id); - -CREATE INDEX device_codes_expiry_status_idx - ON device_codes(status, expires_at); - -CREATE INDEX files_expiry_idx - ON files(expires_at, deleted_at); diff --git a/apps/web/migrations/0005_semantic_lifecycle.sql b/apps/web/migrations/0005_semantic_lifecycle.sql deleted file mode 100644 index ef216d7..0000000 --- a/apps/web/migrations/0005_semantic_lifecycle.sql +++ /dev/null @@ -1,18 +0,0 @@ -ALTER TABLE files ADD COLUMN purge_attempts INTEGER NOT NULL DEFAULT 0 - CHECK (purge_attempts >= 0); -ALTER TABLE files ADD COLUMN purge_error TEXT; -ALTER TABLE files ADD COLUMN purge_next_run_at TEXT; - -ALTER TABLE pending_vector_deletes ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0 - CHECK (attempts >= 0); -ALTER TABLE pending_vector_deletes ADD COLUMN last_error TEXT; -ALTER TABLE pending_vector_deletes ADD COLUMN next_run_at TEXT; - -CREATE INDEX files_index_jobs_idx - ON files(index_state, index_next_run_at, updated_at); - -CREATE INDEX files_purge_jobs_idx - ON files(purge_state, purge_next_run_at, purge_at, expires_at); - -CREATE INDEX pending_vector_deletes_retry_idx - ON pending_vector_deletes(next_run_at, queued_at); diff --git a/apps/web/migrations/0006_index_leases.sql b/apps/web/migrations/0006_index_leases.sql deleted file mode 100644 index 71bfdce..0000000 --- a/apps/web/migrations/0006_index_leases.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE files ADD COLUMN index_lease_token TEXT; diff --git a/apps/web/migrations/0007_instance_secrets.sql b/apps/web/migrations/0007_instance_secrets.sql deleted file mode 100644 index c42a0b0..0000000 --- a/apps/web/migrations/0007_instance_secrets.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE instance_secrets ( - id INTEGER PRIMARY KEY CHECK (id = 1), - content_grant_signing_key TEXT NOT NULL CHECK ( - length(content_grant_signing_key) = 43 - AND content_grant_signing_key NOT GLOB '*[^A-Za-z0-9_-]*' - ), - created_at TEXT NOT NULL -); diff --git a/apps/web/migrations/0008_api_key_scopes.sql b/apps/web/migrations/0008_api_key_scopes.sql deleted file mode 100644 index 8f8a436..0000000 --- a/apps/web/migrations/0008_api_key_scopes.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Read-only keys can list and download but not mutate; expiry is optional --- and enforced at authorization time. Existing keys keep full access. -ALTER TABLE api_keys ADD COLUMN scope TEXT NOT NULL DEFAULT 'read-write' - CHECK (scope IN ('read-only', 'read-write')); -ALTER TABLE api_keys ADD COLUMN expires_at TEXT; diff --git a/apps/web/migrations/0009_credential_state.sql b/apps/web/migrations/0009_credential_state.sql deleted file mode 100644 index 055f1f6..0000000 --- a/apps/web/migrations/0009_credential_state.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Tracks the hash of the active passcode so a rotation (a new PASSCODE --- secret on the Worker) can be detected and revoke sessions/device codes. -CREATE TABLE credential_state ( - id INTEGER PRIMARY KEY CHECK (id = 1), - passcode_hash TEXT NOT NULL CHECK (length(passcode_hash) = 64), - rotated_at TEXT NOT NULL -); diff --git a/apps/web/migrations/0010_thumbnail_storage.sql b/apps/web/migrations/0010_thumbnail_storage.sql deleted file mode 100644 index 8e4b6e6..0000000 --- a/apps/web/migrations/0010_thumbnail_storage.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE file_versions - ADD COLUMN thumbnail_r2_key TEXT; - -ALTER TABLE file_versions - ADD COLUMN thumbnail_size_bytes INTEGER NOT NULL DEFAULT 0 - CHECK (thumbnail_size_bytes >= 0); diff --git a/apps/web/package.json b/apps/web/package.json index b1144a6..8687537 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -16,7 +16,6 @@ }, "dependencies": { "@adrive/shared": "workspace:^", - "@effect/sql-d1": "4.0.0-beta.102", "@effect/sql-pg": "4.0.0-beta.102", "@modelcontextprotocol/server": "2.0.0", "agents": "^0.20.1", diff --git a/apps/web/scripts/d1-to-postgres.mjs b/apps/web/scripts/d1-to-postgres.mjs new file mode 100644 index 0000000..63fa388 --- /dev/null +++ b/apps/web/scripts/d1-to-postgres.mjs @@ -0,0 +1,213 @@ +// One-off data move from the D1 export of a single-tenant adrive instance +// into Postgres. Reads the SQL dump produced by +// `wrangler d1 export DB --env production --remote --output d1.sql`, +// replays it into an in-memory SQLite database, then copies each table +// row by row with type conversion (0/1 -> boolean, ISO text -> timestamptz). +// +// bun apps/web/scripts/d1-to-postgres.mjs --dump d1.sql --url postgres://... +// bun apps/web/scripts/d1-to-postgres.mjs --dump d1.sql --url ... --wipe # truncate first +// +// R2 objects do not move. Keyword search documents are rebuilt from the +// copied rows; semantic vectors are not carried over (Vectorize is gone), +// so every file is left in index_state = 'pending' for the queue/cron to +// re-embed. +import { readFileSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import Pg from 'pg'; + +const arg = (flag) => { + const index = process.argv.indexOf(flag); + return index >= 0 ? process.argv[index + 1] : undefined; +}; + +const dumpPath = arg('--dump'); +const url = arg('--url') ?? process.env.DATABASE_URL; +const wipe = process.argv.includes('--wipe'); +if (!dumpPath || !url) { + console.error( + 'Usage: d1-to-postgres.mjs --dump --url [--wipe]' + ); + process.exit(1); +} + +const sqlite = new DatabaseSync(':memory:'); +// D1 exports include the FTS virtual tables and their shadow tables, which +// node:sqlite may not be able to create. Skip anything mentioning them. +const skip = /files_fts|files_trgm|sqlite_sequence|_cf_KV|d1_migrations/i; +const statements = readFileSync(dumpPath, 'utf8') + .split(/;\s*\n/) + .map((statement) => statement.trim()) + .filter((statement) => statement && !skip.test(statement)); +for (const statement of statements) sqlite.exec(`${statement};`); + +const rows = (table) => sqlite.prepare(`SELECT * FROM ${table}`).all(); +const bool = (value) => value === 1 || value === true; +const ts = (value) => (value == null ? null : new Date(value).toISOString()); + +const copy = async (client, table, columns, transform) => { + const source = rows(table); + let count = 0; + for (const row of source) { + const values = transform(row); + const placeholders = columns.map((_, index) => `$${index + 1}`).join(', '); + await client.query( + `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${placeholders}) ON CONFLICT DO NOTHING`, + columns.map((column) => values[column]) + ); + count += 1; + } + console.log(`${table}: ${count}`); +}; + +const client = new Pg.Client({ connectionString: url }); +await client.connect(); +try { + await client.query('BEGIN'); + if (wipe) { + await client.query( + 'TRUNCATE files, tags, api_keys, device_codes, site_upload_sessions, pending_site_asset_deletes, instance_secrets CASCADE' + ); + } + await copy( + client, + 'files', + [ + 'id', + 'display_name', + 'content_type', + 'kind', + 'current_version', + 'size_bytes', + 'public', + 'is_site', + 'created_at', + 'updated_at', + 'deleted_at', + 'purge_at', + 'purge_state', + 'purge_attempts', + 'purge_error', + 'purge_next_run_at', + 'expires_at', + 'download_count', + 'last_download_at', + 'index_state', + 'indexed_version', + 'index_cursor', + 'index_attempts', + 'index_error', + 'index_next_run_at', + 'index_lease_token' + ], + (row) => ({ + ...row, + public: bool(row.public), + is_site: bool(row.is_site), + created_at: ts(row.created_at), + updated_at: ts(row.updated_at), + deleted_at: ts(row.deleted_at), + purge_at: ts(row.purge_at), + purge_next_run_at: ts(row.purge_next_run_at), + expires_at: ts(row.expires_at), + last_download_at: ts(row.last_download_at), + // Vectors do not move; every file re-embeds through the queue. + index_state: row.index_state === 'disabled' ? 'disabled' : 'pending', + indexed_version: null, + index_cursor: 0, + index_attempts: 0, + index_error: null, + index_next_run_at: null, + index_lease_token: null + }) + ); + await copy( + client, + 'file_versions', + [ + 'file_id', + 'version', + 'r2_key', + 'size_bytes', + 'sha256', + 'content_type', + 'created_at', + 'text_content', + 'thumbnail_r2_key', + 'thumbnail_size_bytes' + ], + (row) => ({ + ...row, + created_at: ts(row.created_at), + thumbnail_size_bytes: row.thumbnail_size_bytes ?? 0 + }) + ); + await copy( + client, + 'tags', + ['id', 'name', 'normalized_name', 'color', 'created_at'], + (row) => ({ ...row, created_at: ts(row.created_at) }) + ); + await copy(client, 'file_tags', ['file_id', 'tag_id'], (row) => row); + await copy( + client, + 'site_assets', + ['file_id', 'version', 'path', 'r2_key', 'content_type', 'size_bytes'], + (row) => row + ); + await copy( + client, + 'api_keys', + [ + 'id', + 'name', + 'prefix', + 'secret_hash', + 'scope', + 'created_at', + 'expires_at', + 'last_used_at', + 'revoked_at' + ], + (row) => ({ + ...row, + scope: row.scope ?? 'read-write', + created_at: ts(row.created_at), + expires_at: ts(row.expires_at), + last_used_at: ts(row.last_used_at), + revoked_at: ts(row.revoked_at) + }) + ); + await copy( + client, + 'pending_site_asset_deletes', + ['r2_key', 'file_id', 'version', 'queued_at', 'attempts', 'last_error'], + (row) => ({ ...row, queued_at: ts(row.queued_at) }) + ); + await copy( + client, + 'instance_secrets', + ['id', 'content_grant_signing_key', 'created_at'], + (row) => ({ ...row, created_at: ts(row.created_at) }) + ); + // Sessions, device codes, and the passcode hash are not carried over; + // everyone signs in again. Keyword search documents are rebuilt. + await client.query(` + INSERT INTO search_documents (file_id, chunk_no, name, tags, body) + SELECT f.id, 0, f.display_name, + COALESCE((SELECT string_agg(t.name, ' ' ORDER BY t.normalized_name) + FROM file_tags ft JOIN tags t ON t.id = ft.tag_id WHERE ft.file_id = f.id), ''), + left(COALESCE(v.text_content, ''), 65536) + FROM files f JOIN file_versions v ON v.file_id = f.id AND v.version = f.current_version + ON CONFLICT (file_id, chunk_no) DO NOTHING`); + const summary = await client.query( + `SELECT (SELECT count(*) FROM files) AS files, (SELECT count(*) FROM file_versions) AS versions, + (SELECT count(*) FROM search_documents) AS documents` + ); + console.log('postgres totals', summary.rows[0]); + await client.query('COMMIT'); +} catch (cause) { + await client.query('ROLLBACK'); + throw cause; +} finally { + await client.end(); +} diff --git a/apps/web/src/lib/server/edge.ts b/apps/web/src/lib/server/edge.ts index cbac219..9a1d216 100644 --- a/apps/web/src/lib/server/edge.ts +++ b/apps/web/src/lib/server/edge.ts @@ -32,7 +32,6 @@ import type { Lifecycle } from './services/lifecycle'; import type { GrantSecrets } from './services/grant-secrets'; export type AppServices = - | SqlClient.SqlClient | PgSql | AppConfig | AuthGuard diff --git a/apps/web/src/lib/server/layer.ts b/apps/web/src/lib/server/layer.ts index 7dfb6cf..0bc1c66 100644 --- a/apps/web/src/lib/server/layer.ts +++ b/apps/web/src/lib/server/layer.ts @@ -1,9 +1,8 @@ -import * as D1 from '@effect/sql-d1/D1Client'; import { Effect, Layer } from 'effect'; import { ConfigLive } from './config'; import { AuthGuardLive } from './services/auth-guard'; import { AuthLive } from './services/auth'; -import { AuthGuardStore, Db, Bucket, Pg } from './services/bindings'; +import { AuthGuardStore, Bucket, Pg } from './services/bindings'; import { pgLayer } from './pg'; import { BlobsLive } from './services/blobs'; import { FilesLive } from './services/files'; @@ -15,25 +14,20 @@ import { IndexingLive } from './services/indexing'; import { LifecycleLive } from './services/lifecycle'; import { GrantSecretsLive } from './services/grant-secrets'; -const SqlLive = Layer.unwrap(Effect.map(Db, (db) => D1.layer({ db }))); -// Postgres runs beside D1 until the port is complete. Services move over -// one cluster at a time by yielding PgSql instead of SqlClient. export const PgLive = Layer.unwrap( Effect.map(Pg, (hyperdrive) => pgLayer(hyperdrive)) ); export const requestLayer = (env: Env) => { const bindings = Layer.mergeAll( - Layer.succeed(Db, env.DB), Layer.succeed(Pg, env.HYPERDRIVE), Layer.succeed(Bucket, env.BUCKET), Layer.succeed(AuthGuardStore, env.AUTH_GUARD), ConfigLive(env) ); - const sql = SqlLive.pipe(Layer.provide(bindings)); const pg = PgLive.pipe(Layer.provide(bindings)); const blobs = BlobsLive.pipe(Layer.provide(bindings)); - const infrastructure = Layer.mergeAll(bindings, sql, pg, blobs); + const infrastructure = Layer.mergeAll(bindings, pg, blobs); // The vector index reads and writes file_chunks, so it sits on Postgres // like every other service; only the embedder still binds Workers AI. const semantic = SemanticBindingsLive(env).pipe( diff --git a/apps/web/src/lib/server/services/bindings.ts b/apps/web/src/lib/server/services/bindings.ts index 5a1b75d..ee7a95b 100644 --- a/apps/web/src/lib/server/services/bindings.ts +++ b/apps/web/src/lib/server/services/bindings.ts @@ -1,7 +1,5 @@ import { Context } from 'effect'; -export class Db extends Context.Service()('app/Db') {} - // Hyperdrive binding. Only the connection string is used; the pool itself // is built per request in pg.ts. export class Pg extends Context.Service()('app/Pg') {} diff --git a/apps/web/worker-configuration.d.ts b/apps/web/worker-configuration.d.ts index 2429469..e024ca9 100644 --- a/apps/web/worker-configuration.d.ts +++ b/apps/web/worker-configuration.d.ts @@ -1,10 +1,9 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: a28431120a2ba6281544be2aa23c9aa1) +// Generated by Wrangler by running `wrangler types` (hash: caead1cb99063067c29ee1f0ece3f1ab) // Runtime types generated with workerd@1.20260722.1 2026-07-27 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { AUTH_GUARD: KVNamespace; BUCKET: R2Bucket; - DB: D1Database; HYPERDRIVE: Hyperdrive; BROWSER: BrowserRun; AI?: Ai; @@ -24,7 +23,6 @@ declare namespace Cloudflare { interface ProductionEnv { AUTH_GUARD: KVNamespace; BUCKET: R2Bucket; - DB: D1Database; HYPERDRIVE: Hyperdrive; BROWSER: BrowserRun; AI: Ai; diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index 21b042e..ca7d9b3 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -30,14 +30,6 @@ "localConnectionString": "postgres://adrive:adrive@127.0.0.1:5432/adrive" } ], - "d1_databases": [ - { - "binding": "DB", - "database_name": "adrive", - "database_id": "replace-with-your-d1-database-id", - "migrations_dir": "migrations" - } - ], "r2_buckets": [ { "binding": "BUCKET", @@ -72,7 +64,7 @@ // 2. Create the production R2 bucket and KV namespace; paste the KV id. // 3. Point DNS for both hostnames at the Worker (routes below). // 4. `wrangler secret put PASSCODE --env production` - // 5. `wrangler d1 migrations apply DB --env production --remote` + // 5. `DATABASE_URL=postgres://... bun db:pg:migrate` (see docs/release.md) "env": { "production": { // Place the Worker near D1 for dashboard/search queries. Content @@ -107,14 +99,6 @@ "id": "replace-with-production-hyperdrive-id" } ], - "d1_databases": [ - { - "binding": "DB", - "database_name": "adrive-production", - "database_id": "29891a68-07c7-47e4-94db-b0123982fc1f", - "migrations_dir": "migrations" - } - ], "r2_buckets": [ { "binding": "BUCKET", diff --git a/bun.lock b/bun.lock index 1018169..86eed9d 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,6 @@ "version": "0.1.0", "dependencies": { "@adrive/shared": "workspace:^", - "@effect/sql-d1": "4.0.0-beta.102", "@effect/sql-pg": "4.0.0-beta.102", "@modelcontextprotocol/server": "2.0.0", "agents": "^0.20.1", @@ -166,8 +165,6 @@ "@effect/platform-node-shared": ["@effect/platform-node-shared@4.0.0-rc.117", "", { "dependencies": { "@types/ws": "^8.18.1", "ws": "^8.21.3" }, "peerDependencies": { "effect": "^4.0.0-rc.117" } }, "sha512-GyRkEfLBG49MwJe+/XDPiXYcQ44RA9p2z0AurlsWhR1cd1PqOUMEZU1pORL+/R5hR8aH6ZuqoJYD8lAc6h+wlA=="], - "@effect/sql-d1": ["@effect/sql-d1@4.0.0-beta.102", "", { "dependencies": { "@cloudflare/workers-types": "^5.20260708.1" }, "peerDependencies": { "effect": "^4.0.0-beta.102" } }, "sha512-p6V1BqXEKF/EVH1JkWjjslPtg5Knl2TWopMnDdgto2jGfRpCHfhEaI8fA2OmR5J5H/TIH9ihYBiUA6Awru2Kvg=="], - "@effect/sql-pg": ["@effect/sql-pg@4.0.0-beta.102", "", { "dependencies": { "pg": "^8.22.0", "pg-connection-string": "2.14.0", "pg-cursor": "^2.21.0", "pg-pool": "^3.14.0", "pg-types": "^4.1.0" }, "peerDependencies": { "effect": "^4.0.0-beta.102" } }, "sha512-02a+fNfECWCZIZcgHl7OLQN2jZj42gq90EdwdnvR3KA6GLGypY02swABS4zFtaW9H80rTa0ukqVnTdcO/DT5GQ=="], "@effect/tsgo": ["@effect/tsgo@0.24.3", "", { "optionalDependencies": { "@effect/tsgo-darwin-arm64": "0.24.3", "@effect/tsgo-darwin-x64": "0.24.3", "@effect/tsgo-linux-arm": "0.24.3", "@effect/tsgo-linux-arm64": "0.24.3", "@effect/tsgo-linux-x64": "0.24.3", "@effect/tsgo-win32-arm64": "0.24.3", "@effect/tsgo-win32-x64": "0.24.3" }, "bin": { "effect-tsgo": "dist/effect-tsgo.js" } }, "sha512-WQxKU3MFzWzI38GbE9cf0POkVX4y0xahD8QqDjLfixW1AEValuHNfOQvyKM2MDWOLdGff4hP8VnIOeduwQt66A=="], @@ -994,8 +991,6 @@ "@cspotcode/source-map-support/@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], - "@effect/sql-d1/@cloudflare/workers-types": ["@cloudflare/workers-types@5.20260925.2", "", {}, "sha512-J/cE9J5BnJV+x++x8Q4l02kOCqEDYIalCbZnxdXsveFLUefTcbqk4EOXZYlfgGow5iIRoXc/PlE4nz8nRVBMqg=="], - "@effect/sql-pg/pg-types": ["pg-types@4.1.0", "", { "dependencies": { "pg-int8": "1.0.1", "pg-numeric": "1.0.2", "postgres-array": "~3.0.1", "postgres-bytea": "~3.0.0", "postgres-date": "~2.1.0", "postgres-interval": "^3.0.0", "postgres-range": "^1.1.1" } }, "sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg=="], "@sveltejs/vite-plugin-svelte/magic-string": ["magic-string@1.4.2", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.6.0" } }, "sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g=="], @@ -1016,8 +1011,6 @@ "accepts/mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="], - "agents/yargs": ["yargs@18.2.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^8.2.1", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w=="], - "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], "express/cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], @@ -1074,8 +1067,6 @@ "accepts/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], - "agents/yargs/string-width": ["string-width@8.3.0", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ=="], - "express/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], "send/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 0ed4e64..f649a80 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -12,14 +12,14 @@ anything important. Nightly cron on the backup host (02:17 local, `scripts/backup/backup.sh`) writes to `~/Backups/a-drive`: -| Path | Contents | Retention | -| ------------------------ | --------------------------------------------- | ----------- | -| `r2-mirror/` | Full mirror of the production R2 bucket | Live mirror | -| `r2-deleted//` | Objects deleted/overwritten upstream that day | 30 days | -| `d1/daily/` | Nightly gzipped D1 SQL export | 30 days | -| `d1/monthly/` | First export of each month | 12 months | -| `manifests/` | Object keys, sizes, hashes, D1 digest per run | 30 days | -| `logs/`, `last-run.json` | Run logs and machine-readable status | 30 days | +| Path | Contents | Retention | +| ------------------------ | ----------------------------------------------- | ----------- | +| `r2-mirror/` | Full mirror of the production R2 bucket | Live mirror | +| `r2-deleted//` | Objects deleted/overwritten upstream that day | 30 days | +| `postgres/daily/` | Nightly gzipped `pg_dump` | 30 days | +| `postgres/monthly/` | First dump of each month | 12 months | +| `manifests/` | Object keys, sizes, hashes, dump digest per run | 30 days | +| `logs/`, `last-run.json` | Run logs and machine-readable status | 30 days | Deletions are **not** mirrored immediately: `rclone sync --backup-dir` moves upstream-deleted objects into the dated trash directory, so an @@ -33,7 +33,8 @@ Failures and suspicious shrinkage post to `ALERT_WEBHOOK_URL` from - rclone uses an R2 API token scoped **read-only** to the production bucket — it cannot delete or overwrite anything upstream. -- The D1 export uses a Cloudflare API token with **D1:Read only**. +- The Postgres dump uses a read-only database role (`DATABASE_URL` in + `backup.env`). - Neither the PASSCODE, session secrets, nor deploy-capable tokens exist on the backup host. `backup.env` is `chmod 600` and gitignored. @@ -48,43 +49,45 @@ write access (deliberately _not_ stored on the backup host). (keys look like `v//`). 2. Copy it back: `rclone copyto ~/Backups/a-drive/r2-mirror/ adrive-r2-rw:/` (or from `r2-deleted//` if it was deleted). -3. If the D1 row was also lost, restore metadata via the full-database +3. If the database row was also lost, restore metadata via the full-database procedure or reinsert the `files`/`file_versions` rows from the daily export. ### All versions of one file -Same as above for every `v//*` key in the manifest; the daily D1 -export holds the matching `file_versions` rows. +Same as above for every `v//*` key in the manifest; the daily +dump holds the matching `file_versions` rows. ### Metadata and tags only -1. `gunzip -k d1/daily/adrive-.sql.gz` +1. `gunzip -k postgres/daily/adrive-.sql.gz` 2. Extract the rows you need (`files`, `file_versions`, `tags`, - `file_tags`) and apply them with - `wrangler d1 execute DB --env production --remote --file .sql`. + `file_tags`) and apply them with `psql "$DATABASE_URL" -f .sql`. -### Complete D1 database +### Complete Postgres database -Always restore into a fresh database and cut over — never import over the -production database, so it stays untouched for rollback. +PlanetScale keeps its own automatic backups with point-in-time restore; +that is the first option. The nightly `pg_dump` here is the independent +copy for when the account itself is unavailable. Always restore into a +fresh database and cut over — never import over the production database, +so it stays untouched for rollback. -1. `wrangler d1 create adrive-restore` -2. `gunzip -k d1/daily/adrive-.sql.gz` -3. `wrangler d1 execute adrive-restore --remote --file adrive-.sql` +1. Create a new PlanetScale Postgres database (or a local one for a drill). +2. `gunzip -k postgres/daily/adrive-.sql.gz` +3. `psql "" -f adrive-.sql` 4. Validate before cutover: spot-check table row counts and schema - (`wrangler d1 execute adrive-restore --remote --command "SELECT COUNT(*) FROM files"`, - same for `file_versions`, `tags`) against expectations from the - manifest. -5. Point `wrangler.jsonc` `env.production.d1_databases[0].database_id` at - the restored database and redeploy. Keep the previous database until - the deployment is verified. + (`psql ... -c "SELECT COUNT(*) FROM files"`, same for `file_versions`, + `tags`) against expectations from the manifest. +5. Point the Hyperdrive config at the restored database (or create a new + one and update `wrangler.jsonc` `env.production.hyperdrive[0].id`) and + redeploy. Keep the previous database until the deployment is verified. ### Whole application in a clean Cloudflare account -1. Create D1 database, R2 bucket, and KV namespace; paste their ids into - `apps/web/wrangler.jsonc` under `env.production`. -2. Restore D1 from the latest export (above). +1. Create the Postgres database, its Hyperdrive config, the R2 bucket, and + the KV namespace; paste the ids into `apps/web/wrangler.jsonc` under + `env.production`. +2. Restore Postgres from the latest dump (above). 3. Restore R2: `rclone sync ~/Backups/a-drive/r2-mirror adrive-r2-rw:` 4. Set the secret: `wrangler secret put PASSCODE --env production`. 5. Deploy: `bun release` (or `wrangler deploy --env production`). @@ -103,8 +106,8 @@ date and outcome at the bottom of this file: 1. Restore one file and verify its checksum matches the manifest. 2. Restore all versions of one multi-version file. -3. Restore metadata and tags for one file into a scratch D1 database. -4. Restore the complete D1 database into a scratch database and spot-check +3. Restore metadata and tags for one file into a scratch Postgres database. +4. Restore the complete Postgres dump into a scratch database and spot-check row counts against production. 5. Reconstruct the application in a clean environment (separate Worker name or account) end to end, including one download through the diff --git a/docs/release.md b/docs/release.md index 1f8b718..45509b2 100644 --- a/docs/release.md +++ b/docs/release.md @@ -6,8 +6,18 @@ App Worker commands run from `apps/web`. Landing-site commands run from `apps/site` (no `--env`). `bun release` and the backup installer run from the repository root. -1. From `apps/web`: `wrangler d1 create adrive-production` — paste the id - into `wrangler.jsonc` `env.production.d1_databases[0].database_id`. +1. Create a PlanetScale Postgres database (region close to most users) + with the `vector` and `pg_trgm` extensions available, then a Hyperdrive + config pointing at its direct port 5432 with caching disabled: + + ``` + wrangler hyperdrive create adrive-production --connection-string="postgres://..." --caching-disabled + ``` + + Paste the id into `wrangler.jsonc` `env.production.hyperdrive[0].id`. + Export the same connection string as `DATABASE_URL` when releasing; + `bun release` runs `apps/web/scripts/pg-migrate.mjs` against it. + 2. From `apps/web`: `wrangler r2 bucket create adrive-production` 3. From `apps/web`: `wrangler kv namespace create AUTH_GUARD --env production` — paste the id into `env.production.kv_namespaces[0].id`. @@ -53,7 +63,7 @@ bun release The script refuses a dirty tree or placeholder ids in the target env, then runs format check → type/lint checks → tests → audit → build → -app deploy dry run → landing-site dry run → D1 migrations → app deploy → +app deploy dry run → landing-site dry run → Postgres migrations → app deploy → landing-site deploy, and appends the deployed commit to `.release-history`. If the landing site fails after the app Worker is live, the script still records the app commit and prints rollback @@ -82,8 +92,8 @@ bun x wrangler deployments list --env production # find the previous version bun x wrangler rollback --env production # interactive picker ``` -Rollback redeploys the previous Worker bundle. It does not touch D1, R2, -KV, or secrets — which is why the migration rule above matters. +Rollback redeploys the previous Worker bundle. It does not touch +Postgres, R2, KV, or secrets — which is why the migration rule above matters. ### Landing site @@ -96,14 +106,13 @@ bun x wrangler deployments list bun x wrangler rollback ``` -### D1 +### Postgres There is no in-place downgrade. Recovery options, in order of blast radius: -1. **Cloudflare Time Travel** (point-in-time restore, 30-day window): - `wrangler d1 time-travel info DB --env production` then - `wrangler d1 time-travel restore DB --env production --timestamp `. +1. **PlanetScale backups** (automatic daily, plus point-in-time): + PlanetScale point-in-time restore from the database's Backups page. This rewinds the whole database — anything written after the timestamp is lost. 2. **Nightly export**: restore per `docs/backup-restore.md` (full @@ -120,3 +129,22 @@ pending device codes automatically (API keys stay). `.release-history` in the repo root accumulates ` ` lines locally. The deployed commit is also visible via `wrangler deployments list --env production`. + +## One-off move from D1 + +The single existing instance moves its metadata from D1 to Postgres once. +R2 does not move. Sessions, device codes, and the passcode hash are not +carried over; sign in again afterwards. Semantic vectors are not carried +over either; every file is left `pending` and re-embeds through the +indexing sweep. + +``` +cd apps/web +bun x wrangler d1 export DB --env production --remote --output /tmp/adrive-d1.sql # from the last D1 deploy +DATABASE_URL=postgres://... bun scripts/pg-migrate.mjs --url "$DATABASE_URL" +bun scripts/d1-to-postgres.mjs --dump /tmp/adrive-d1.sql --url "$DATABASE_URL" +``` + +The script prints per-table counts and the Postgres totals at the end. +Compare them with `SELECT COUNT(*)` on the D1 export before flipping +DNS. Run it with `--wipe` to truncate and retry. diff --git a/package.json b/package.json index 3a227d1..f40275c 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,10 @@ "check": "bun run --filter '@adrive/*' --if-present check && bun run check:forbidden && bun run check:wrangler-drift", "check:forbidden": "bun scripts/check-forbidden.mjs", "check:wrangler-drift": "bun scripts/check-wrangler-drift.mjs --allow-placeholders", - "db:migrate:local": "cd apps/web && bun x wrangler d1 migrations apply DB --local", "db:pg:up": "docker compose up -d", + "db:pg:migrate": "bun apps/web/scripts/pg-migrate.mjs", "db:pg:migrate:local": "bun apps/web/scripts/pg-migrate.mjs", + "db:migrate:d1-to-postgres": "bun apps/web/scripts/d1-to-postgres.mjs", "format": "prettier --write .", "format:check": "prettier --check .", "key:create:local": "bun scripts/create-local-key.mjs", diff --git a/scripts/backup/backup.env.example b/scripts/backup/backup.env.example index 0b456a4..60cb082 100644 --- a/scripts/backup/backup.env.example +++ b/scripts/backup/backup.env.example @@ -11,12 +11,9 @@ BACKUP_ROOT="${HOME}/Backups/a-drive" # Use an R2 API token scoped to READ-ONLY on the adrive-production bucket. RCLONE_REMOTE="adrive-r2:adrive-production" -# Cloudflare account and the PRODUCTION D1 database id (wrangler.jsonc). -CLOUDFLARE_ACCOUNT_ID="" -CLOUDFLARE_D1_DATABASE_ID="" - -# API token with D1:Read permission only. Do not reuse a deploy token. -CLOUDFLARE_API_TOKEN="" +# Read-only Postgres role on the production database (PlanetScale direct +# port 5432). pg_dump must be installed on the backup host. +DATABASE_URL="" # Where failures and anomalies are announced (ntfy topic, Discord webhook, # or similar). Posts {"text": "..."} JSON. Leave empty to log only. diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index 25872f6..87ef1f6 100755 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# a-drive nightly backup: R2 sync + D1 export + retention + manifest. +# a-drive nightly backup: R2 sync + Postgres dump + retention + manifest. # Runs on the backup host from cron. Configuration comes from backup.env next to # this script (see backup.env.example). Never prints secrets. set -euo pipefail @@ -13,9 +13,7 @@ source "${SCRIPT_DIR}/backup.env" : "${BACKUP_ROOT:?BACKUP_ROOT is required (e.g. ~/Backups/a-drive)}" : "${RCLONE_REMOTE:?RCLONE_REMOTE is required (e.g. adrive-r2:adrive-production)}" -: "${CLOUDFLARE_ACCOUNT_ID:?CLOUDFLARE_ACCOUNT_ID is required}" -: "${CLOUDFLARE_D1_DATABASE_ID:?CLOUDFLARE_D1_DATABASE_ID is required}" -: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${DATABASE_URL:?DATABASE_URL is required (read-only Postgres role)}" # ALERT_WEBHOOK_URL is optional but strongly recommended. # Serialize runs: a manual invocation overlapping cron (or a run outliving @@ -73,14 +71,14 @@ NOW_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" MIRROR_DIR="${BACKUP_ROOT}/r2-mirror" TRASH_DIR="${BACKUP_ROOT}/r2-deleted/${STAMP_DAY}" -D1_DIR="${BACKUP_ROOT}/d1/daily" -D1_MONTHLY_DIR="${BACKUP_ROOT}/d1/monthly" +PG_DIR="${BACKUP_ROOT}/postgres/daily" +PG_MONTHLY_DIR="${BACKUP_ROOT}/postgres/monthly" MANIFEST_DIR="${BACKUP_ROOT}/manifests" LOG_DIR="${BACKUP_ROOT}/logs" STATUS_FILE="${BACKUP_ROOT}/last-run.json" LOG_FILE="${LOG_DIR}/backup-${STAMP_DAY}.log" -mkdir -p "${MIRROR_DIR}" "${TRASH_DIR}" "${D1_DIR}" "${D1_MONTHLY_DIR}" \ +mkdir -p "${MIRROR_DIR}" "${TRASH_DIR}" "${PG_DIR}" "${PG_MONTHLY_DIR}" \ "${MANIFEST_DIR}" "${LOG_DIR}" log() { @@ -129,63 +127,31 @@ if (( PREVIOUS_COUNT > 0 && CURRENT_COUNT + SHRINK_TOLERANCE < PREVIOUS_COUNT )) alert "object count dropped ${PREVIOUS_COUNT} -> ${CURRENT_COUNT}; deleted objects retained in ${TRASH_DIR}" fi -# --- 2. D1 export ----------------------------------------------------------- -D1_EXPORT="${D1_DIR}/adrive-${STAMP_DAY}.sql.gz" -EXPORT_RESPONSE=$(curl -fsS -m 600 -X POST \ - "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/d1/database/${CLOUDFLARE_D1_DATABASE_ID}/export" \ - -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d '{"output_format":"polling"}') || fail "d1-export-start" - -# The polling export API returns signed_url/at_bookmark under `result` -# (per Cloudflare's D1 backup example); accept a doubly nested -# `result.result` too in case the response shape shifts. -extract_export_field() { - python3 -c ' -import json, sys -body = json.load(sys.stdin) -result = body.get("result") or {} -inner = result.get("result") if isinstance(result.get("result"), dict) else {} -print(result.get(sys.argv[1]) or inner.get(sys.argv[1]) or "") -' "$1" -} - -SIGNED_URL="" -for _attempt in $(seq 1 60); do - SIGNED_URL=$(printf '%s' "${EXPORT_RESPONSE}" | extract_export_field signed_url) - [[ -n "${SIGNED_URL}" ]] && break - BOOKMARK=$(printf '%s' "${EXPORT_RESPONSE}" | extract_export_field at_bookmark) - [[ -z "${BOOKMARK}" ]] && fail "d1-export-poll" - sleep 5 - EXPORT_RESPONSE=$(curl -fsS -m 600 -X POST \ - "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/d1/database/${CLOUDFLARE_D1_DATABASE_ID}/export" \ - -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "{\"output_format\":\"polling\",\"current_bookmark\":\"${BOOKMARK}\"}") \ - || fail "d1-export-poll" -done -[[ -n "${SIGNED_URL}" ]] || fail "d1-export-timeout" - -# Download to a temp file first so a failed rerun never truncates the -# day's previously good export. -D1_TMP="$(mktemp "${D1_DIR}/.export-XXXXXX")" -CLEANUP_FILES+=("${D1_TMP}") -curl -fsS -m 600 "${SIGNED_URL}" | gzip >"${D1_TMP}" || fail "d1-download" -gzip -t "${D1_TMP}" || fail "d1-corrupt-export" -[[ -s "${D1_TMP}" ]] || fail "d1-empty-export" -mv "${D1_TMP}" "${D1_EXPORT}" -log "d1 export written: $(du -h "${D1_EXPORT}" | cut -f1) ${D1_EXPORT}" +# --- 2. Postgres dump ------------------------------------------------------- +# Plain SQL so a restore is `psql -f`. The dump excludes the derived +# search_documents and file_chunks tables: both rebuild from the copied +# rows (scripts/pg-rebuild-search.mjs and the indexing sweep). +PG_EXPORT="${PG_DIR}/adrive-${STAMP_DAY}.sql.gz" +PG_TMP="$(mktemp "${PG_DIR}/.export-XXXXXX")" +CLEANUP_FILES+=("${PG_TMP}") +pg_dump "${DATABASE_URL}" --no-owner --no-privileges \ + --exclude-table-data=search_documents --exclude-table-data=file_chunks \ + | gzip >"${PG_TMP}" || fail "pg-dump" +gzip -t "${PG_TMP}" || fail "pg-corrupt-dump" +[[ -s "${PG_TMP}" ]] || fail "pg-empty-dump" +mv "${PG_TMP}" "${PG_EXPORT}" +log "postgres dump written: $(du -h "${PG_EXPORT}" | cut -f1) ${PG_EXPORT}" # Monthly snapshot: first successful run of each month is kept for a year. -D1_MONTHLY="${D1_MONTHLY_DIR}/adrive-${STAMP_MONTH}.sql.gz" -if [[ ! -f "${D1_MONTHLY}" ]]; then - cp "${D1_EXPORT}" "${D1_MONTHLY}" - log "monthly d1 snapshot recorded: ${D1_MONTHLY}" +PG_MONTHLY="${PG_MONTHLY_DIR}/adrive-${STAMP_MONTH}.sql.gz" +if [[ ! -f "${PG_MONTHLY}" ]]; then + cp "${PG_EXPORT}" "${PG_MONTHLY}" + log "monthly postgres snapshot recorded: ${PG_MONTHLY}" fi # --- 3. Manifest ------------------------------------------------------------ # Object keys, sizes, and rclone-side hashes for every mirrored object, plus -# the D1 export digest — enough to audit a restore without the live account. +# the dump digest — enough to audit a restore without the live account. # The enumeration is written to a temp file and validated before the final # manifest is atomically published, so a failed lsjson can't publish a # malformed manifest under a success status. @@ -200,9 +166,9 @@ rclone lsjson -R --hash "${MIRROR_DIR}" --files-only >"${OBJECTS_TMP}" \ printf '{"generatedAt":"%s","objects":' "${NOW_ISO}" cat "${OBJECTS_TMP}" printf ',"objectCount":%s' "${CURRENT_COUNT}" - printf ',"d1Export":{"file":"%s","sha256":"%s"}' \ - "$(basename "${D1_EXPORT}")" \ - "$(shasum -a 256 "${D1_EXPORT}" | cut -d' ' -f1)" + printf ',"postgresDump":{"file":"%s","sha256":"%s"}' \ + "$(basename "${PG_EXPORT}")" \ + "$(shasum -a 256 "${PG_EXPORT}" | cut -d' ' -f1)" printf '}\n' } >"${MANIFEST_TMP}" || fail "manifest" python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${MANIFEST_TMP}" \ @@ -211,17 +177,17 @@ mv "${MANIFEST_TMP}" "${MANIFEST}" log "manifest written: ${MANIFEST}" # --- 4. Retention ----------------------------------------------------------- -# Daily D1 exports, manifests, logs, and deleted-object trash: 30 days. -# Monthly D1 snapshots: 12 months. -find "${D1_DIR}" -name '*.sql.gz' -mtime +30 -delete +# Daily dumps, manifests, logs, and deleted-object trash: 30 days. +# Monthly snapshots: 12 months. +find "${PG_DIR}" -name '*.sql.gz' -mtime +30 -delete find "${MANIFEST_DIR}" -name 'manifest-*.json' -mtime +30 -delete find "${LOG_DIR}" -name 'backup-*.log' -mtime +30 -delete find "${BACKUP_ROOT}/r2-deleted" -mindepth 1 -maxdepth 1 -type d \ -mtime +30 -exec rm -rf {} + -find "${D1_MONTHLY_DIR}" -name '*.sql.gz' -mtime +366 -delete +find "${PG_MONTHLY_DIR}" -name '*.sql.gz' -mtime +366 -delete # --- 5. Status -------------------------------------------------------------- printf '{"status":"ok","at":"%s","objects":%s,"d1Export":"%s"}\n' \ - "${NOW_ISO}" "${CURRENT_COUNT}" "$(basename "${D1_EXPORT}")" \ + "${NOW_ISO}" "${CURRENT_COUNT}" "$(basename "${PG_EXPORT}")" \ >"${STATUS_FILE}" log "backup run complete" diff --git a/scripts/backup/install-backup-host.sh b/scripts/backup/install-backup-host.sh index 615c2b8..7428cee 100755 --- a/scripts/backup/install-backup-host.sh +++ b/scripts/backup/install-backup-host.sh @@ -41,7 +41,7 @@ fi echo echo "Next steps:" -echo " 1. Edit ${INSTALL_DIR}/backup.env (rclone remote, account id, D1 id, API token, webhook)." +echo " 1. Edit ${INSTALL_DIR}/backup.env (rclone remote, DATABASE_URL, webhook)." echo " 2. rclone config create adrive-r2 s3 provider=Cloudflare ... (read-only R2 token)" echo " 3. Run ${INSTALL_DIR}/backup.sh once by hand and check \$BACKUP_ROOT/last-run.json." echo " 4. Perform the restore drill in docs/backup-restore.md before trusting it." diff --git a/scripts/check-wrangler-drift.mjs b/scripts/check-wrangler-drift.mjs index 1487373..e011145 100644 --- a/scripts/check-wrangler-drift.mjs +++ b/scripts/check-wrangler-drift.mjs @@ -84,13 +84,6 @@ export const sameJson = (left, right) => JSON.stringify(normalize(left)) === JSON.stringify(normalize(right)); const placeholderIds = (block, label, drift) => { - for (const entry of block.d1_databases ?? []) { - if (String(entry.database_id ?? '').includes('replace-with-')) { - drift.push( - `${label} D1 ${entry.binding ?? 'DB'}: still has a placeholder database_id` - ); - } - } for (const entry of block.kv_namespaces ?? []) { if (String(entry.id ?? '').includes('replace-with-')) { drift.push( @@ -181,12 +174,7 @@ const main = () => { const bindingNames = (entries) => (entries ?? []).map((entry) => entry.binding).sort(); - for (const key of [ - 'd1_databases', - 'r2_buckets', - 'kv_namespaces', - 'hyperdrive' - ]) { + for (const key of ['r2_buckets', 'kv_namespaces', 'hyperdrive']) { if (!sameJson(bindingNames(config[key]), bindingNames(prod[key]))) { drift.push( `${key} bindings: local=${JSON.stringify(bindingNames(config[key]))} production=${JSON.stringify(bindingNames(prod[key]))}` diff --git a/scripts/release.sh b/scripts/release.sh index 536bb57..c4ba0c6 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -21,7 +21,7 @@ echo "Releasing ${COMMIT} to env ${ENV_NAME}" # Only the target env's resource ids must be real. The top-level D1 id is # an intentional local-dev placeholder and is ignored here. if ! bun "${ROOT}/scripts/check-wrangler-drift.mjs" --placeholders-only --env "${ENV_NAME}"; then - echo "Create the D1 database / KV namespace and paste their ids first." >&2 + echo "Create the Hyperdrive config / KV namespace and paste their ids first." >&2 exit 1 fi @@ -45,13 +45,15 @@ step "Deploy dry run" bun x wrangler deploy --dry-run --env "${ENV_NAME}" (cd "${ROOT}/apps/site" && bun x wrangler deploy --dry-run) -step "D1 migrations" +step "Postgres migrations" # Migrations run before the new Worker so both old and new code briefly run # against the migrated schema. Keep every migration backwards-compatible # for at least one release (additive columns/tables; no drops or renames # until the release after the code stops using them) so wrangler rollback -# stays safe. -bun x wrangler d1 migrations apply DB --env "${ENV_NAME}" --remote +# stays safe. DATABASE_URL must point at the production database with a +# role that owns the schema (see docs/release.md). +: "${DATABASE_URL:?DATABASE_URL is required to run Postgres migrations}" +bun "${WEB}/scripts/pg-migrate.mjs" --url "${DATABASE_URL}" step "Deploy" bun x wrangler deploy --env "${ENV_NAME}" From 89581235e7e08cab3c3091dea1a84b0cfd91b5f6 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 02:42:13 -0700 Subject: [PATCH 6/7] FIX: Move D1 data from per-table exports wrangler d1 export refuses databases with FTS5 virtual tables, so the runbook and script take a directory of per-table exports. Verified against the local D1 state. Co-Authored-By: Claude Fable 5.1 --- apps/web/scripts/d1-to-postgres.mjs | 58 +++++++++++++++++++++-------- docs/release.md | 26 ++++++++++--- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/apps/web/scripts/d1-to-postgres.mjs b/apps/web/scripts/d1-to-postgres.mjs index 63fa388..4b09692 100644 --- a/apps/web/scripts/d1-to-postgres.mjs +++ b/apps/web/scripts/d1-to-postgres.mjs @@ -1,17 +1,24 @@ // One-off data move from the D1 export of a single-tenant adrive instance -// into Postgres. Reads the SQL dump produced by -// `wrangler d1 export DB --env production --remote --output d1.sql`, -// replays it into an in-memory SQLite database, then copies each table -// row by row with type conversion (0/1 -> boolean, ISO text -> timestamptz). +// into Postgres. `wrangler d1 export` refuses databases that contain FTS5 +// virtual tables, so export one table at a time into a directory: // -// bun apps/web/scripts/d1-to-postgres.mjs --dump d1.sql --url postgres://... -// bun apps/web/scripts/d1-to-postgres.mjs --dump d1.sql --url ... --wipe # truncate first +// for t in files file_versions tags file_tags site_assets api_keys \ +// pending_site_asset_deletes instance_secrets; do +// wrangler d1 export DB --env production --remote --table $t --output d1/$t.sql +// done +// +// Each file is replayed into an in-memory SQLite database, then copied row +// by row with type conversion (0/1 -> boolean, ISO text -> timestamptz). +// +// bun apps/web/scripts/d1-to-postgres.mjs --dump d1/ --url postgres://... +// bun apps/web/scripts/d1-to-postgres.mjs --dump d1/ --url ... --wipe # truncate first // // R2 objects do not move. Keyword search documents are rebuilt from the // copied rows; semantic vectors are not carried over (Vectorize is gone), // so every file is left in index_state = 'pending' for the queue/cron to // re-embed. -import { readFileSync } from 'node:fs'; +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import Pg from 'pg'; @@ -31,16 +38,35 @@ if (!dumpPath || !url) { } const sqlite = new DatabaseSync(':memory:'); -// D1 exports include the FTS virtual tables and their shadow tables, which -// node:sqlite may not be able to create. Skip anything mentioning them. -const skip = /files_fts|files_trgm|sqlite_sequence|_cf_KV|d1_migrations/i; -const statements = readFileSync(dumpPath, 'utf8') - .split(/;\s*\n/) - .map((statement) => statement.trim()) - .filter((statement) => statement && !skip.test(statement)); -for (const statement of statements) sqlite.exec(`${statement};`); +const dumpFiles = statSync(dumpPath).isDirectory() + ? readdirSync(dumpPath) + .filter((name) => name.endsWith('.sql')) + .sort() + .map((name) => join(dumpPath, name)) + : [dumpPath]; +// Per-table exports repeat PRAGMA lines and reference tables that another +// file creates. The scratch schema only exists so the INSERTs replay, so +// foreign key clauses are dropped rather than ordering the files. +const skip = + /files_fts|files_trgm|sqlite_sequence|_cf_KV|d1_migrations|^PRAGMA/i; +const withoutForeignKeys = (statement) => + statement.replace(/\s+REFERENCES\s+\w+\s*\([^)]*\)(\s+ON DELETE \w+)?/gi, ''); +for (const file of dumpFiles) { + const statements = readFileSync(file, 'utf8') + .split(/;\s*\n/) + .map((statement) => statement.trim()) + .filter((statement) => statement && !skip.test(statement)); + for (const statement of statements) { + sqlite.exec(`${withoutForeignKeys(statement)};`); + } +} +const tableExists = (table) => + sqlite + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table) !== undefined; -const rows = (table) => sqlite.prepare(`SELECT * FROM ${table}`).all(); +const rows = (table) => + tableExists(table) ? sqlite.prepare(`SELECT * FROM ${table}`).all() : []; const bool = (value) => value === 1 || value === true; const ts = (value) => (value == null ? null : new Date(value).toISOString()); diff --git a/docs/release.md b/docs/release.md index 45509b2..1374841 100644 --- a/docs/release.md +++ b/docs/release.md @@ -138,13 +138,29 @@ carried over; sign in again afterwards. Semantic vectors are not carried over either; every file is left `pending` and re-embeds through the indexing sweep. +`wrangler d1 export` refuses databases that contain FTS5 virtual tables, +so export one table at a time from a checkout that still has the D1 +binding (the commit before this one): + +``` +cd apps/web +mkdir -p /tmp/adrive-d1 +for t in files file_versions tags file_tags site_assets api_keys \ + pending_site_asset_deletes instance_secrets; do + bun x wrangler d1 export DB --env production --remote --table $t --output /tmp/adrive-d1/$t.sql +done +``` + +Then from this checkout: + ``` cd apps/web -bun x wrangler d1 export DB --env production --remote --output /tmp/adrive-d1.sql # from the last D1 deploy -DATABASE_URL=postgres://... bun scripts/pg-migrate.mjs --url "$DATABASE_URL" -bun scripts/d1-to-postgres.mjs --dump /tmp/adrive-d1.sql --url "$DATABASE_URL" +export DATABASE_URL=postgres://... +bun scripts/pg-migrate.mjs --url "$DATABASE_URL" +bun scripts/d1-to-postgres.mjs --dump /tmp/adrive-d1 --url "$DATABASE_URL" ``` The script prints per-table counts and the Postgres totals at the end. -Compare them with `SELECT COUNT(*)` on the D1 export before flipping -DNS. Run it with `--wipe` to truncate and retry. +Compare them with the row counts in the exports before flipping DNS. Run +it with `--wipe` to truncate and retry. Tested against the local D1 state +on 2026-09-09. From 0477bda394e4f24715f5e75f1ce480044a672901 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:15:02 -0700 Subject: [PATCH 7/7] BUGFIX: Preserve data and serialize Postgres writes --- .../0003_search_normalization.sql | 20 ++++ apps/web/scripts/d1-to-postgres.mjs | 99 +++++++++++++++---- apps/web/src/lib/server/indexing-sql.ts | 4 +- apps/web/src/lib/server/search-candidates.ts | 40 +++++--- apps/web/src/lib/server/search-index.ts | 41 +++++--- apps/web/src/lib/server/services/auth.ts | 24 ++--- apps/web/src/lib/server/services/semantic.ts | 52 +++++++--- .../lib/server/services/sites/internals.ts | 10 +- .../src/lib/server/services/sites/sessions.ts | 64 ++++++++---- apps/web/src/lib/server/services/tags.ts | 18 ++-- docs/backup-restore.md | 4 + docs/release.md | 12 ++- scripts/backup/backup.sh | 8 +- scripts/backup/install-backup-host.sh | 1 + 14 files changed, 279 insertions(+), 118 deletions(-) create mode 100644 apps/web/migrations-pg/0003_search_normalization.sql diff --git a/apps/web/migrations-pg/0003_search_normalization.sql b/apps/web/migrations-pg/0003_search_normalization.sql new file mode 100644 index 0000000..cb167f8 --- /dev/null +++ b/apps/web/migrations-pg/0003_search_normalization.sql @@ -0,0 +1,20 @@ +-- migrate:up +-- Keep literal metadata lexemes, and give every field the same English +-- normalization so one query can match terms spanning metadata and body. +-- Merge each field's canonical vector text to retain original positions: +-- tsvector concatenation would shift aliases and create synthetic phrases. +-- PostgreSQL rewrites existing generated values and maintains the GIN index. +ALTER TABLE search_documents ALTER COLUMN tsv SET EXPRESSION AS ( + (setweight(to_tsvector('simple', name), 'A')::text + || ' ' || setweight(to_tsvector('english', name), 'A')::text)::tsvector + || (setweight(to_tsvector('simple', tags), 'B')::text + || ' ' || setweight(to_tsvector('english', tags), 'B')::text)::tsvector + || setweight(to_tsvector('english', body), 'C') +); + +-- migrate:down +ALTER TABLE search_documents ALTER COLUMN tsv SET EXPRESSION AS ( + setweight(to_tsvector('simple', name), 'A') + || setweight(to_tsvector('simple', tags), 'B') + || setweight(to_tsvector('english', body), 'C') +); diff --git a/apps/web/scripts/d1-to-postgres.mjs b/apps/web/scripts/d1-to-postgres.mjs index 4b09692..c01af56 100644 --- a/apps/web/scripts/d1-to-postgres.mjs +++ b/apps/web/scripts/d1-to-postgres.mjs @@ -3,7 +3,8 @@ // virtual tables, so export one table at a time into a directory: // // for t in files file_versions tags file_tags site_assets api_keys \ -// pending_site_asset_deletes instance_secrets; do +// pending_site_asset_deletes instance_secrets \ +// site_upload_sessions staged_site_assets; do // wrangler d1 export DB --env production --remote --table $t --output d1/$t.sql // done // @@ -19,7 +20,7 @@ // re-embed. import { readdirSync, readFileSync, statSync } from 'node:fs'; import { join } from 'node:path'; -import { DatabaseSync } from 'node:sqlite'; +import { constants, DatabaseSync } from 'node:sqlite'; import Pg from 'pg'; const arg = (flag) => { @@ -37,36 +38,75 @@ if (!dumpPath || !url) { process.exit(1); } -const sqlite = new DatabaseSync(':memory:'); +const sqlite = new DatabaseSync(':memory:', { + enableForeignKeyConstraints: false +}); const dumpFiles = statSync(dumpPath).isDirectory() ? readdirSync(dumpPath) .filter((name) => name.endsWith('.sql')) .sort() .map((name) => join(dumpPath, name)) : [dumpPath]; -// Per-table exports repeat PRAGMA lines and reference tables that another -// file creates. The scratch schema only exists so the INSERTs replay, so -// foreign key clauses are dropped rather than ordering the files. -const skip = - /files_fts|files_trgm|sqlite_sequence|_cf_KV|d1_migrations|^PRAGMA/i; -const withoutForeignKeys = (statement) => - statement.replace(/\s+REFERENCES\s+\w+\s*\([^)]*\)(\s+ON DELETE \w+)?/gi, ''); +// Let SQLite parse the complete exports, including multiline SQL literals. +// Foreign keys stay disabled while files referring to each other replay. +// Ignore actual export PRAGMAs through SQLite's parser, not text matching. +sqlite.setAuthorizer((action) => + action === constants.SQLITE_PRAGMA + ? constants.SQLITE_IGNORE + : constants.SQLITE_OK +); for (const file of dumpFiles) { - const statements = readFileSync(file, 'utf8') - .split(/;\s*\n/) - .map((statement) => statement.trim()) - .filter((statement) => statement && !skip.test(statement)); - for (const statement of statements) { - sqlite.exec(`${withoutForeignKeys(statement)};`); - } + sqlite.exec(readFileSync(file, 'utf8')); } +sqlite.setAuthorizer(null); const tableExists = (table) => sqlite .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?") .get(table) !== undefined; +const missingTables = [ + 'files', + 'file_versions', + 'tags', + 'file_tags', + 'site_assets', + 'api_keys', + 'pending_site_asset_deletes', + 'instance_secrets', + 'site_upload_sessions', + 'staged_site_assets' +].filter((table) => !tableExists(table)); +if (missingTables.length > 0) { + sqlite.close(); + throw new Error( + `D1 export is missing required tables: ${missingTables.join(', ')}` + ); +} -const rows = (table) => - tableExists(table) ? sqlite.prepare(`SELECT * FROM ${table}`).all() : []; +const rows = (table) => sqlite.prepare(`SELECT * FROM ${table}`).all(); +// Staging is not resumed after cutover, but its stored objects still need +// cleanup ownership. Never enqueue an object referenced by published assets. +const storedStagedAssets = sqlite + .prepare( + ` + SELECT a.r2_key, s.file_id, s.version, + EXISTS (SELECT 1 FROM site_assets live WHERE live.r2_key = a.r2_key) AS is_live + FROM staged_site_assets a + LEFT JOIN site_upload_sessions s ON s.id = a.session_id + WHERE a.r2_key IS NOT NULL +` + ) + .all(); +if ( + storedStagedAssets.some( + (asset) => asset.file_id == null || asset.version == null + ) +) { + sqlite.close(); + throw new Error( + 'D1 export contains stored staged assets without their upload sessions' + ); +} +const stagedCleanup = storedStagedAssets.filter((asset) => asset.is_live === 0); const bool = (value) => value === 1 || value === true; const ts = (value) => (value == null ? null : new Date(value).toISOString()); @@ -91,7 +131,7 @@ try { await client.query('BEGIN'); if (wipe) { await client.query( - 'TRUNCATE files, tags, api_keys, device_codes, site_upload_sessions, pending_site_asset_deletes, instance_secrets CASCADE' + 'TRUNCATE files, tags, api_keys, device_codes, dashboard_sessions, credential_state, site_upload_sessions, pending_site_asset_deletes, instance_secrets CASCADE' ); } await copy( @@ -164,6 +204,10 @@ try { (row) => ({ ...row, created_at: ts(row.created_at), + text_content: + row.text_content == null + ? null + : row.text_content.replaceAll('\u0000', ''), thumbnail_size_bytes: row.thumbnail_size_bytes ?? 0 }) ); @@ -215,6 +259,20 @@ try { ['id', 'content_grant_signing_key', 'created_at'], (row) => ({ ...row, created_at: ts(row.created_at) }) ); + const queuedAt = new Date().toISOString(); + let queuedStagedAssets = 0; + for (const asset of stagedCleanup) { + const queued = await client.query( + `INSERT INTO pending_site_asset_deletes (r2_key, file_id, version, queued_at) + SELECT $1, $2, $3, $4 + WHERE NOT EXISTS (SELECT 1 FROM site_assets WHERE r2_key = $1) + ON CONFLICT (r2_key) DO NOTHING + RETURNING r2_key`, + [asset.r2_key, asset.file_id, asset.version, queuedAt] + ); + queuedStagedAssets += queued.rowCount; + } + console.log(`staged_site_assets queued for cleanup: ${queuedStagedAssets}`); // Sessions, device codes, and the passcode hash are not carried over; // everyone signs in again. Keyword search documents are rebuilt. await client.query(` @@ -236,4 +294,5 @@ try { throw cause; } finally { await client.end(); + sqlite.close(); } diff --git a/apps/web/src/lib/server/indexing-sql.ts b/apps/web/src/lib/server/indexing-sql.ts index d306fe0..7771f52 100644 --- a/apps/web/src/lib/server/indexing-sql.ts +++ b/apps/web/src/lib/server/indexing-sql.ts @@ -88,8 +88,10 @@ export const storeExtractedText = ( .withTransaction( Effect.gen(function* () { yield* holdLease(sql, lease); + // PostgreSQL text rejects NUL bytes; match the semantic chunker's + // cleanup before writing either the source text or keyword index. yield* sql` - UPDATE file_versions SET text_content = ${text} + UPDATE file_versions SET text_content = ${text.replaceAll('\u0000', '')} WHERE file_id = ${lease.fileId} AND version = ${lease.version}`; yield* sql` UPDATE files SET index_cursor = 1 diff --git a/apps/web/src/lib/server/search-candidates.ts b/apps/web/src/lib/server/search-candidates.ts index c0cb3e1..d1e12f5 100644 --- a/apps/web/src/lib/server/search-candidates.ts +++ b/apps/web/src/lib/server/search-candidates.ts @@ -1,4 +1,5 @@ import type { PgClient } from '@effect/sql-pg'; +import { Effect } from 'effect'; // Candidate pool for one ranking pass. Search pages 50 at a time out of // this list; keep it large enough for a few pages without scanning the @@ -6,8 +7,8 @@ import type { PgClient } from '@effect/sql-pg'; export const SEARCH_CANDIDATE_LIMIT = 200; // word_similarity scores how well the query matches any substring of the -// name, so a typo ("reprot") still finds "Quarterly report.pdf". Kept in -// code rather than pg_trgm's session threshold so the cutoff is explicit. +// name, so a typo ("reprot") still finds "Quarterly report.pdf". Apply the +// cutoff transaction-locally so the index operator uses the same threshold. export const TRIGRAM_THRESHOLD = 0.3; export interface RankedRow { @@ -37,9 +38,9 @@ const visibleFile = (sql: PgClient.PgClient, filter: CandidateFilter) => AND (f.expires_at IS NULL OR f.expires_at > ${filter.now}) ${selectedTagFilter(sql, filter.tagIds)}`; -// Names and tags are indexed with the `simple` dictionary and bodies with -// `english`, so the query is parsed both ways and OR-ed: "Quarterly" must -// match the name token `quarterly` and the body stem `quarter`. +// Every field has English lexemes, so one native websearch query preserves +// phrase/OR/NOT semantics across metadata and body. Literal metadata lexemes +// also support identifiers consisting entirely of stopwords (such as "the"). export const fullTextCandidates = ( sql: PgClient.PgClient, query: string, @@ -50,8 +51,10 @@ export const fullTextCandidates = ( FROM search_documents d JOIN files f ON f.id = d.file_id CROSS JOIN ( - SELECT websearch_to_tsquery('simple', ${query}) - || websearch_to_tsquery('english', ${query}) AS query + SELECT CASE WHEN numnode(english_query) = 0 + THEN websearch_to_tsquery('simple', ${query}) + ELSE english_query END AS query + FROM (SELECT websearch_to_tsquery('english', ${query}) AS english_query) parsed ) q WHERE d.tsv @@ q.query AND ${visibleFile(sql, filter)} @@ -64,12 +67,17 @@ export const trigramCandidates = ( query: string, filter: CandidateFilter ) => - sql` - SELECT d.file_id, word_similarity(${query}, d.name) AS score - FROM search_documents d - JOIN files f ON f.id = d.file_id - WHERE d.chunk_no = 0 - AND word_similarity(${query}, d.name) > ${TRIGRAM_THRESHOLD}::real - AND ${visibleFile(sql, filter)} - ORDER BY score DESC, d.file_id - LIMIT ${SEARCH_CANDIDATE_LIMIT}`; + sql.withTransaction( + sql`SELECT set_config('pg_trgm.word_similarity_threshold', ${String(TRIGRAM_THRESHOLD)}, true)`.pipe( + Effect.andThen(sql` + SELECT d.file_id, word_similarity(${query}, d.name) AS score + FROM search_documents d + JOIN files f ON f.id = d.file_id + WHERE d.chunk_no = 0 + AND d.name %> ${query} + AND word_similarity(${query}, d.name) > ${TRIGRAM_THRESHOLD}::real + AND ${visibleFile(sql, filter)} + ORDER BY score DESC, d.file_id + LIMIT ${SEARCH_CANDIDATE_LIMIT}`) + ) + ); diff --git a/apps/web/src/lib/server/search-index.ts b/apps/web/src/lib/server/search-index.ts index 4d7b515..fce2f28 100644 --- a/apps/web/src/lib/server/search-index.ts +++ b/apps/web/src/lib/server/search-index.ts @@ -1,13 +1,22 @@ import type { PgClient } from '@effect/sql-pg'; import { Effect } from 'effect'; +// Every keyword writer shares this transaction lock. Acquire it in a +// separate statement so reads after waiting use a fresh database snapshot. +// A nested helper retains the lock until its caller's transaction commits. +// Stable advisory namespace: ASCII "adri" / "sear" for aDrive search. +const keywordWriteLock = (sql: PgClient.PgClient) => + sql`SELECT pg_advisory_xact_lock(1633972841, 1936023922)`; + // Postgres keyword index. One search_documents row per file today, so a // refresh is delete then insert; chunk_no exists for splitting long bodies -// later. Runs inside whatever transaction the caller holds. +// later. Tag renames and document replacements must serialize together. export const refreshSearchDocument = (sql: PgClient.PgClient, fileId: string) => - sql`DELETE FROM search_documents WHERE file_id = ${fileId}`.pipe( - Effect.andThen( - sql`INSERT INTO search_documents (file_id, chunk_no, name, tags, body) + sql.withTransaction( + Effect.gen(function* () { + yield* keywordWriteLock(sql); + yield* sql`DELETE FROM search_documents WHERE file_id = ${fileId}`; + yield* sql`INSERT INTO search_documents (file_id, chunk_no, name, tags, body) SELECT f.id, 0, @@ -22,16 +31,20 @@ export const refreshSearchDocument = (sql: PgClient.PgClient, fileId: string) => FROM files f JOIN file_versions v ON v.file_id = f.id AND v.version = f.current_version - WHERE f.id = ${fileId}` - ), - Effect.asVoid + WHERE f.id = ${fileId}`; + }) ); export const refreshAllIndexedTags = (sql: PgClient.PgClient) => - sql`UPDATE search_documents d - SET tags = COALESCE(( - SELECT string_agg(t.name, ' ' ORDER BY t.normalized_name) - FROM file_tags ft - JOIN tags t ON t.id = ft.tag_id - WHERE ft.file_id = d.file_id - ), '')`.pipe(Effect.asVoid); + sql.withTransaction( + Effect.gen(function* () { + yield* keywordWriteLock(sql); + yield* sql`UPDATE search_documents d + SET tags = COALESCE(( + SELECT string_agg(t.name, ' ' ORDER BY t.normalized_name) + FROM file_tags ft + JOIN tags t ON t.id = ft.tag_id + WHERE ft.file_id = d.file_id + ), '')`; + }) + ); diff --git a/apps/web/src/lib/server/services/auth.ts b/apps/web/src/lib/server/services/auth.ts index dcb1fce..5e2f446 100644 --- a/apps/web/src/lib/server/services/auth.ts +++ b/apps/web/src/lib/server/services/auth.ts @@ -639,24 +639,24 @@ const makeAuth = Effect.gen(function* () { const generated = yield* makeApiKey(row.name); const consumedAt = now.toISOString(); - // Guarded insert plus consume in one transaction: a concurrent poll - // that already consumed the code sees zero rows from both statements - // and the key it never minted is rolled back with it. + // Lock the approval before inserting a key. A concurrent poll must + // observe the consumed state before it can mint another credential. const completed = yield* sql .withTransaction( Effect.gen(function* () { - const inserted = yield* sql` + const approved = yield* sql` + SELECT device_code_hash FROM device_codes + WHERE device_code_hash = ${hash} AND status = 'approved' + FOR UPDATE + `; + if (approved.length !== 1) return false; + yield* sql` INSERT INTO api_keys ( id, name, prefix, secret_hash, created_at ) - SELECT ${generated.row.id}, ${generated.row.name}, + VALUES (${generated.row.id}, ${generated.row.name}, ${generated.row.prefix}, ${generated.secretHash}, - ${generated.row.createdAt} - WHERE EXISTS ( - SELECT 1 FROM device_codes - WHERE device_code_hash = ${hash} AND status = 'approved' - ) - RETURNING id + ${generated.row.createdAt}) `; const consumed = yield* sql` UPDATE device_codes @@ -665,7 +665,7 @@ const makeAuth = Effect.gen(function* () { WHERE device_code_hash = ${hash} AND status = 'approved' RETURNING device_code_hash `; - return inserted.length === 1 && consumed.length === 1; + return consumed.length === 1; }) ) .pipe(storageError('complete device authorization')); diff --git a/apps/web/src/lib/server/services/semantic.ts b/apps/web/src/lib/server/services/semantic.ts index b6e111d..cd8c1f4 100644 --- a/apps/web/src/lib/server/services/semantic.ts +++ b/apps/web/src/lib/server/services/semantic.ts @@ -4,11 +4,18 @@ import { upsertFileChunks, type VectorChunk } from '../indexing-sql'; import { PgSql } from '../pg'; import { createQueryEmbeddingCache } from '../query-embedding-cache'; import { selectedTagFilter, type CandidateFilter } from '../search-candidates'; +import { CHUNK_CHARACTERS, CHUNK_OVERLAP_CHARACTERS } from '../semantic-policy'; +import { searchTextLimit } from '../search-text'; const EMBEDDING_MODEL = '@cf/baai/bge-small-en-v1.5'; // Vector candidates feed reciprocal rank fusion beside the keyword pool; // one hundred nearest files is plenty for a few result pages. const SEMANTIC_CANDIDATE_LIMIT = 100; +// Reserve enough chunk candidates for 100 distinct files even when each +// nearer file has the maximum number produced by chunkSearchText. +const SEMANTIC_CHUNK_LIMIT = + SEMANTIC_CANDIDATE_LIMIT * + Math.ceil(searchTextLimit / (CHUNK_CHARACTERS - CHUNK_OVERLAP_CHARACTERS)); export interface EmbedderShape { readonly enabled: boolean; @@ -107,26 +114,39 @@ export const makeVectorIndex = ( ), // The nearest chunk stands in for its file, and visibility and tags // are filtered here so the caller never sees a file it cannot open. + // pgvector >= 0.8 iterative scans continue past filtered-out neighbors; + // the chunk limit is applied before grouping so HNSW can serve the sort. search: (vector, filter) => vector === null ? Effect.succeed([]) - : sql<{ file_id: string }>` - SELECT c.file_id - FROM file_chunks c - JOIN files f ON f.id = c.file_id AND f.current_version = c.version - WHERE c.embedding IS NOT NULL - AND f.deleted_at IS NULL - AND (f.expires_at IS NULL OR f.expires_at > ${filter.now}) - ${selectedTagFilter(sql, filter.tagIds)} - GROUP BY c.file_id - ORDER BY MIN(c.embedding <=> ${vectorLiteral(vector)}::vector), c.file_id - LIMIT ${SEMANTIC_CANDIDATE_LIMIT}`.pipe( - Effect.map((rows) => rows.map((row) => ({ fileId: row.file_id }))), - Effect.mapError( - (cause) => - new StorageError({ operation: 'query semantic vectors', cause }) + : sql + .withTransaction( + sql`SET LOCAL hnsw.iterative_scan = strict_order`.pipe( + Effect.andThen(sql<{ file_id: string }>` + WITH nearest_chunks AS MATERIALIZED ( + SELECT c.file_id, c.embedding <=> ${vectorLiteral(vector)}::vector AS distance + FROM file_chunks c + JOIN files f ON f.id = c.file_id AND f.current_version = c.version + WHERE c.embedding IS NOT NULL + AND f.deleted_at IS NULL + AND (f.expires_at IS NULL OR f.expires_at > ${filter.now}) + ${selectedTagFilter(sql, filter.tagIds)} + ORDER BY c.embedding <=> ${vectorLiteral(vector)}::vector + LIMIT ${SEMANTIC_CHUNK_LIMIT} + ) + SELECT file_id FROM nearest_chunks + GROUP BY file_id + ORDER BY MIN(distance), file_id + LIMIT ${SEMANTIC_CANDIDATE_LIMIT}`) + ) ) - ), + .pipe( + Effect.map((rows) => rows.map((row) => ({ fileId: row.file_id }))), + Effect.mapError( + (cause) => + new StorageError({ operation: 'query semantic vectors', cause }) + ) + ), count: sql<{ count: number }>` SELECT COUNT(*)::integer AS count FROM file_chunks WHERE embedding IS NOT NULL`.pipe( Effect.map((rows) => rows[0]?.count ?? 0), diff --git a/apps/web/src/lib/server/services/sites/internals.ts b/apps/web/src/lib/server/services/sites/internals.ts index 50ca185..15be274 100644 --- a/apps/web/src/lib/server/services/sites/internals.ts +++ b/apps/web/src/lib/server/services/sites/internals.ts @@ -189,6 +189,13 @@ export const createInternals = ({ sql, blobs, config }: CoreDeps) => { yield* sql .withTransaction( Effect.gen(function* () { + // Claim cleanup before reading keys. A concurrent publisher owns + // this row until commit; after it completes, its blobs are live. + const claimed = yield* sql<{ id: string }>` + UPDATE site_upload_sessions SET status = ${status} + WHERE id = ${session.id} AND status IN ('open', 'committing') + RETURNING id`; + if (claimed.length === 0) return; yield* sql` INSERT INTO pending_site_asset_deletes ( r2_key, file_id, version, queued_at @@ -197,9 +204,6 @@ export const createInternals = ({ sql, blobs, config }: CoreDeps) => { FROM staged_site_assets WHERE session_id = ${session.id} AND r2_key IS NOT NULL ON CONFLICT (r2_key) DO NOTHING`; - yield* sql` - UPDATE site_upload_sessions SET status = ${status} - WHERE id = ${session.id} AND status IN ('open', 'committing')`; yield* sql` DELETE FROM staged_site_assets WHERE session_id = ${session.id}`; }) diff --git a/apps/web/src/lib/server/services/sites/sessions.ts b/apps/web/src/lib/server/services/sites/sessions.ts index a47f8ed..157f19c 100644 --- a/apps/web/src/lib/server/services/sites/sessions.ts +++ b/apps/web/src/lib/server/services/sites/sessions.ts @@ -185,27 +185,37 @@ export const sessionOps = ( asset.contentType ); const uploadedAt = new Date().toISOString(); - const updated = yield* sql<{ path: string }>` - UPDATE staged_site_assets - SET r2_key = ${r2Key}, stored_size_bytes = ${stored.size}, - uploaded_at = ${uploadedAt} - WHERE session_id = ${session.id} AND path = ${path} AND r2_key IS NULL - AND EXISTS ( - SELECT 1 FROM site_upload_sessions + const updated = yield* sql + .withTransaction( + Effect.gen(function* () { + // Cleanup also locks the session before touching staged rows. + // Recheck after waiting so a late upload cannot lose its blob + // record behind an abort or an expiry sweep. + const open = yield* sql` + SELECT id FROM site_upload_sessions WHERE id = ${session.id} AND status = 'open' AND expires_at > ${uploadedAt} - ) - RETURNING path`.pipe( - Effect.mapError( - (cause) => - new StorageError({ operation: 'record staged site asset', cause }) - ), - Effect.catch((failure) => - compensateStagedBlob(session, r2Key).pipe( - Effect.andThen(Effect.fail(failure)) - ) + FOR UPDATE`; + if (open.length === 0) return []; + return yield* sql<{ path: string }>` + UPDATE staged_site_assets + SET r2_key = ${r2Key}, stored_size_bytes = ${stored.size}, + uploaded_at = ${uploadedAt} + WHERE session_id = ${session.id} AND path = ${path} AND r2_key IS NULL + RETURNING path`; + }) ) - ); + .pipe( + Effect.mapError( + (cause) => + new StorageError({ operation: 'record staged site asset', cause }) + ), + Effect.catch((failure) => + compensateStagedBlob(session, r2Key).pipe( + Effect.andThen(Effect.fail(failure)) + ) + ) + ); if (updated.length !== 1) { yield* compensateStagedBlob(session, r2Key); return yield* new InvalidRequest({ @@ -265,6 +275,24 @@ export const sessionOps = ( const commit = sql .withTransaction( Effect.gen(function* () { + if (session.version > 1) { + // Purge claims this row before enumerating blobs. Hold it + // through publication so purge either sees the new version + // or makes this publish ineligible before any promotion. + const current = yield* sql<{ id: string }>` + SELECT id FROM files + WHERE id = ${session.fileId} AND is_site = true + AND current_version = ${session.version - 1} + AND deleted_at IS NULL AND purge_state = 'none' + AND (expires_at IS NULL OR expires_at > ${publishedAt}) + FOR UPDATE`; + if (current.length !== 1) { + return yield* new StorageError({ + operation: 'commit site version', + cause: 'The site changed while it was publishing' + }); + } + } const guarded = yield* guard; if (guarded.length !== 1) { return yield* new StorageError({ diff --git a/apps/web/src/lib/server/services/tags.ts b/apps/web/src/lib/server/services/tags.ts index 17e910d..3108641 100644 --- a/apps/web/src/lib/server/services/tags.ts +++ b/apps/web/src/lib/server/services/tags.ts @@ -256,17 +256,15 @@ const makeTags = Effect.gen(function* () { forgetTagListCache(); }), setFileTags: Effect.fn('Tags.setFileTags')(function* (fileId, names) { - const file = yield* sql` - SELECT id FROM files WHERE id = ${fileId} LIMIT 1`.pipe( - Effect.mapError( - (cause) => new StorageError({ operation: 'find tagged file', cause }) - ) - ); - if (file.length === 0) return yield* new NotFound({ id: fileId }); - const resolved = yield* resolveNames(names); yield* sql .withTransaction( Effect.gen(function* () { + // Replacements must serialize even when the file has no tags. + // Locking only existing file_tags rows leaves that case unprotected. + const file = yield* sql` + SELECT id FROM files WHERE id = ${fileId} FOR UPDATE`; + if (file.length === 0) return yield* new NotFound({ id: fileId }); + const resolved = yield* resolveNames(names); yield* sql`DELETE FROM file_tags WHERE file_id = ${fileId}`; if (resolved.length > 0) { yield* sql` @@ -278,8 +276,8 @@ const makeTags = Effect.gen(function* () { }) ) .pipe( - Effect.mapError( - (cause) => new StorageError({ operation: 'set file tags', cause }) + Effect.catchTag('SqlError', (cause) => + Effect.fail(new StorageError({ operation: 'set file tags', cause })) ) ); forgetTagListCache(); diff --git a/docs/backup-restore.md b/docs/backup-restore.md index f649a80..3e32956 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -72,6 +72,10 @@ copy for when the account itself is unavailable. Always restore into a fresh database and cut over — never import over the production database, so it stays untouched for rollback. +The dump includes keyword search documents and semantic vectors, preserving +the search state of files already marked ready. No reindex is required for +a complete restore. + 1. Create a new PlanetScale Postgres database (or a local one for a drill). 2. `gunzip -k postgres/daily/adrive-.sql.gz` 3. `psql "" -f adrive-.sql` diff --git a/docs/release.md b/docs/release.md index 1374841..180f8a3 100644 --- a/docs/release.md +++ b/docs/release.md @@ -6,8 +6,8 @@ App Worker commands run from `apps/web`. Landing-site commands run from `apps/site` (no `--env`). `bun release` and the backup installer run from the repository root. -1. Create a PlanetScale Postgres database (region close to most users) - with the `vector` and `pg_trgm` extensions available, then a Hyperdrive +1. Create a PlanetScale Postgres 17+ database (region close to most users) + with `vector` 0.8 or newer and the `pg_trgm` extension available, then a Hyperdrive config pointing at its direct port 5432 with caching disabled: ``` @@ -146,7 +146,8 @@ binding (the commit before this one): cd apps/web mkdir -p /tmp/adrive-d1 for t in files file_versions tags file_tags site_assets api_keys \ - pending_site_asset_deletes instance_secrets; do + pending_site_asset_deletes instance_secrets \ + site_upload_sessions staged_site_assets; do bun x wrangler d1 export DB --env production --remote --table $t --output /tmp/adrive-d1/$t.sql done ``` @@ -164,3 +165,8 @@ The script prints per-table counts and the Postgres totals at the end. Compare them with the row counts in the exports before flipping DNS. Run it with `--wipe` to truncate and retry. Tested against the local D1 state on 2026-09-09. + +All listed tables must be exported, including empty ones. Unfinished site +uploads are not resumed: their stored assets enter the cleanup queue unless +the key is referenced by a published site asset. Export while writes to the +old drive are paused so the table snapshots describe the same state. diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index 87ef1f6..965d052 100755 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -128,14 +128,12 @@ if (( PREVIOUS_COUNT > 0 && CURRENT_COUNT + SHRINK_TOLERANCE < PREVIOUS_COUNT )) fi # --- 2. Postgres dump ------------------------------------------------------- -# Plain SQL so a restore is `psql -f`. The dump excludes the derived -# search_documents and file_chunks tables: both rebuild from the copied -# rows (scripts/pg-rebuild-search.mjs and the indexing sweep). +# Plain SQL so a restore is `psql -f`. Include search documents and vectors +# so files marked ready remain searchable immediately after restore. PG_EXPORT="${PG_DIR}/adrive-${STAMP_DAY}.sql.gz" PG_TMP="$(mktemp "${PG_DIR}/.export-XXXXXX")" CLEANUP_FILES+=("${PG_TMP}") pg_dump "${DATABASE_URL}" --no-owner --no-privileges \ - --exclude-table-data=search_documents --exclude-table-data=file_chunks \ | gzip >"${PG_TMP}" || fail "pg-dump" gzip -t "${PG_TMP}" || fail "pg-corrupt-dump" [[ -s "${PG_TMP}" ]] || fail "pg-empty-dump" @@ -187,7 +185,7 @@ find "${BACKUP_ROOT}/r2-deleted" -mindepth 1 -maxdepth 1 -type d \ find "${PG_MONTHLY_DIR}" -name '*.sql.gz' -mtime +366 -delete # --- 5. Status -------------------------------------------------------------- -printf '{"status":"ok","at":"%s","objects":%s,"d1Export":"%s"}\n' \ +printf '{"status":"ok","at":"%s","objects":%s,"postgresDump":"%s"}\n' \ "${NOW_ISO}" "${CURRENT_COUNT}" "$(basename "${PG_EXPORT}")" \ >"${STATUS_FILE}" log "backup run complete" diff --git a/scripts/backup/install-backup-host.sh b/scripts/backup/install-backup-host.sh index 7428cee..f7d851c 100755 --- a/scripts/backup/install-backup-host.sh +++ b/scripts/backup/install-backup-host.sh @@ -13,6 +13,7 @@ command -v rclone >/dev/null || { } command -v curl >/dev/null || { echo "curl is required" >&2; exit 1; } command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 1; } +command -v pg_dump >/dev/null || { echo "pg_dump is required (Postgres client tools)" >&2; exit 1; } umask 077 mkdir -p "${INSTALL_DIR}"