diff --git a/apps/server/src/lib/http.ts b/apps/server/src/lib/http.ts index 4892990..6d960b6 100644 --- a/apps/server/src/lib/http.ts +++ b/apps/server/src/lib/http.ts @@ -1,7 +1,8 @@ -import type { Request, Response, NextFunction, RequestHandler } from 'express'; +import path from 'node:path'; +import type { Request as ExpressRequest, Response, NextFunction, RequestHandler } from 'express'; /** Express 4 doesn't forward async rejections to the error handler. */ -export function wrap(fn: (req: Request, res: Response, next: NextFunction) => Promise): RequestHandler { +export function wrap(fn: (req: ExpressRequest, res: Response, next: NextFunction) => Promise): RequestHandler { return (req, res, next) => { fn(req, res, next).catch(next); }; @@ -12,3 +13,55 @@ export function intParam(value: string): number { if (!Number.isInteger(n) || n < 0) throw Object.assign(new Error('Invalid id'), { status: 400 }); return n; } + +/** Buffer the raw body and let Bun's fetch primitives parse the multipart form. */ +export async function readFormData(req: ExpressRequest) { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(chunk as Buffer); + const request = new Request('http://localhost' + req.originalUrl, { + method: 'POST', + headers: { 'content-type': req.headers['content-type'] ?? '' }, + body: Buffer.concat(chunks), + }); + return request.formData(); +} + +/** The File parts of a form field; string parts are dropped. */ +export function formFiles(form: Awaited>, field: string) { + return form.getAll(field).filter((f) => typeof f !== 'string'); +} + +/** + * A client-supplied filename made safe to join onto a directory. The name comes straight from the + * multipart header, so a crafted request can send `../../x` and write outside the target folder. + */ +export function sanitizeFilename(name: string): string { + return path + .basename(name) + .replace(/[^\w.\-()[\] ]+/g, '_') + .slice(-120); +} + +/** + * Start a server-sent event stream. `send` writes one JSON event; the returned cleanup hook runs + * once the client disconnects, alongside stopping the keep-alive ping. + */ +export function openEventStream( + req: ExpressRequest, + res: Response, +): { send: (event: unknown) => void; onClose: (fn: () => void) => void } { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + const heartbeat = setInterval(() => res.write(': ping\n\n'), 25_000); + const cleanups: (() => void)[] = [() => clearInterval(heartbeat)]; + req.on('close', () => { + for (const fn of cleanups) fn(); + }); + return { + send: (event) => res.write(`data: ${JSON.stringify(event)}\n\n`), + onClose: (fn) => cleanups.push(fn), + }; +} diff --git a/apps/server/src/lib/rowToMedia.ts b/apps/server/src/lib/rowToMedia.ts index 6b0a36f..c655277 100644 --- a/apps/server/src/lib/rowToMedia.ts +++ b/apps/server/src/lib/rowToMedia.ts @@ -1,7 +1,47 @@ import type { Media } from '@sakuya/shared'; +/** + * A media row joined with its library name and tag count — the shape rowToMedia expects. Append + * WHERE/ORDER BY; the media table is aliased `m`. + */ +export const MEDIA_SELECT = ` + SELECT m.*, l.name AS library_name, + (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count + FROM media m LEFT JOIN libraries l ON l.id = m.library_id`; + +/** A raw sqlite media row: snake_case columns plus the joined ones some queries add. */ +export interface MediaSqlRow { + id: number; + library_id: number; + source: Media['source']; + path: string; + filename: string; + type: Media['type']; + width: number | null; + height: number | null; + size_bytes: number; + duration_seconds: number | null; + content_hash: string | null; + created_at: number; + indexed_at: number | null; + tagged_at: number | null; + last_viewed_at: number | null; + view_progress: number; + view_count: number; + watched_seconds: number; + dwell_seconds: number; + liked: number; + liked_at: number | null; + perceptual_hash: string | null; + tag_count?: number; + library_name?: string | null; + sort_key?: number | string; + reason_tag?: string | null; + reason_related?: number | null; +} + /** Maps a raw sqlite media row (snake_case, joined columns) to the shared Media shape. */ -export function rowToMedia(row: any): Media { +export function rowToMedia(row: MediaSqlRow): Media { return { id: row.id, libraryId: row.library_id, diff --git a/apps/server/src/requestValidation.test.ts b/apps/server/src/requestValidation.test.ts new file mode 100644 index 0000000..20f5d71 --- /dev/null +++ b/apps/server/src/requestValidation.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from 'bun:test'; +import fs from 'node:fs'; +import path from 'node:path'; +import { useTestConfig } from './testConfig'; + +useTestConfig('validation', { port: 38780 }); +const BASE = 'http://localhost:38780'; + +const { db, sqlite, schema } = await import('./db'); +const { DOWNLOADER_COOKIES_DIR, DOWNLOADER_DIR } = await import('./lib/config'); +await import('./index'); + +const send = (method: string, url: string, body: unknown) => + fetch(BASE + url, { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + +test('an uploaded cookie file cannot be written outside the cookies folder', async () => { + const form = new FormData(); + form.append('files', new File(['# Netscape HTTP Cookie File'], '../../../escaped.txt')); + const res = await fetch(BASE + '/api/downloader/cookies', { method: 'POST', body: form }); + expect(res.status).toBe(201); + + const row = db.select().from(schema.downloadCookies).get()!; + expect(path.dirname(row.storedPath)).toBe(DOWNLOADER_COOKIES_DIR); + expect(fs.existsSync(row.storedPath)).toBe(true); + expect(fs.existsSync(path.join(DOWNLOADER_DIR, 'escaped.txt'))).toBe(false); +}); + +test('adding a deleted media id to a board skips it instead of failing the whole add', async () => { + const lib = db.insert(schema.libraries).values({ name: 'lib', createdAt: Date.now() }).returning().get(); + const media = db + .insert(schema.media) + .values({ + libraryId: lib.id, + source: 'upload', + path: '/tmp/kept.png', + filename: 'kept.png', + type: 'image', + createdAt: Date.now(), + }) + .returning() + .get(); + const board = (await (await send('POST', '/api/boards', { name: 'b' })).json()) as { id: number }; + + const res = await send('POST', `/api/boards/${board.id}/media`, { mediaIds: [media.id, 999_999] }); + expect(res.status).toBe(200); + expect(((await res.json()) as { itemCount: number }).itemCount).toBe(1); + + const removed = await send('POST', `/api/boards/${board.id}/media/remove-batch`, { mediaIds: [media.id, 999_999] }); + expect(((await removed.json()) as { itemCount: number }).itemCount).toBe(0); +}); + +test('a settings patch with one unknown key changes nothing', async () => { + const before = sqlite.query(`SELECT value FROM settings WHERE key = 'accent_color'`).get() as { + value: string; + } | null; + const res = await send('PATCH', '/api/settings', { accent_color: '#123456', not_a_setting: '1' }); + expect(res.status).toBe(400); + const after = sqlite.query(`SELECT value FROM settings WHERE key = 'accent_color'`).get() as { value: string } | null; + expect(after?.value).toBe(before?.value); +}); + +test('tag search treats _ and % as literal characters', async () => { + const lib = db.insert(schema.libraries).values({ name: 'tags', createdAt: Date.now() }).returning().get(); + const media = db + .insert(schema.media) + .values({ + libraryId: lib.id, + source: 'upload', + path: '/tmp/tagged.png', + filename: 'tagged.png', + type: 'image', + createdAt: Date.now(), + }) + .returning() + .get(); + await send('PATCH', `/api/media/${media.id}/tags`, { add: ['long_hair', 'longxhair', '100%_done'] }); + + const names = async (url: string) => + ((await (await fetch(BASE + url)).json()) as { name: string }[]).map((t) => t.name).sort(); + for (const scope of ['', `&libraryId=${lib.id}`]) { + expect(await names(`/api/tags?q=long_h${scope}`)).toEqual(['long_hair']); + expect(await names(`/api/tags?q=${encodeURIComponent('0%_')}${scope}`)).toEqual(['100%_done']); + } +}); + +test('attaching the same folder twice is refused', async () => { + const lib = db.insert(schema.libraries).values({ name: 'folders', createdAt: Date.now() }).returning().get(); + const dir = fs.mkdtempSync(path.join(DOWNLOADER_DIR, 'folder-')); + expect((await send('POST', `/api/libraries/${lib.id}/folders`, { path: dir })).status).toBe(201); + expect((await send('POST', `/api/libraries/${lib.id}/folders`, { path: dir })).status).toBe(409); +}); + +test('system info reports the thumbnail folder size', async () => { + const { THUMBS_DIR } = await import('./lib/config'); + fs.writeFileSync(path.join(THUMBS_DIR, 'size-probe.webp'), Buffer.alloc(1234)); + const info = (await (await fetch(BASE + '/api/system')).json()) as { thumbBytes: number }; + expect(info.thumbBytes).toBeGreaterThanOrEqual(1234); +}); diff --git a/apps/server/src/routes/boards.ts b/apps/server/src/routes/boards.ts index 6d90646..0e2a8e9 100644 --- a/apps/server/src/routes/boards.ts +++ b/apps/server/src/routes/boards.ts @@ -1,8 +1,9 @@ import { Router } from 'express'; import { z } from 'zod'; -import { and, eq } from 'drizzle-orm'; +import { and, eq, inArray } from 'drizzle-orm'; import { db, sqlite, schema } from '../db'; import { wrap, intParam } from '../lib/http'; +import { chunkIds } from '../lib/mediaByIds'; import type { BoardWithStats } from '@sakuya/shared'; export const boardsRouter = Router(); @@ -76,10 +77,18 @@ boardsRouter.post( if (!board) return res.status(404).json({ error: 'Not found' }); const { mediaIds } = mediaIdsBody.parse(req.body); const now = Date.now(); - db.insert(schema.boardMedia) - .values(mediaIds.map((mediaId) => ({ boardId: id, mediaId, addedAt: now }))) - .onConflictDoNothing() - .run(); + // Selected from media rather than inserted as given: an id deleted since the client loaded it + // would otherwise fail the foreign key and 500 the whole add. + sqlite.transaction(() => { + for (const part of chunkIds(mediaIds)) { + sqlite + .query( + `INSERT OR IGNORE INTO board_media (board_id, media_id, added_at) + SELECT ?, id, ? FROM media WHERE id IN (${part.map(() => '?').join(',')})`, + ) + .run(id, now, ...part); + } + })(); res.json(boardWithStats(board)); }), ); @@ -93,14 +102,13 @@ boardsRouter.post( const board = db.select().from(schema.boards).where(eq(schema.boards.id, id)).get(); if (!board) return res.status(404).json({ error: 'Not found' }); const { mediaIds } = mediaIdsBody.parse(req.body); - const remove = sqlite.transaction(() => { - for (const mediaId of mediaIds) { + sqlite.transaction(() => { + for (const part of chunkIds(mediaIds)) { db.delete(schema.boardMedia) - .where(and(eq(schema.boardMedia.boardId, id), eq(schema.boardMedia.mediaId, mediaId))) + .where(and(eq(schema.boardMedia.boardId, id), inArray(schema.boardMedia.mediaId, part))) .run(); } - }); - remove(); + })(); res.json(boardWithStats(board)); }), ); diff --git a/apps/server/src/routes/dashboard.ts b/apps/server/src/routes/dashboard.ts index 70e5d90..5dd2511 100644 --- a/apps/server/src/routes/dashboard.ts +++ b/apps/server/src/routes/dashboard.ts @@ -2,16 +2,11 @@ import { Router } from 'express'; import { sqlite } from '../db'; import { wrap } from '../lib/http'; import { libraryWithStats, librariesInOrder } from './libraries'; -import { rowToMedia } from '../lib/rowToMedia'; +import { rowToMedia, MEDIA_SELECT, type MediaSqlRow } from '../lib/rowToMedia'; import type { DashboardResponse } from '@sakuya/shared'; export const dashboardRouter = Router(); -const MEDIA_SELECT = ` - SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id`; - dashboardRouter.get( '/api/dashboard', wrap(async (_req, res) => { @@ -19,15 +14,15 @@ dashboardRouter.get( const continueWatching = ( sqlite .query(`${MEDIA_SELECT} WHERE m.view_progress > 0.01 AND m.view_progress < 0.98 ORDER BY m.last_viewed_at DESC LIMIT 12`) - .all() as any[] + .all() as MediaSqlRow[] ).map(rowToMedia); const recentlyViewed = ( sqlite .query(`${MEDIA_SELECT} WHERE m.last_viewed_at IS NOT NULL ORDER BY m.last_viewed_at DESC LIMIT 12`) - .all() as any[] + .all() as MediaSqlRow[] ).map(rowToMedia); const recentlyAdded = ( - sqlite.query(`${MEDIA_SELECT} ORDER BY m.created_at DESC, m.id DESC LIMIT 12`).all() as any[] + sqlite.query(`${MEDIA_SELECT} ORDER BY m.created_at DESC, m.id DESC LIMIT 12`).all() as MediaSqlRow[] ).map(rowToMedia); const likedCount = (sqlite.query('SELECT COUNT(*) AS c FROM media WHERE liked = 1').get() as { c: number }).c; const likedSample = sqlite diff --git a/apps/server/src/routes/discover.ts b/apps/server/src/routes/discover.ts index 64d01fe..d669f61 100644 --- a/apps/server/src/routes/discover.ts +++ b/apps/server/src/routes/discover.ts @@ -2,7 +2,8 @@ import { Router } from 'express'; import { z } from 'zod'; import { sqlite } from '../db'; import { wrap } from '../lib/http'; -import { rowToMedia } from '../lib/rowToMedia'; +import type { SQLQueryBindings } from 'bun:sqlite'; +import { rowToMedia, type MediaSqlRow } from '../lib/rowToMedia'; import { tasteVersion } from '../lib/tasteVersion'; import type { MediaListResponse } from '@sakuya/shared'; @@ -220,7 +221,7 @@ discoverRouter.get( const { maxScore } = snap; const conds: string[] = []; - const params: unknown[] = []; + const params: SQLQueryBindings[] = []; if (query.type) { conds.push('m.type = ?'); params.push(query.type); @@ -244,7 +245,7 @@ discoverRouter.get( : ( sqlite .query(`SELECT COUNT(*) AS c FROM media m ${conds.length ? 'WHERE ' + conds.join(' AND ') : ''}`) - .get(...(params as any[])) as { c: number } + .get(...params) as { c: number } ).c; const pageConds = [...conds]; @@ -277,7 +278,7 @@ discoverRouter.get( JOIN media m ON m.id = page.id LEFT JOIN libraries l ON l.id = m.library_id ORDER BY page.sort_key DESC, m.id DESC`; - const items = sqlite.query(sql).all(...(pageParams as any[]), query.limit) as any[]; + const items = sqlite.query(sql).all(...pageParams, query.limit) as MediaSqlRow[]; let nextCursor: string | null = null; if (items.length === query.limit) { diff --git a/apps/server/src/routes/downloader.ts b/apps/server/src/routes/downloader.ts index dc3f214..20ca35b 100644 --- a/apps/server/src/routes/downloader.ts +++ b/apps/server/src/routes/downloader.ts @@ -6,7 +6,7 @@ import crypto from 'node:crypto'; import { z } from 'zod'; import { eq } from 'drizzle-orm'; import { db, schema } from '../db'; -import { wrap, intParam } from '../lib/http'; +import { wrap, intParam, readFormData, formFiles, sanitizeFilename, openEventStream } from '../lib/http'; import { DOWNLOADER_COOKIES_DIR } from '../lib/config'; import { detectGalleryDl, installGalleryDl } from '../services/galleryDl'; import { @@ -78,22 +78,14 @@ downloaderRouter.get( downloaderRouter.post( '/api/downloader/cookies', wrap(async (req, res) => { - const chunks: Buffer[] = []; - for await (const chunk of req) chunks.push(chunk as Buffer); - const request = new Request('http://localhost/api/downloader/cookies', { - method: 'POST', - headers: { 'content-type': req.headers['content-type'] ?? '' }, - body: Buffer.concat(chunks), - }); - const form = await request.formData(); - const files = form.getAll('files').filter((f) => typeof f !== 'string'); + const files = formFiles(await readFormData(req), 'files'); if (files.length === 0) return res.status(400).json({ error: 'No files provided' }); const created: DownloadCookie[] = []; for (const file of files) { - const filename = (file as any).name || 'cookies.txt'; + const filename = sanitizeFilename(file.name || 'cookies.txt'); const dest = path.join(DOWNLOADER_COOKIES_DIR, `${crypto.randomBytes(6).toString('hex')}-${filename}`); - await fsp.writeFile(dest, Buffer.from(await (file as any).arrayBuffer())); + await fsp.writeFile(dest, Buffer.from(await file.arrayBuffer())); const row = db .insert(schema.downloadCookies) .values({ filename, storedPath: dest, uploadedAt: Date.now() }) @@ -261,47 +253,32 @@ downloaderRouter.post( ); downloaderRouter.get('/api/downloader/console/stream', (req, res) => { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }); - res.write(`data: ${JSON.stringify({ type: 'snapshot', buffer: getConsoleBuffer(), status: getConsoleStatus() })}\n\n`); + const stream = openEventStream(req, res); + stream.send({ type: 'snapshot', buffer: getConsoleBuffer(), status: getConsoleStatus() }); - const onData = (chunk: string) => res.write(`data: ${JSON.stringify({ type: 'data', chunk })}\n\n`); - const onStatus = (status: ConsoleSessionStatus) => res.write(`data: ${JSON.stringify({ type: 'status', status })}\n\n`); + const onData = (chunk: string) => stream.send({ type: 'data', chunk }); + const onStatus = (status: ConsoleSessionStatus) => stream.send({ type: 'status', status }); consoleEvents.on('data', onData); consoleEvents.on('status', onStatus); - const heartbeat = setInterval(() => res.write(': ping\n\n'), 25_000); - - req.on('close', () => { - clearInterval(heartbeat); + stream.onClose(() => { consoleEvents.off('data', onData); consoleEvents.off('status', onStatus); }); }); downloaderRouter.get('/api/downloader/stream', (req, res) => { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }); - res.write(`data: ${JSON.stringify({ type: 'snapshot', batches: listBatches() })}\n\n`); + const stream = openEventStream(req, res); + stream.send({ type: 'snapshot', batches: listBatches() }); - const onBatch = (batch: DownloadBatchWithItems) => res.write(`data: ${JSON.stringify({ type: 'batch', batch })}\n\n`); - const onItem = (item: DownloadItem) => res.write(`data: ${JSON.stringify({ type: 'item', item })}\n\n`); - const onLog = (log: DownloadLogLine) => res.write(`data: ${JSON.stringify({ type: 'log', log })}\n\n`); - const onRemoved = (payload: { id: number; batchId: number }) => - res.write(`data: ${JSON.stringify({ type: 'removed', ...payload })}\n\n`); + const onBatch = (batch: DownloadBatchWithItems) => stream.send({ type: 'batch', batch }); + const onItem = (item: DownloadItem) => stream.send({ type: 'item', item }); + const onLog = (log: DownloadLogLine) => stream.send({ type: 'log', log }); + const onRemoved = (payload: { id: number; batchId: number }) => stream.send({ type: 'removed', ...payload }); downloaderEvents.on('batch', onBatch); downloaderEvents.on('item', onItem); downloaderEvents.on('log', onLog); downloaderEvents.on('removed', onRemoved); - const heartbeat = setInterval(() => res.write(': ping\n\n'), 25_000); - - req.on('close', () => { - clearInterval(heartbeat); + stream.onClose(() => { downloaderEvents.off('batch', onBatch); downloaderEvents.off('item', onItem); downloaderEvents.off('log', onLog); diff --git a/apps/server/src/routes/jobs.ts b/apps/server/src/routes/jobs.ts index ba768b0..d81248b 100644 --- a/apps/server/src/routes/jobs.ts +++ b/apps/server/src/routes/jobs.ts @@ -1,6 +1,6 @@ import { Router } from 'express'; import { z } from 'zod'; -import { wrap } from '../lib/http'; +import { wrap, openEventStream } from '../lib/http'; import { listJobs, jobEvents } from '../services/jobQueue'; import { runAllNow } from '../services/jobScheduler'; import type { Job } from '@sakuya/shared'; @@ -15,23 +15,12 @@ jobsRouter.get( ); jobsRouter.get('/api/jobs/stream', (req, res) => { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }); - res.write(`data: ${JSON.stringify({ type: 'snapshot', jobs: listJobs() })}\n\n`); + const stream = openEventStream(req, res); + stream.send({ type: 'snapshot', jobs: listJobs() }); - const onJob = (job: Job) => { - res.write(`data: ${JSON.stringify({ type: 'job', job })}\n\n`); - }; + const onJob = (job: Job) => stream.send({ type: 'job', job }); jobEvents.on('job', onJob); - const heartbeat = setInterval(() => res.write(': ping\n\n'), 25_000); - - req.on('close', () => { - clearInterval(heartbeat); - jobEvents.off('job', onJob); - }); + stream.onClose(() => jobEvents.off('job', onJob)); }); jobsRouter.post( diff --git a/apps/server/src/routes/libraries.ts b/apps/server/src/routes/libraries.ts index 6a14680..5afcaa2 100644 --- a/apps/server/src/routes/libraries.ts +++ b/apps/server/src/routes/libraries.ts @@ -6,7 +6,7 @@ import crypto from 'node:crypto'; import { z } from 'zod'; import { eq, and, asc } from 'drizzle-orm'; import { db, sqlite, schema } from '../db'; -import { wrap, intParam } from '../lib/http'; +import { wrap, intParam, readFormData } from '../lib/http'; import { enqueueScanJob } from '../services/scanner'; import { scheduleAll } from '../services/jobScheduler'; import { removeMediaRows } from '../services/mediaRemoval'; @@ -144,18 +144,22 @@ librariesRouter.delete( const folderBodySchema = z.object({ path: z.string().min(1) }); /** Attach a folder to a library, reusing an already-attached row if one exists at that exact path. */ -export function attachFolder(libraryId: number, folderPath: string): typeof schema.folders.$inferSelect { +export function attachFolder( + libraryId: number, + folderPath: string, +): { folder: typeof schema.folders.$inferSelect; created: boolean } { const existing = db .select() .from(schema.folders) .where(and(eq(schema.folders.libraryId, libraryId), eq(schema.folders.path, folderPath))) .get(); - if (existing) return existing; - return db + if (existing) return { folder: existing, created: false }; + const folder = db .insert(schema.folders) .values({ libraryId, path: folderPath, status: 'pending', createdAt: Date.now() }) .returning() .get(); + return { folder, created: true }; } librariesRouter.post( @@ -168,14 +172,9 @@ librariesRouter.post( if (!fs.existsSync(folderPath) || !fs.statSync(folderPath).isDirectory()) { return res.status(400).json({ error: `Not a directory: ${folderPath}` }); } - const dup = db - .select() - .from(schema.folders) - .where(and(eq(schema.folders.libraryId, libraryId), eq(schema.folders.path, folderPath))) - .get(); - if (dup) return res.status(409).json({ error: 'Folder already attached' }); - const row = attachFolder(libraryId, folderPath); - res.status(201).json(row); + const { folder, created } = attachFolder(libraryId, folderPath); + if (!created) return res.status(409).json({ error: 'Folder already attached' }); + res.status(201).json(folder); }), ); @@ -225,20 +224,12 @@ librariesRouter.post( const lib = db.select().from(schema.libraries).where(eq(schema.libraries.id, id)).get(); if (!lib) return res.status(404).json({ error: 'Not found' }); - const chunks: Buffer[] = []; - for await (const chunk of req) chunks.push(chunk as Buffer); - const request = new Request('http://localhost/api/libraries/cover', { - method: 'POST', - headers: { 'content-type': req.headers['content-type'] ?? '' }, - body: Buffer.concat(chunks), - }); - const form = await request.formData(); - const file = form.get('file'); + const file = (await readFormData(req)).get('file'); if (!file || typeof file === 'string') return res.status(400).json({ error: 'No file provided' }); - const ext = (path.extname((file as any).name || '') || '.jpg').toLowerCase(); + const ext = (path.extname(file.name || '') || '.jpg').toLowerCase(); const dest = path.join(UPLOADS_DIR, `cover-${id}-${crypto.randomBytes(4).toString('hex')}${ext}`); - await fsp.writeFile(dest, Buffer.from(await (file as any).arrayBuffer())); + await fsp.writeFile(dest, Buffer.from(await file.arrayBuffer())); // Remove any previous custom cover file. if (lib.customImagePath) await fsp.unlink(lib.customImagePath).catch(() => {}); diff --git a/apps/server/src/routes/media.ts b/apps/server/src/routes/media.ts index 97727cd..46c7c6d 100644 --- a/apps/server/src/routes/media.ts +++ b/apps/server/src/routes/media.ts @@ -4,13 +4,14 @@ import path from 'node:path'; import { spawn } from 'node:child_process'; import { z } from 'zod'; import { eq, and, sql, inArray } from 'drizzle-orm'; +import type { SQLQueryBindings } from 'bun:sqlite'; import { db, sqlite, schema } from '../db'; import { wrap, intParam } from '../lib/http'; import { thumbPathFor, generateThumbnail, enqueueThumbnailRegenerate } from '../services/thumbnailer'; import { enqueueTagJob, modelReady, upsertTag, refreshUsageCounts } from '../services/tagger'; import { playablePathFor } from '../services/transcoder'; import { removeMediaRows } from '../services/mediaRemoval'; -import { rowToMedia } from '../lib/rowToMedia'; +import { rowToMedia, MEDIA_SELECT, type MediaSqlRow } from '../lib/rowToMedia'; import { mediaRowsByIds, chunkIds } from '../lib/mediaByIds'; import { bumpTasteVersion } from '../lib/tasteVersion'; import { thumbnailCacheEnabled } from '../lib/settings'; @@ -70,14 +71,14 @@ function ftsMatchExpr(terms: string[]): string { * WHERE fragments + bound params for a filter query. Shared by the paginated list and by * `/api/media/ids`, so "select all matching" can never drift from what the grid shows. */ -function buildMediaFilter(query: MediaFilterQuery): { conds: string[]; params: unknown[] } { +function buildMediaFilter(query: MediaFilterQuery): { conds: string[]; params: SQLQueryBindings[] } { const tagNames = (query.tags ?? '') .split(',') .map((t) => t.trim().toLowerCase()) .filter(Boolean); const conds: string[] = []; - const params: unknown[] = []; + const params: SQLQueryBindings[] = []; if (query.libraryId !== undefined) { conds.push('m.library_id = ?'); params.push(query.libraryId); @@ -145,7 +146,7 @@ mediaRouter.get( : ( sqlite .query(`SELECT COUNT(*) AS c FROM media m ${conds.length ? 'WHERE ' + conds.join(' AND ') : ''}`) - .get(...(params as any[])) as { c: number } + .get(...params) as { c: number } ).c; const pageConds = [...conds]; @@ -168,7 +169,7 @@ mediaRouter.get( ${pageConds.length ? 'WHERE ' + pageConds.join(' AND ') : ''} ORDER BY ${keyExpr} ${dir === 'asc' ? 'ASC' : 'DESC'}, m.id ${dir === 'asc' ? 'ASC' : 'DESC'} LIMIT ?`; - const rows = sqlite.query(sql).all(...(pageParams as any[]), query.limit) as any[]; + const rows = sqlite.query(sql).all(...pageParams, query.limit) as MediaSqlRow[]; let nextCursor: string | null = null; if (rows.length === query.limit) { @@ -194,7 +195,7 @@ mediaRouter.get( ${conds.length ? 'WHERE ' + conds.join(' AND ') : ''} ORDER BY ${keyExpr} ${dir === 'asc' ? 'ASC' : 'DESC'}, m.id ${dir === 'asc' ? 'ASC' : 'DESC'}`, ) - .all(...(params as any[])) as { id: number }[]; + .all(...params) as { id: number }[]; const body: MediaIdsResponse = { ids: rows.map((r) => r.id), total: rows.length }; res.json(body); }), @@ -208,9 +209,7 @@ mediaRouter.get( // round trips. The IN subquery keeps the group set in SQL, so nothing has to be bound. const rows = sqlite .query( - `SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id + `${MEDIA_SELECT} WHERE m.content_hash IN ( SELECT content_hash FROM media WHERE content_hash IS NOT NULL @@ -218,13 +217,13 @@ mediaRouter.get( ) ORDER BY m.content_hash, m.created_at ASC`, ) - .all() as any[]; + .all() as MediaSqlRow[]; // Rows arrive grouped and oldest-first within each group, so the first item of a group is // the original and the rest are what a cleanup would reclaim. const byHash = new Map[]>(); for (const row of rows) { - const hash = row.content_hash as string; + const hash = row.content_hash!; const list = byHash.get(hash); if (list) list.push(rowToMedia(row)); else byHash.set(hash, [rowToMedia(row)]); @@ -276,14 +275,9 @@ mediaRouter.post( const { ids } = idsBody.parse(req.body); const placeholders = ids.map(() => '?').join(','); const rows = sqlite - .query( - `SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id - WHERE m.id IN (${placeholders})`, - ) - .all(...(ids as any[])) as any[]; - const byId = new Map(rows.map((row) => [row.id as number, rowToMedia(row)])); + .query(`${MEDIA_SELECT} WHERE m.id IN (${placeholders})`) + .all(...ids) as MediaSqlRow[]; + const byId = new Map(rows.map((row) => [row.id, rowToMedia(row)])); res.json(ids.map((id) => byId.get(id)).filter(Boolean)); }), ); @@ -301,7 +295,7 @@ mediaRouter.post( WHERE mt.media_id IN (${placeholders}) GROUP BY t.id ORDER BY count DESC, t.name`, ) - .all(...(ids as any[])); + .all(...ids); res.json(rows); }), ); @@ -488,13 +482,7 @@ mediaRouter.post( ); function getDetail(id: number): MediaDetail | null { - const row = sqlite - .query( - `SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id WHERE m.id = ?`, - ) - .get(id) as any; + const row = sqlite.query(`${MEDIA_SELECT} WHERE m.id = ?`).get(id) as MediaSqlRow | null; if (!row) return null; const tagRows = sqlite .query( @@ -503,14 +491,14 @@ function getDetail(id: number): MediaDetail | null { WHERE mt.media_id = ? ORDER BY CASE t.category WHEN 'rating' THEN 0 WHEN 'character' THEN 1 ELSE 2 END, mt.confidence DESC, t.name`, ) - .all(id) as any[]; + .all(id) as MediaDetail['tags']; const boardRows = sqlite .query( `SELECT b.id, b.name, b.created_at AS createdAt FROM board_media bm JOIN boards b ON b.id = bm.board_id WHERE bm.media_id = ? ORDER BY b.name`, ) - .all(id) as any[]; + .all(id) as MediaDetail['boards']; return { ...rowToMedia(row), tags: tagRows, boards: boardRows }; } @@ -697,13 +685,8 @@ mediaRouter.get( // Exact duplicates: same cheap content hash (size + first 4 MB). const duplicates = row.contentHash ? (sqlite - .query( - `SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id - WHERE m.content_hash = ? AND m.id != ? LIMIT 24`, - ) - .all(row.contentHash, id) as any[]).map(rowToMedia) + .query(`${MEDIA_SELECT} WHERE m.content_hash = ? AND m.id != ? LIMIT 24`) + .all(row.contentHash, id) as MediaSqlRow[]).map(rowToMedia) : []; // Visual similarity: perceptual-hash hamming distance (images only). @@ -719,17 +702,15 @@ mediaRouter.get( const bandUnion = bands.map((_, i) => `SELECT id FROM media WHERE phash_b${i} = ?`).join(' UNION '); const candidates = sqlite .query( - `SELECT m.*, l.name AS library_name, - (SELECT COUNT(*) FROM media_tags mt WHERE mt.media_id = m.id) AS tag_count - FROM media m LEFT JOIN libraries l ON l.id = m.library_id + `${MEDIA_SELECT} WHERE m.id IN (${bandUnion}) AND m.type = 'image' AND m.perceptual_hash IS NOT NULL AND m.id != ?`, ) - .all(...(bands as number[]), id) as any[]; - const scored: { media: any; dist: number }[] = []; + .all(...bands, id) as MediaSqlRow[]; + const scored: { media: MediaSqlRow; dist: number }[] = []; for (const cand of candidates) { if (dupIds.has(cand.id)) continue; - const dist = hammingDistance(row.perceptualHash, cand.perceptual_hash); + const dist = hammingDistance(row.perceptualHash, cand.perceptual_hash!); if (dist <= 10) scored.push({ media: cand, dist }); } scored.sort((a, b) => a.dist - b.dist); diff --git a/apps/server/src/routes/settings.ts b/apps/server/src/routes/settings.ts index ce8ce99..50af6c1 100644 --- a/apps/server/src/routes/settings.ts +++ b/apps/server/src/routes/settings.ts @@ -43,8 +43,11 @@ settingsRouter.patch( '/api/settings', wrap(async (req, res) => { const body = z.record(z.string()).parse(req.body); + // Checked up front so a rejected request changes nothing, rather than applying every key + // ahead of the bad one and then reporting failure. + const invalid = Object.keys(body).find((key) => !EDITABLE_KEYS.has(key)); + if (invalid) return res.status(400).json({ error: `Setting not editable: ${invalid}` }); for (const [key, value] of Object.entries(body)) { - if (!EDITABLE_KEYS.has(key)) return res.status(400).json({ error: `Setting not editable: ${key}` }); if (key === 'gifs_as_videos' && value !== getSetting(key)) { enqueueGifReclassifyJob(value === '1'); } @@ -54,16 +57,21 @@ settingsRouter.patch( }), ); -function dirSize(dir: string): number { - let total = 0; - try { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { +/** + * Async on purpose: the thumbnail folder holds one file per media item, and statting tens of + * thousands of them synchronously stalled every other request while the settings page loaded. + */ +async function dirSize(dir: string): Promise { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }).catch(() => []); + const sizes = await Promise.all( + entries.map((entry) => { const full = path.join(dir, entry.name); - if (entry.isFile()) total += fs.statSync(full).size; - else if (entry.isDirectory()) total += dirSize(full); - } - } catch {} - return total; + if (entry.isFile()) return fs.promises.stat(full).then((s) => s.size, () => 0); + if (entry.isDirectory()) return dirSize(full); + return 0; + }), + ); + return sizes.reduce((a, b) => a + b, 0); } settingsRouter.get( @@ -78,7 +86,7 @@ settingsRouter.get( mediaCount: media.c, mediaBytes: media.b, dbBytes: fs.existsSync(DB_PATH) ? fs.statSync(DB_PATH).size : 0, - thumbBytes: dirSize(THUMBS_DIR), + thumbBytes: await dirSize(THUMBS_DIR), }; res.json(info); }), diff --git a/apps/server/src/routes/tags.ts b/apps/server/src/routes/tags.ts index f13180d..a87231e 100644 --- a/apps/server/src/routes/tags.ts +++ b/apps/server/src/routes/tags.ts @@ -1,4 +1,5 @@ import { Router } from 'express'; +import type { SQLQueryBindings } from 'bun:sqlite'; import { z } from 'zod'; import { sqlite } from '../db'; import { wrap } from '../lib/http'; @@ -6,6 +7,14 @@ import type { TagCount } from '@sakuya/shared'; export const tagsRouter = Router(); +/** + * LIKE pattern matching `q` as a literal substring. Unescaped, `_` is a single-character wildcard, + * and it appears in most tag names: searching "long_h" also matched "longXh…". + */ +function likeContains(q: string): string { + return `%${q.replace(/[\\%_]/g, (c) => '\\' + c)}%`; +} + const querySchema = z.object({ q: z.string().optional(), libraryId: z.coerce.number().int().optional(), @@ -30,30 +39,30 @@ tagsRouter.get( FROM media_tags mt JOIN tags t ON t.id = mt.tag_id JOIN media m ON m.id = mt.media_id - WHERE m.library_id = ? ${q ? 'AND t.name LIKE ?' : ''} ${categories ? `AND t.category IN (${categoryPlaceholders})` : ''} + WHERE m.library_id = ? ${q ? "AND t.name LIKE ? ESCAPE '\\'" : ''} ${categories ? `AND t.category IN (${categoryPlaceholders})` : ''} GROUP BY t.id ORDER BY count DESC, t.name ${applyLimit ? 'LIMIT ?' : ''}`, ) .all( ...([ libraryId, - ...(q ? [`%${q}%`] : []), + ...(q ? [likeContains(q)] : []), ...(categories ?? []), ...(applyLimit ? [limit] : []), - ] as any[]), + ] as SQLQueryBindings[]), ) as TagCount[]; } else { rows = sqlite .query( `SELECT name, category, usage_count AS count FROM tags - WHERE usage_count > 0 ${q ? 'AND name LIKE ?' : ''} ${categories ? `AND category IN (${categoryPlaceholders})` : ''} + WHERE usage_count > 0 ${q ? "AND name LIKE ? ESCAPE '\\'" : ''} ${categories ? `AND category IN (${categoryPlaceholders})` : ''} ORDER BY usage_count DESC, name ${applyLimit ? 'LIMIT ?' : ''}`, ) .all( ...([ - ...(q ? [`%${q}%`] : []), + ...(q ? [likeContains(q)] : []), ...(categories ?? []), ...(applyLimit ? [limit] : []), - ] as any[]), + ] as SQLQueryBindings[]), ) as TagCount[]; } res.json(rows); diff --git a/apps/server/src/routes/uploads.ts b/apps/server/src/routes/uploads.ts index d31dc50..9c09086 100644 --- a/apps/server/src/routes/uploads.ts +++ b/apps/server/src/routes/uploads.ts @@ -3,37 +3,23 @@ import path from 'node:path'; import fs from 'node:fs/promises'; import crypto from 'node:crypto'; import { UPLOADS_DIR } from '../lib/config'; -import { wrap } from '../lib/http'; +import { wrap, readFormData, formFiles, sanitizeFilename } from '../lib/http'; import { indexFile, mediaTypeForExt } from '../services/scanner'; import { aiTaggingEnabled } from '../lib/settings'; import { enqueueTagJob, modelReady } from '../services/tagger'; export const uploadsRouter = Router(); -function sanitize(name: string): string { - return path.basename(name).replace(/[^\w.\-()[\] ]+/g, '_').slice(-120); -} - uploadsRouter.post( '/api/uploads', wrap(async (req, res) => { - // Buffer the raw body and let Bun's fetch primitives parse the multipart form. - const chunks: Buffer[] = []; - for await (const chunk of req) chunks.push(chunk as Buffer); - const request = new Request('http://localhost/api/uploads', { - method: 'POST', - headers: { 'content-type': req.headers['content-type'] ?? '' }, - body: Buffer.concat(chunks), - }); - const form = await request.formData(); + const form = await readFormData(req); const libraryId = Number(form.get('libraryId')); if (!Number.isInteger(libraryId) || libraryId <= 0) { return res.status(400).json({ error: 'libraryId is required' }); } - const files = form - .getAll('files') - .filter((f) => typeof f !== 'string') as unknown as Array<{ name: string; arrayBuffer(): Promise }>; + const files = formFiles(form, 'files'); if (!files.length) return res.status(400).json({ error: 'No files provided' }); const mediaIds: number[] = []; @@ -43,7 +29,7 @@ uploadsRouter.post( rejected.push(file.name); continue; } - const unique = `${Date.now().toString(36)}-${crypto.randomBytes(3).toString('hex')}-${sanitize(file.name)}`; + const unique = `${Date.now().toString(36)}-${crypto.randomBytes(3).toString('hex')}-${sanitizeFilename(file.name)}`; const dest = path.join(UPLOADS_DIR, unique); await fs.writeFile(dest, Buffer.from(await file.arrayBuffer())); try { diff --git a/apps/server/src/services/scanner.ts b/apps/server/src/services/scanner.ts index c7e009c..70fca98 100644 --- a/apps/server/src/services/scanner.ts +++ b/apps/server/src/services/scanner.ts @@ -216,7 +216,9 @@ export async function indexFile( } /** - * Drop rows for files under `rootPath` that no longer exist. + * Drop rows for files under `rootPath` that no longer exist. `walked` is every file this scan's + * walk found; those are known to exist, so only the rest cost a filesystem check — a rescan of an + * unchanged library used to stat every indexed file a second time here. * * `rootHasFiles` is whether this scan's walk found any media under the root. When it found none * and every indexed file there is missing too, the likelier story is an unmounted drive, a @@ -229,6 +231,7 @@ async function pruneMissing( libraryId: number, rootPath: string, rootHasFiles: boolean, + walked: Set, ): Promise<{ removed: number; withheld: number }> { const rows = db .select({ id: schema.media.id, path: schema.media.path }) @@ -238,6 +241,7 @@ async function pruneMissing( .filter((row) => isUnder(row.path, rootPath)); const gone: number[] = []; for (const row of rows) { + if (walked.has(row.path)) continue; try { await fs.access(row.path); } catch { @@ -345,8 +349,9 @@ export function enqueueScanJob(libraryId: number) { const unreachable: string[] = []; let withheld = 0; + const walked = new Set(files); for (const folder of libFolders) { - const result = await pruneMissing(libraryId, folder.path, rootHasFiles.get(folder.id) ?? false); + const result = await pruneMissing(libraryId, folder.path, rootHasFiles.get(folder.id) ?? false, walked); pruned += result.removed; withheld += result.withheld; if (result.withheld) unreachable.push(folder.path); diff --git a/apps/web/src/routes/downloader/DownloaderPage.tsx b/apps/web/src/routes/downloader/DownloaderPage.tsx index 8a26ea6..995d566 100644 --- a/apps/web/src/routes/downloader/DownloaderPage.tsx +++ b/apps/web/src/routes/downloader/DownloaderPage.tsx @@ -334,11 +334,11 @@ function ItemRow({ item }: { item: DownloadItem }) { staleTime: Infinity, }); - const liveLogs = logs[item.id] ?? []; + const liveLogs = logs[item.id]; const merged = useMemo(() => { const map = new Map(); for (const l of historyLogs ?? []) map.set(l.id, l.line); - for (const l of liveLogs) map.set(l.id, l.line); + for (const l of liveLogs ?? []) map.set(l.id, l.line); return [...map.entries()].sort((a, b) => a[0] - b[0]).map(([, line]) => line); }, [historyLogs, liveLogs]); diff --git a/docs/API.md b/docs/API.md index 1a4ac09..e49e38e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -254,7 +254,7 @@ Update playback/view progress. Body: `{ progress: number (0-1) }`. Response: `{ ### `GET /api/tags` Tag usage counts, optionally scoped to a library. -Query params: `q?` (substring filter), `libraryId?` (number), `category?` (comma-separated list), +Query params: `q?` (literal substring filter; `_` and `%` match themselves), `libraryId?` (number), `category?` (comma-separated list), `limit?` (1-100, default 30; ignored when filtering to only the `rating` category). Response: `TagCount[]`. @@ -313,10 +313,10 @@ All settings key/value pairs. Response: `Settings`. ### `PATCH /api/settings` Update one or more settings. Body: `Record` — keys must be one of the editable keys (`ai_tagging_enabled`, `confidence_threshold`, `accent_color`, `remember_mute_state`, -`remember_volume_level`, `autosearch_first_tag`, `continue_where_left`, -`thumbnail_cache_enabled`, `board_remember_filters`, `downloader_concurrency`, -`gifs_as_videos`); unknown keys return `400`. Toggling `gifs_as_videos` enqueues a GIF -reclassification job. Response: `Settings`. +`remember_volume_level`, `continue_where_left`, `thumbnail_cache_enabled`, +`board_remember_filters`, `downloader_concurrency`, `gifs_as_videos`, `video_transcode_enabled`, +`discover_enabled`); an unknown key returns `400` and nothing in the request is saved. Toggling +`gifs_as_videos` enqueues a GIF reclassification job. Response: `Settings`. ## System