From 5749966fff138636fe5115309c879222d6d38712 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:09:01 -0700 Subject: [PATCH 01/11] FEATURE: Enforce per-org rate limits with Workers bindings Four rate limit bindings (RL_UPLOAD, RL_PUBLISH, RL_AUTH, RL_ANON) replace the KV counters in auth-guard.ts. Uploads and site sessions are keyed by org, device auth by client address, and anonymous content fetches past the edge cache by IP. The wrangler drift check compares the bindings and their limits across environments; route tests swap the bindings for fakes with a per-name denial switch. The AUTH_GUARD namespace stays for the slug and query embedding caches. Co-Authored-By: Claude Fable 5.1 --- .../lib/server/auth-rate-limit-response.ts | 11 +- apps/web/src/lib/server/edge.ts | 4 +- apps/web/src/lib/server/layer.ts | 20 +- apps/web/src/lib/server/mcp/server.ts | 16 +- .../web/src/lib/server/services/auth-guard.ts | 205 ------------------ apps/web/src/lib/server/services/bindings.ts | 17 ++ .../src/lib/server/services/rate-limits.ts | 104 +++++++++ .../web/src/routes/api/auth/device/+server.ts | 17 +- .../routes/api/auth/device/token/+server.ts | 17 +- apps/web/src/routes/api/files/+server.ts | 16 +- .../routes/api/files/[id]/versions/+server.ts | 16 +- .../src/routes/api/sites/sessions/+server.ts | 9 +- apps/web/src/routes/f/[id]/+server.ts | 16 +- .../src/routes/s/[id]/[...path]/+server.ts | 15 +- .../t/[id]/[version]/grid.webp/+server.ts | 14 +- apps/web/worker-configuration.d.ts | 10 +- apps/web/wrangler.jsonc | 51 ++++- scripts/check-wrangler-drift.mjs | 11 + 18 files changed, 297 insertions(+), 272 deletions(-) delete mode 100644 apps/web/src/lib/server/services/auth-guard.ts create mode 100644 apps/web/src/lib/server/services/rate-limits.ts diff --git a/apps/web/src/lib/server/auth-rate-limit-response.ts b/apps/web/src/lib/server/auth-rate-limit-response.ts index 57766ec..54523de 100644 --- a/apps/web/src/lib/server/auth-rate-limit-response.ts +++ b/apps/web/src/lib/server/auth-rate-limit-response.ts @@ -1,8 +1,9 @@ -import type { BlockedAuthAttempt } from './services/auth-guard'; +// Every rate limit binding counts over a 60 second window +// (wrangler.jsonc), so a refused request can try again after it. +export const RATE_LIMIT_PERIOD_SECONDS = 60; -export const authRateLimitResponse = ( - decision: BlockedAuthAttempt, - message = 'Too many authentication requests. Try again later.' +export const rateLimitResponse = ( + message = 'Too many requests. Try again later.' ) => Response.json( { message }, @@ -10,7 +11,7 @@ export const authRateLimitResponse = ( status: 429, headers: { 'Cache-Control': 'private, no-store', - 'Retry-After': String(decision.retryAfterSeconds) + 'Retry-After': String(RATE_LIMIT_PERIOD_SECONDS) } } ); diff --git a/apps/web/src/lib/server/edge.ts b/apps/web/src/lib/server/edge.ts index 521428c..bc6a52f 100644 --- a/apps/web/src/lib/server/edge.ts +++ b/apps/web/src/lib/server/edge.ts @@ -19,7 +19,6 @@ import type { ProgramIdentity, ProgramTenant } from './identity'; import { requestLayer } from './layer'; import { PgSql } from './pg'; import type { AppConfig } from './config'; -import type { AuthGuard } from './services/auth-guard'; import type { Auth } from './services/auth'; import type { AuthGuardStore } from './services/bindings'; import type { Blobs } from './services/blobs'; @@ -34,12 +33,13 @@ import type { Lifecycle } from './services/lifecycle'; import type { GrantSecrets } from './services/grant-secrets'; import type { JobQueue } from './services/jobs'; import type { Org } from './services/org'; +import type { RateLimits } from './services/rate-limits'; import type { WorkOSClient } from './services/workos'; export type AppServices = | PgSql | AppConfig - | AuthGuard + | RateLimits | AuthGuardStore | Auth | Blobs diff --git a/apps/web/src/lib/server/layer.ts b/apps/web/src/lib/server/layer.ts index 0b93e4c..98fcb9a 100644 --- a/apps/web/src/lib/server/layer.ts +++ b/apps/web/src/lib/server/layer.ts @@ -3,9 +3,14 @@ import { ConfigLive } from './config'; import { OrgMissing, StorageError } from './errors'; import type { ProgramTenant } from './identity'; import { PgSql } from './pg'; -import { AuthGuardLive } from './services/auth-guard'; import { AuthLive } from './services/auth'; -import { AuthGuardStore, Bucket, Jobs, Pg } from './services/bindings'; +import { + AuthGuardStore, + Bucket, + Jobs, + Pg, + RateLimiters +} from './services/bindings'; import { pgLayer } from './pg'; import { BlobsLive } from './services/blobs'; import { @@ -24,6 +29,7 @@ import { LifecycleLive } from './services/lifecycle'; import { GrantSecretsLive } from './services/grant-secrets'; import { JobQueueLive } from './services/jobs'; import { OrgLive } from './services/org'; +import { RateLimitsLive } from './services/rate-limits'; import { WorkOSLive } from './services/workos'; export const PgLive = Layer.unwrap( @@ -74,6 +80,12 @@ export const requestLayer = (env: Env, tenant: ProgramTenant | null) => { Layer.succeed(Bucket, env.BUCKET), Layer.succeed(AuthGuardStore, env.AUTH_GUARD), Layer.succeed(Jobs, env.JOBS), + Layer.succeed(RateLimiters, { + upload: env.RL_UPLOAD, + publish: env.RL_PUBLISH, + auth: env.RL_AUTH, + anonymous: env.RL_ANON + }), Layer.succeed( CurrentUser, tenant?.userId ? { id: tenant.userId } : anonymousUser @@ -94,7 +106,7 @@ export const requestLayer = (env: Env, tenant: ProgramTenant | null) => { const auth = AuthLive.pipe( Layer.provide(Layer.merge(infrastructure, workos)) ); - const authGuard = AuthGuardLive().pipe(Layer.provide(bindings)); + const rateLimits = RateLimitsLive.pipe(Layer.provide(bindings)); const orgService = OrgLive.pipe(Layer.provide(infrastructure)); const grantSecrets = GrantSecretsLive.pipe(Layer.provide(infrastructure)); const tags = TagsLive.pipe(Layer.provide(infrastructure)); @@ -117,7 +129,7 @@ export const requestLayer = (env: Env, tenant: ProgramTenant | null) => { semantic, workos, auth, - authGuard, + rateLimits, orgService, grantSecrets, tags, diff --git a/apps/web/src/lib/server/mcp/server.ts b/apps/web/src/lib/server/mcp/server.ts index 053dcf0..5731ed6 100644 --- a/apps/web/src/lib/server/mcp/server.ts +++ b/apps/web/src/lib/server/mcp/server.ts @@ -8,11 +8,11 @@ import { InvalidRequest, validate } from '../errors'; import { maxPreviewBytes, previewKind } from '../file-preview'; import { resolveFileContentLink } from '../file-content-link'; import type { AuthContext } from '../identity'; -import { AuthGuard } from '../services/auth-guard'; import { Blobs } from '../services/blobs'; import { currentContentOrigin } from '../services/current-org'; import { Files } from '../services/files'; import { Indexing } from '../services/indexing'; +import { RateLimits } from '../services/rate-limits'; import { Search } from '../services/search'; import { Sites } from '../services/sites'; import { Tags } from '../services/tags'; @@ -328,12 +328,9 @@ const registerWriteTools = (server: McpServer, input: McpServerInput) => { env, credential, Effect.gen(function* () { - const authGuard = yield* AuthGuard; + const rateLimits = yield* RateLimits; const files = yield* Files; - const rate = yield* authGuard.consume( - 'upload', - credential.credentialId - ); + const rate = yield* rateLimits.upload(credential.orgId); if (!rate.allowed) { return { kind: 'rate-limit' as const, @@ -524,12 +521,9 @@ const registerWriteTools = (server: McpServer, input: McpServerInput) => { env, credential, Effect.gen(function* () { - const authGuard = yield* AuthGuard; + const rateLimits = yield* RateLimits; const sites = yield* Sites; - const rate = yield* authGuard.consume( - 'upload', - credential.credentialId - ); + const rate = yield* rateLimits.upload(credential.orgId); if (!rate.allowed) { return { kind: 'rate-limit' as const, diff --git a/apps/web/src/lib/server/services/auth-guard.ts b/apps/web/src/lib/server/services/auth-guard.ts deleted file mode 100644 index 2305654..0000000 --- a/apps/web/src/lib/server/services/auth-guard.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { Context, Effect, Layer, Schema } from 'effect'; -import { StorageError } from '../errors'; -import { AuthGuardStore } from './bindings'; - -const CounterState = Schema.Struct({ - count: Schema.Int, - resetAtMs: Schema.Int -}); - -const ratePolicies = { - deviceCreate: { - key: 'device-create', - limit: 5, - windowSeconds: 10 * 60 - }, - devicePoll: { - key: 'device-poll', - limit: 150, - windowSeconds: 10 * 60 - }, - // Keyed by credential id, not client address: bounds how fast a single - // leaked API key can pump objects into the bucket. KV allows roughly one - // write per second per key and is eventually consistent, so this is an - // approximate abuse brake, not an exact ceiling — legitimate concurrent - // uploads must not be failed by a KV write conflict (tolerateWriteFailure), - // and a burst may briefly exceed the nominal limit. - upload: { - key: 'upload', - limit: 120, - windowSeconds: 10 * 60, - tolerateWriteFailure: true - } -}; - -export type AuthRatePolicy = keyof typeof ratePolicies; - -export interface AllowedAuthAttempt { - readonly allowed: true; - readonly remaining: number; - readonly resetAtMs: number; -} - -export interface BlockedAuthAttempt { - readonly allowed: false; - readonly reason: 'rate-limit'; - readonly retryAfterSeconds: number; - readonly resetAtMs: number; -} - -export type AuthAttemptDecision = AllowedAuthAttempt | BlockedAuthAttempt; - -export interface AuthGuardShape { - readonly consume: ( - policy: AuthRatePolicy, - clientId: string - ) => Effect.Effect; -} - -export class AuthGuard extends Context.Service()( - 'app/AuthGuard' -) {} - -const MINIMUM_KV_TTL_SECONDS = 60; - -const retryAfter = (resetAtMs: number, nowMs: number) => - Math.max(1, Math.ceil((resetAtMs - nowMs) / 1_000)); - -const allowed = (remaining: number, resetAtMs: number) => - ({ - allowed: true, - remaining, - resetAtMs - }) satisfies AllowedAuthAttempt; - -const blocked = (resetAtMs: number, nowMs: number) => - ({ - allowed: false, - reason: 'rate-limit', - retryAfterSeconds: retryAfter(resetAtMs, nowMs), - resetAtMs - }) satisfies BlockedAuthAttempt; - -const normalizeClientId = (value: string) => - value.normalize('NFKC').trim().slice(0, 256) || 'unknown-client'; - -const toHex = (bytes: ArrayBuffer) => - Array.from(new Uint8Array(bytes), (byte) => - byte.toString(16).padStart(2, '0') - ).join(''); - -const hashClientId = (value: string) => - Effect.promise(() => - crypto.subtle.digest( - 'SHA-256', - new TextEncoder().encode(normalizeClientId(value)) - ) - ).pipe(Effect.map(toHex)); - -const decodeStored = ( - schema: Schema.Codec, - value: string | null -) => { - if (value === null) return null; - const parsed: unknown = JSON.parse(value); - const decoded = Schema.decodeUnknownOption(schema)(parsed); - if (decoded._tag === 'None') - throw new Error('Stored auth guard state is invalid'); - return decoded.value; -}; - -const makeAuthGuard = (now: () => Date) => - Effect.gen(function* () { - const store = yield* AuthGuardStore; - - const read = ( - key: string, - schema: Schema.Codec, - operation: string - ) => - Effect.tryPromise({ - try: async () => decodeStored(schema, await store.get(key)), - catch: (cause) => new StorageError({ operation, cause }) - }); - - const write = ( - key: string, - value: unknown, - expirationTtl: number, - operation: string - ) => - Effect.tryPromise({ - try: () => - store.put(key, JSON.stringify(value), { - expirationTtl: Math.max( - MINIMUM_KV_TTL_SECONDS, - Math.ceil(expirationTtl) - ) - }), - catch: (cause) => new StorageError({ operation, cause }) - }); - - const clientKey = Effect.fn('AuthGuard.clientKey')(function* ( - prefix: string, - clientId: string - ) { - const clientHash = yield* hashClientId(clientId); - return `v1:${prefix}:${clientHash}`; - }); - - return AuthGuard.of({ - consume: Effect.fn('AuthGuard.consume')(function* (policyName, clientId) { - const policy = ratePolicies[policyName]; - const key = yield* clientKey(policy.key, clientId); - const currentTime = now().getTime(); - const current = yield* read( - key, - CounterState, - `read ${policy.key} rate limit` - ); - const resetAtMs = - current === null || current.resetAtMs <= currentTime - ? currentTime + policy.windowSeconds * 1_000 - : current.resetAtMs; - const count = - current === null || current.resetAtMs <= currentTime - ? 0 - : current.count; - - if (count >= policy.limit) { - return blocked(resetAtMs, currentTime); - } - - const nextCount = count + 1; - const persist = write( - key, - { count: nextCount, resetAtMs }, - retryAfter(resetAtMs, currentTime) + MINIMUM_KV_TTL_SECONDS, - `update ${policy.key} rate limit` - ); - // High-frequency policies (uploads) hit KV's ~1 write/sec/key - // ceiling under normal concurrent use; failing the counter - // write must not fail the legitimate request there. Abuse - // bootstrap policies (device flow) stay fail-closed: a - // contended write turns into a short rate limit. - if ('tolerateWriteFailure' in policy && policy.tolerateWriteFailure) { - yield* persist.pipe(Effect.ignore); - return allowed(policy.limit - nextCount, resetAtMs); - } - const outcome = yield* persist.pipe( - Effect.match({ - onFailure: () => - blocked( - currentTime + MINIMUM_KV_TTL_SECONDS * 1_000, - currentTime - ), - onSuccess: () => allowed(policy.limit - nextCount, resetAtMs) - }) - ); - return outcome; - }) - }); - }); - -export const AuthGuardLive = (now = () => new Date()) => - Layer.effect(AuthGuard, makeAuthGuard(now)); diff --git a/apps/web/src/lib/server/services/bindings.ts b/apps/web/src/lib/server/services/bindings.ts index 9d146e7..680ae99 100644 --- a/apps/web/src/lib/server/services/bindings.ts +++ b/apps/web/src/lib/server/services/bindings.ts @@ -7,6 +7,23 @@ export class Pg extends Context.Service()('app/Pg') {} export class Bucket extends Context.Service()('app/Bucket') {} +// The four Workers rate limit bindings (wrangler.jsonc `ratelimits`), +// consumed through services/rate-limits.ts. +export interface RateLimiterBindings { + readonly upload: RateLimit; + readonly publish: RateLimit; + readonly auth: RateLimit; + readonly anonymous: RateLimit; +} + +export class RateLimiters extends Context.Service< + RateLimiters, + RateLimiterBindings +>()('app/RateLimiters') {} + +// The AUTH_GUARD KV namespace. Its name predates the rate limit bindings; +// today it only caches org slug lookups (content-host.ts) and query +// embeddings (query-embedding-cache.ts). export interface AuthGuardStoreShape { readonly get: (key: string) => Promise; readonly put: ( diff --git a/apps/web/src/lib/server/services/rate-limits.ts b/apps/web/src/lib/server/services/rate-limits.ts new file mode 100644 index 0000000..df93cfb --- /dev/null +++ b/apps/web/src/lib/server/services/rate-limits.ts @@ -0,0 +1,104 @@ +import { Context, Effect, Layer } from 'effect'; +import { RateLimiters } from './bindings'; + +// Abuse brakes on the Workers rate limit bindings (wrangler.jsonc +// `ratelimits`). Each is keyed by the principal that would be abusing it: +// uploads and publishes by org, credential bootstrap by client, anonymous +// content fetches by IP. The bindings count per colo and are approximate, +// which is enough to stop a runaway client and not a billing input. A +// binding that errors lets the request through and logs: a broken brake +// must not take legitimate traffic down with it. + +export interface RateLimitDecision { + readonly allowed: boolean; +} + +export interface RateLimitsShape { + readonly upload: (orgId: string) => Effect.Effect; + readonly publish: (orgId: string) => Effect.Effect; + readonly auth: (clientId: string) => Effect.Effect; + readonly anonymous: (ip: string) => Effect.Effect; +} + +export type RateLimitName = keyof RateLimitsShape; + +export class RateLimits extends Context.Service()( + 'app/RateLimits' +) {} + +// The bindings reject keys over 64 bytes; hash longer principals so an +// oversized value cannot bypass the counter by erroring out. +const MAX_KEY_LENGTH = 64; + +const normalizeKey = (value: string) => { + const trimmed = value.normalize('NFKC').trim() || 'unknown'; + return trimmed.length <= MAX_KEY_LENGTH ? trimmed : null; +}; + +const hashedKey = (value: string) => + Effect.promise(() => + crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)) + ).pipe( + Effect.map((digest) => + Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, '0') + ).join('') + ) + ); + +const limitWith = (name: RateLimitName, binding: RateLimit) => + Effect.fn(`RateLimits.${name}`)(function* (principal: string) { + const key = normalizeKey(principal) ?? (yield* hashedKey(principal)); + return yield* Effect.tryPromise(() => binding.limit({ key })).pipe( + Effect.map((outcome) => ({ allowed: outcome.success })), + Effect.catchCause((cause) => + Effect.sync(() => { + console.error( + JSON.stringify({ + message: 'rate limit binding failed; allowing the request', + limit: name, + cause: String(cause) + }) + ); + return { allowed: true }; + }) + ) + ); + }); + +export const RateLimitsLive = Layer.effect( + RateLimits, + Effect.map(RateLimiters, (bindings) => + RateLimits.of({ + upload: limitWith('upload', bindings.upload), + publish: limitWith('publish', bindings.publish), + auth: limitWith('auth', bindings.auth), + anonymous: limitWith('anonymous', bindings.anonymous) + }) + ) +); + +// For tests and tools that run outside the Worker. Every check is allowed +// unless its name is in `denied`, which a test flips to exercise the 429 +// path deterministically instead of racing a real counter. +export const RateLimitsNull = ( + denied: ReadonlySet = new Set() +) => { + const decide = (name: RateLimitName) => () => + Effect.succeed({ allowed: !denied.has(name) }); + return Layer.succeed( + RateLimits, + RateLimits.of({ + upload: decide('upload'), + publish: decide('publish'), + auth: decide('auth'), + anonymous: decide('anonymous') + }) + ); +}; + +// A stand-in for one RateLimit binding, for route tests that hand the +// consumer and the routes a real Env: allowed unless `denied` says no. +export const rateLimitBinding = (denied: () => boolean): RateLimit => ({ + limit: async () => ({ success: !denied() }) +}); diff --git a/apps/web/src/routes/api/auth/device/+server.ts b/apps/web/src/routes/api/auth/device/+server.ts index 3fa6556..f85d64b 100644 --- a/apps/web/src/routes/api/auth/device/+server.ts +++ b/apps/web/src/routes/api/auth/device/+server.ts @@ -1,24 +1,25 @@ import { DeviceAuthorizationCreateSchema } from '@adrive/shared'; import type { RequestHandler } from './$types'; import { Effect } from 'effect'; -import { authRateLimitResponse } from '$lib/server/auth-rate-limit-response'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { AppConfig } from '$lib/server/config'; import { runEdge } from '$lib/server/edge'; import { decodeJson } from '$lib/server/request-json'; import { Auth } from '$lib/server/services/auth'; -import { AuthGuard } from '$lib/server/services/auth-guard'; +import { RateLimits } from '$lib/server/services/rate-limits'; export const POST: RequestHandler = ({ request, getClientAddress }) => runEdge( Effect.gen(function* () { const auth = yield* Auth; - const authGuard = yield* AuthGuard; + const rateLimits = yield* RateLimits; const config = yield* AppConfig; - const rateLimit = yield* authGuard.consume( - 'deviceCreate', - getClientAddress() - ); - if (!rateLimit.allowed) return authRateLimitResponse(rateLimit); + const rateLimit = yield* rateLimits.auth(getClientAddress()); + if (!rateLimit.allowed) { + return rateLimitResponse( + 'Too many authentication requests. Try again later.' + ); + } const input = yield* decodeJson( request, DeviceAuthorizationCreateSchema, diff --git a/apps/web/src/routes/api/auth/device/token/+server.ts b/apps/web/src/routes/api/auth/device/token/+server.ts index cbc7d77..50c5d14 100644 --- a/apps/web/src/routes/api/auth/device/token/+server.ts +++ b/apps/web/src/routes/api/auth/device/token/+server.ts @@ -1,22 +1,23 @@ import { DeviceTokenRequestSchema } from '@adrive/shared'; import type { RequestHandler } from './$types'; import { Effect } from 'effect'; -import { authRateLimitResponse } from '$lib/server/auth-rate-limit-response'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { decodeJson } from '$lib/server/request-json'; import { Auth } from '$lib/server/services/auth'; -import { AuthGuard } from '$lib/server/services/auth-guard'; +import { RateLimits } from '$lib/server/services/rate-limits'; export const POST: RequestHandler = ({ request, getClientAddress }) => runEdge( Effect.gen(function* () { const auth = yield* Auth; - const authGuard = yield* AuthGuard; - const rateLimit = yield* authGuard.consume( - 'devicePoll', - getClientAddress() - ); - if (!rateLimit.allowed) return authRateLimitResponse(rateLimit); + const rateLimits = yield* RateLimits; + const rateLimit = yield* rateLimits.auth(getClientAddress()); + if (!rateLimit.allowed) { + return rateLimitResponse( + 'Too many authentication requests. Try again later.' + ); + } const input = yield* decodeJson( request, DeviceTokenRequestSchema, diff --git a/apps/web/src/routes/api/files/+server.ts b/apps/web/src/routes/api/files/+server.ts index edf5e5e..6aca304 100644 --- a/apps/web/src/routes/api/files/+server.ts +++ b/apps/web/src/routes/api/files/+server.ts @@ -4,10 +4,10 @@ import { runEdge } from '$lib/server/edge'; import { requireAuth, requireWrite } from '$lib/server/request-auth'; import { AppConfig } from '$lib/server/config'; import { validateExpiration } from '$lib/server/auth-policy'; -import { authRateLimitResponse } from '$lib/server/auth-rate-limit-response'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { InvalidRequest } from '$lib/server/errors'; import { parsePageSize } from '$lib/server/list-cursor'; -import { AuthGuard } from '$lib/server/services/auth-guard'; +import { RateLimits } from '$lib/server/services/rate-limits'; import { currentContentOrigin } from '$lib/server/services/current-org'; import { Files } from '$lib/server/services/files'; import { Indexing } from '$lib/server/services/indexing'; @@ -120,18 +120,12 @@ export const PUT: RequestHandler = (event) => { const { request, url } = event; return runEdge( Effect.gen(function* () { - const authGuard = yield* AuthGuard; + const rateLimits = yield* RateLimits; const files = yield* Files; const credential = yield* requireWrite(event); - const rateLimit = yield* authGuard.consume( - 'upload', - credential.credentialId - ); + const rateLimit = yield* rateLimits.upload(credential.orgId); if (!rateLimit.allowed) { - return authRateLimitResponse( - rateLimit, - 'Too many uploads. Try again later.' - ); + return rateLimitResponse('Too many uploads. Try again later.'); } const displayName = yield* Effect.try({ try: () => decodeName(request.headers.get('x-adrive-file-name')), diff --git a/apps/web/src/routes/api/files/[id]/versions/+server.ts b/apps/web/src/routes/api/files/[id]/versions/+server.ts index 27a3857..fb84b68 100644 --- a/apps/web/src/routes/api/files/[id]/versions/+server.ts +++ b/apps/web/src/routes/api/files/[id]/versions/+server.ts @@ -1,27 +1,21 @@ import type { RequestHandler } from './$types'; import { Effect } from 'effect'; -import { authRateLimitResponse } from '$lib/server/auth-rate-limit-response'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { requireWrite } from '$lib/server/request-auth'; -import { AuthGuard } from '$lib/server/services/auth-guard'; +import { RateLimits } from '$lib/server/services/rate-limits'; import { Files } from '$lib/server/services/files'; export const PUT: RequestHandler = (event) => { const { params, request } = event; return runEdge( Effect.gen(function* () { - const authGuard = yield* AuthGuard; + const rateLimits = yield* RateLimits; const files = yield* Files; const credential = yield* requireWrite(event); - const rateLimit = yield* authGuard.consume( - 'upload', - credential.credentialId - ); + const rateLimit = yield* rateLimits.upload(credential.orgId); if (!rateLimit.allowed) { - return authRateLimitResponse( - rateLimit, - 'Too many uploads. Try again later.' - ); + return rateLimitResponse('Too many uploads. Try again later.'); } const result = yield* files.uploadVersion({ id: params.id, diff --git a/apps/web/src/routes/api/sites/sessions/+server.ts b/apps/web/src/routes/api/sites/sessions/+server.ts index 1927080..0a92208 100644 --- a/apps/web/src/routes/api/sites/sessions/+server.ts +++ b/apps/web/src/routes/api/sites/sessions/+server.ts @@ -4,10 +4,12 @@ import { } from '@adrive/shared'; import type { RequestHandler } from './$types'; import { Effect, Schema } from 'effect'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { requireWrite } from '$lib/server/request-auth'; import { InvalidRequest } from '$lib/server/errors'; import { readBoundedJson } from '$lib/server/request-json'; +import { RateLimits } from '$lib/server/services/rate-limits'; import { Sites } from '$lib/server/services/sites'; const MAX_MANIFEST_BYTES = 1024 * 1024; @@ -37,7 +39,12 @@ export const POST: RequestHandler = (event) => { return runEdge( Effect.gen(function* () { const sites = yield* Sites; - yield* requireWrite(event); + const rateLimits = yield* RateLimits; + const credential = yield* requireWrite(event); + const rateLimit = yield* rateLimits.upload(credential.orgId); + if (!rateLimit.allowed) { + return rateLimitResponse('Too many uploads. Try again later.'); + } const input: SiteSessionCreate = yield* readManifest(request); return Response.json(yield* sites.createSession(input), { status: 201 }); }) diff --git a/apps/web/src/routes/f/[id]/+server.ts b/apps/web/src/routes/f/[id]/+server.ts index 007cb17..7f738f4 100644 --- a/apps/web/src/routes/f/[id]/+server.ts +++ b/apps/web/src/routes/f/[id]/+server.ts @@ -17,11 +17,13 @@ import { import { AppConfig } from '$lib/server/config'; import { shouldRecordFileDownload } from '$lib/server/auth-policy'; import { decodeRangeHeader, rangeHeaders } from '$lib/server/download-response'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { NotFound, StorageError } from '$lib/server/errors'; import { Blobs } from '$lib/server/services/blobs'; import { Files } from '$lib/server/services/files'; import { GrantSecrets } from '$lib/server/services/grant-secrets'; +import { RateLimits } from '$lib/server/services/rate-limits'; const requestedVersion = (url: URL) => { const value = url.searchParams.get('v'); @@ -30,7 +32,13 @@ const requestedVersion = (url: URL) => { return Number.isSafeInteger(version) && version > 0 ? version : null; }; -const serveFile: RequestHandler = ({ params, platform, request, url }) => +const serveFile: RequestHandler = ({ + getClientAddress, + params, + platform, + request, + url +}) => runEdge( Effect.gen(function* () { const config = yield* AppConfig; @@ -135,6 +143,12 @@ const serveFile: RequestHandler = ({ params, platform, request, url }) => } } + // Past the edge cache every request costs a Postgres read and an R2 + // read, so one client is capped here. + const rateLimits = yield* RateLimits; + const rateLimit = yield* rateLimits.anonymous(getClientAddress()); + if (!rateLimit.allowed) return rateLimitResponse(); + if (request.method === 'HEAD') { const object = yield* blobs.head(content.r2Key); if (!privateResponse && matchesEtag(ifNoneMatch, object.httpEtag)) { diff --git a/apps/web/src/routes/s/[id]/[...path]/+server.ts b/apps/web/src/routes/s/[id]/[...path]/+server.ts index 7d489a9..e532835 100644 --- a/apps/web/src/routes/s/[id]/[...path]/+server.ts +++ b/apps/web/src/routes/s/[id]/[...path]/+server.ts @@ -9,6 +9,7 @@ import { siteCacheControl } from '$lib/server/content-cache'; import { AppConfig } from '$lib/server/config'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { NotFound } from '$lib/server/errors'; import { isSiteVersionRequestServable } from '$lib/server/site-policy'; @@ -16,6 +17,7 @@ import { Blobs } from '$lib/server/services/blobs'; import { Files } from '$lib/server/services/files'; import { Sites } from '$lib/server/services/sites'; import { GrantSecrets } from '$lib/server/services/grant-secrets'; +import { RateLimits } from '$lib/server/services/rate-limits'; export const trailingSlash = 'ignore'; @@ -42,7 +44,12 @@ const siteGrant = (path: string) => { }; }; -const serveSite: RequestHandler = ({ params, request, url }) => +const serveSite: RequestHandler = ({ + getClientAddress, + params, + request, + url +}) => runEdge( Effect.gen(function* () { const grant = siteGrant(params.path ?? ''); @@ -62,6 +69,12 @@ const serveSite: RequestHandler = ({ params, request, url }) => const blobs = yield* Blobs; const files = yield* Files; const config = yield* AppConfig; + const rateLimits = yield* RateLimits; + // Site assets are not held in the edge cache by the Worker (the CDN + // honours their Cache-Control), so every request that reaches here + // is a cache miss and counts. + const rateLimit = yield* rateLimits.anonymous(getClientAddress()); + if (!rateLimit.allowed) return rateLimitResponse(); // The asset row names the owning org, which the grant is bound to, // so it is resolved before the signature is checked. const asset = yield* sites.findAsset( diff --git a/apps/web/src/routes/t/[id]/[version]/grid.webp/+server.ts b/apps/web/src/routes/t/[id]/[version]/grid.webp/+server.ts index bcf1039..9aa332b 100644 --- a/apps/web/src/routes/t/[id]/[version]/grid.webp/+server.ts +++ b/apps/web/src/routes/t/[id]/[version]/grid.webp/+server.ts @@ -20,12 +20,14 @@ import { storeEdgeCache, thumbnailCacheControl } from '$lib/server/content-cache'; +import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { NotFound, StorageError } from '$lib/server/errors'; import { Blobs } from '$lib/server/services/blobs'; import { currentContentOrigin } from '$lib/server/services/current-org'; import { Files } from '$lib/server/services/files'; import { GrantSecrets } from '$lib/server/services/grant-secrets'; +import { RateLimits } from '$lib/server/services/rate-limits'; const parsedVersion = (value: string) => { const version = Number(value); @@ -66,7 +68,13 @@ const publicThumbnailRedirect = (url: URL) => { }); }; -export const GET: RequestHandler = ({ params, platform, request, url }) => +export const GET: RequestHandler = ({ + getClientAddress, + params, + platform, + request, + url +}) => runEdge( Effect.gen(function* () { const version = parsedVersion(params.version); @@ -136,6 +144,10 @@ export const GET: RequestHandler = ({ params, platform, request, url }) => } } + const rateLimits = yield* RateLimits; + const rateLimit = yield* rateLimits.anonymous(getClientAddress()); + if (!rateLimit.allowed) return rateLimitResponse(); + const blobs = yield* Blobs; const cached = content.thumbnailR2Key === null diff --git a/apps/web/worker-configuration.d.ts b/apps/web/worker-configuration.d.ts index aa6ff40..ffb627b 100644 --- a/apps/web/worker-configuration.d.ts +++ b/apps/web/worker-configuration.d.ts @@ -1,11 +1,15 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 6c1287863c727955d614dd4a3d807365) +// Generated by Wrangler by running `wrangler types` (hash: d76966b5dbd8efc44b28675760946f8e) // Runtime types generated with workerd@1.20260722.1 2026-07-27 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { AUTH_GUARD: KVNamespace; BUCKET: R2Bucket; HYPERDRIVE: Hyperdrive; JOBS: Queue; + RL_UPLOAD: RateLimit; + RL_PUBLISH: RateLimit; + RL_AUTH: RateLimit; + RL_ANON: RateLimit; BROWSER: BrowserRun; AI?: Ai; ASSETS: Fetcher; @@ -29,6 +33,10 @@ declare namespace Cloudflare { BUCKET: R2Bucket; HYPERDRIVE: Hyperdrive; JOBS: Queue; + RL_UPLOAD: RateLimit; + RL_PUBLISH: RateLimit; + RL_AUTH: RateLimit; + RL_ANON: RateLimit; BROWSER: BrowserRun; AI: Ai; ASSETS: Fetcher; diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index 69ce047..4ef8e98 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -47,6 +47,30 @@ "id": "419a52adcd9649e0a24e5c47c2748477" } ], + // Abuse brakes (services/rate-limits.ts). Per-colo and approximate: + // fine for stopping a runaway client, not for billing. + "ratelimits": [ + { + "name": "RL_UPLOAD", + "namespace_id": "1001", + "simple": { "limit": 60, "period": 60 } + }, + { + "name": "RL_PUBLISH", + "namespace_id": "1002", + "simple": { "limit": 10, "period": 60 } + }, + { + "name": "RL_AUTH", + "namespace_id": "1003", + "simple": { "limit": 10, "period": 60 } + }, + { + "name": "RL_ANON", + "namespace_id": "1004", + "simple": { "limit": 300, "period": 60 } + } + ], "queues": { "producers": [{ "queue": "adrive-jobs", "binding": "JOBS" }], "consumers": [ @@ -96,8 +120,9 @@ // `*.files.davis7.space` needs explicit certificate coverage on the // davis7.space zone, or use a dedicated content-domain zone apex. // 5. `wrangler secret put` MAINTENANCE_SECRET, WORKOS_API_KEY, - // WORKOS_CLIENT_ID, WORKOS_COOKIE_PASSWORD, WORKOS_WEBHOOK_SECRET - // (all `--env production`) + // WORKOS_CLIENT_ID, WORKOS_COOKIE_PASSWORD, WORKOS_WEBHOOK_SECRET, + // ADMIN_USER_IDS; optionally URLSCAN_API_KEY + CF_ACCOUNT_ID and + // CF_API_TOKEN + CF_ZONE_ID (docs/abuse.md) (all `--env production`) // 6. `DATABASE_URL=postgres://... bun db:pg:migrate` (see docs/release.md) "env": { "production": { @@ -155,6 +180,28 @@ "id": "09b884a95e5e40389d8554b3cd690d3f" } ], + "ratelimits": [ + { + "name": "RL_UPLOAD", + "namespace_id": "2001", + "simple": { "limit": 60, "period": 60 } + }, + { + "name": "RL_PUBLISH", + "namespace_id": "2002", + "simple": { "limit": 10, "period": 60 } + }, + { + "name": "RL_AUTH", + "namespace_id": "2003", + "simple": { "limit": 10, "period": 60 } + }, + { + "name": "RL_ANON", + "namespace_id": "2004", + "simple": { "limit": 300, "period": 60 } + } + ], "queues": { "producers": [{ "queue": "adrive-jobs-production", "binding": "JOBS" }], "consumers": [ diff --git a/scripts/check-wrangler-drift.mjs b/scripts/check-wrangler-drift.mjs index 3409cfa..9a39788 100644 --- a/scripts/check-wrangler-drift.mjs +++ b/scripts/check-wrangler-drift.mjs @@ -181,6 +181,17 @@ const main = () => { ); } } + // Rate limit namespaces differ per environment (1001.. vs 2001..); + // the binding names and their limits must not. + const ratelimits = (block) => + (block.ratelimits ?? []) + .map((entry) => ({ name: entry.name, simple: entry.simple })) + .sort((left, right) => left.name.localeCompare(right.name)); + if (!sameJson(ratelimits(config), ratelimits(prod))) { + drift.push( + `ratelimits: local=${JSON.stringify(ratelimits(config))} production=${JSON.stringify(ratelimits(prod))}` + ); + } const queueProducers = (block) => bindingNames(block.queues?.producers); if (!sameJson(queueProducers(config), queueProducers(prod))) { drift.push( From 8a31dd015c05c4af9044a08ba47f4cf378ed4313 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:12:46 -0700 Subject: [PATCH 02/11] FEATURE: Gate public sharing on org trust level trust-policy.ts decides what each level may do: new orgs stay private, verified and established ones may publish. Signing in with a verified email promotes new to verified; the maintenance tick promotes verified orgs on a paid plan for 14 days to established. Uploads that land public (including HTML, which is forced public), visibility changes, renames to .html, and site sessions and commits all pass through requirePublishAllowed, which answers 403 "Verify your email to share publicly" for a new org. Co-Authored-By: Claude Fable 5.1 --- apps/web/src/lib/server/services/auth.ts | 4 ++ .../lib/server/services/files/internals.ts | 8 +++ .../lib/server/services/files/mutations.ts | 9 ++- .../src/lib/server/services/files/upload.ts | 2 + apps/web/src/lib/server/services/lifecycle.ts | 13 +++- .../src/lib/server/services/sites/sessions.ts | 7 ++ apps/web/src/lib/server/trust-policy.ts | 54 +++++++++++++++ apps/web/src/lib/server/trust.ts | 67 +++++++++++++++++++ .../api/internal/maintenance/+server.ts | 12 ++-- 9 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/lib/server/trust-policy.ts create mode 100644 apps/web/src/lib/server/trust.ts diff --git a/apps/web/src/lib/server/services/auth.ts b/apps/web/src/lib/server/services/auth.ts index 9d9b2fb..3024027 100644 --- a/apps/web/src/lib/server/services/auth.ts +++ b/apps/web/src/lib/server/services/auth.ts @@ -17,6 +17,7 @@ import { import type { AuthContext, ResolvedCredential } from '../identity'; import { PgSql } from '../pg'; import { ensureTenant, personalOrgFor } from '../tenants'; +import { promoteVerified } from '../trust'; import { CurrentOrg, CurrentUser } from './current-org'; import { WorkOSClient } from './workos'; @@ -456,6 +457,9 @@ const makeAuth = Effect.gen(function* () { role }).pipe(storageError('create tenant rows')); } + // Verified email unlocks sharing; keep promotion with tenant bootstrap. + if (exchanged.user.emailVerified) + yield* promoteVerified(sql, orgId); return orgId; }) ) diff --git a/apps/web/src/lib/server/services/files/internals.ts b/apps/web/src/lib/server/services/files/internals.ts index 86f5b5c..6606b10 100644 --- a/apps/web/src/lib/server/services/files/internals.ts +++ b/apps/web/src/lib/server/services/files/internals.ts @@ -15,6 +15,7 @@ import { visibilityForFile } from '../../file-policy'; import { delaySecondsUntil } from '../../job-policy'; import { refreshSearchDocument } from '../../search-index'; import { ensureStorageHeadroom, reserveWithinPlan } from '../../storage-quota'; +import { requirePublishAllowed } from '../../trust'; import type { AppConfig } from '../../config'; import type { Blobs } from '../blobs'; import type { JobQueue } from '../jobs'; @@ -69,6 +70,11 @@ export const createInternals = (deps: CoreDeps) => { } ); + // Every path that turns a file public passes through here first; a + // `new` org is refused with the message that tells it what to do. + const ensurePublishAllowed = (becomesPublic: boolean) => + becomesPublic ? requirePublishAllowed(sql, org.id) : Effect.succeed(null); + // Cheap read before a body streams; the reservation inside the commit // transaction is what actually holds the bytes. const ensureHeadroom = (incomingBytes: number) => @@ -118,6 +124,7 @@ export const createInternals = (deps: CoreDeps) => { current.htmlForcedPublic ? 'text/html' : contentType, current.public ); + yield* ensurePublishAllowed(visibility.public && !current.public); // Optimistic concurrency on current_version: a concurrent upload // that committed first makes this update match nothing. yield* sql @@ -187,6 +194,7 @@ export const createInternals = (deps: CoreDeps) => { jobs, compensateStoredBlob, ensureHeadroom, + ensurePublishAllowed, reserveBytes, sendIndexJob, sendPurgeJob, diff --git a/apps/web/src/lib/server/services/files/mutations.ts b/apps/web/src/lib/server/services/files/mutations.ts index afb37fd..6ccb57a 100644 --- a/apps/web/src/lib/server/services/files/mutations.ts +++ b/apps/web/src/lib/server/services/files/mutations.ts @@ -26,7 +26,12 @@ export const mutationOps = ( | 'recordDownload' > => { const { sql, org } = internals; - const { findDashboardFile, sendIndexJob, sendPurgeJob } = internals; + const { + ensurePublishAllowed, + findDashboardFile, + sendIndexJob, + sendPurgeJob + } = internals; return { setVisibility: Effect.fn('Files.setVisibility')(function* (id, isPublic) { const current = yield* findDashboardFile(id); @@ -41,6 +46,7 @@ export const mutationOps = ( current.htmlForcedPublic ? 'text/html' : current.contentType, isPublic ); + yield* ensurePublishAllowed(visibility.public && !current.public); const updatedAt = new Date().toISOString(); yield* sql` UPDATE files @@ -151,6 +157,7 @@ export const mutationOps = ( current.contentType, current.public ); + yield* ensurePublishAllowed(visibility.public && !current.public); const updatedAt = new Date().toISOString(); const rows = yield* sql .withTransaction( diff --git a/apps/web/src/lib/server/services/files/upload.ts b/apps/web/src/lib/server/services/files/upload.ts index d8f5c63..2ec4bd4 100644 --- a/apps/web/src/lib/server/services/files/upload.ts +++ b/apps/web/src/lib/server/services/files/upload.ts @@ -18,6 +18,7 @@ export const uploadOps = ( const { blobs, sql, config, tags, org } = internals; const { ensureHeadroom, + ensurePublishAllowed, reserveBytes, compensateStoredBlob, commitStoredVersion, @@ -55,6 +56,7 @@ export const uploadOps = ( contentType, input.public ); + yield* ensurePublishAllowed(visibility.public); const id = crypto.randomUUID(); const resolvedTags = yield* tags.resolveNames(input.tags); const r2Key = `v/${id}/${crypto.randomUUID()}`; diff --git a/apps/web/src/lib/server/services/lifecycle.ts b/apps/web/src/lib/server/services/lifecycle.ts index 72a1995..d13a734 100644 --- a/apps/web/src/lib/server/services/lifecycle.ts +++ b/apps/web/src/lib/server/services/lifecycle.ts @@ -1,4 +1,6 @@ import { Context, Effect, Layer } from 'effect'; +import { PgSql } from '../pg'; +import { promoteEstablished } from '../trust'; import { Auth } from './auth'; import { Files } from './files'; import { Indexing } from './indexing'; @@ -18,6 +20,9 @@ export const ORG_SWEEP_LIMIT = 2; export interface LifecycleShape { // Work that is not tenant-scoped (device codes); runs once per tick. readonly global: Effect.Effect; + // Promotes verified orgs that have paid for 14 days to established + // (trust-policy.ts). Returns how many moved. + readonly trust: Effect.Effect; // One org's share of purges, indexing, and site cleanup; runAcrossOrgs // in edge.ts runs it once per randomly chosen live org. readonly org: Effect.Effect; @@ -82,6 +87,7 @@ export const summarize = ( ); const makeLifecycle = Effect.gen(function* () { + const sql = yield* PgSql; const auth = yield* Auth; const files = yield* Files; const indexing = yield* Indexing; @@ -90,6 +96,11 @@ const makeLifecycle = Effect.gen(function* () { const global = recover('authentication', auth.sweepExpired(100), 0).pipe( Effect.withSpan('Lifecycle.global') ); + const trust = recover( + 'trust', + Effect.suspend(() => promoteEstablished(sql, new Date())), + 0 + ).pipe(Effect.withSpan('Lifecycle.trust')); const org = runLifecycleTasks({ authentication: Effect.succeed(0), @@ -98,7 +109,7 @@ const makeLifecycle = Effect.gen(function* () { files: files.sweepPurges(ORG_SWEEP_LIMIT) }).pipe(Effect.withSpan('Lifecycle.org')); - return Lifecycle.of({ global, org }); + return Lifecycle.of({ global, trust, org }); }); export const LifecycleLive = Layer.effect(Lifecycle, makeLifecycle); diff --git a/apps/web/src/lib/server/services/sites/sessions.ts b/apps/web/src/lib/server/services/sites/sessions.ts index e4c8d41..626ca7c 100644 --- a/apps/web/src/lib/server/services/sites/sessions.ts +++ b/apps/web/src/lib/server/services/sites/sessions.ts @@ -4,6 +4,7 @@ import { InvalidRequest, NotFound, StorageError } from '../../errors'; import { delaySecondsUntil } from '../../job-policy'; import { refreshSearchDocument } from '../../search-index'; import { ensureStorageHeadroom, reserveWithinPlan } from '../../storage-quota'; +import { requirePublishAllowed } from '../../trust'; import { assertOpenSiteSession, prepareSiteManifest, @@ -43,6 +44,11 @@ export const sessionOps = ( message: 'Site manifest is invalid' }) }); + // Sites are always public, so an org that may not publish is + // refused before any asset bytes are accepted. + yield* requirePublishAllowed(sql, org.id); + // Declared manifest sizes gate the whole publish before any asset + // bytes are accepted; per-asset uploads re-verify actual lengths. const declaredBytes = prepared.assets.reduce( (total, asset) => total + asset.sizeBytes, 0 @@ -237,6 +243,7 @@ export const sessionOps = ( message: 'Site upload session is unavailable' }) }); + yield* requirePublishAllowed(sql, org.id); const assets = yield* stagedAssets(session.id); const totalSize = yield* Effect.try({ try: () => validateCommittedAssets(assets), diff --git a/apps/web/src/lib/server/trust-policy.ts b/apps/web/src/lib/server/trust-policy.ts new file mode 100644 index 0000000..5522000 --- /dev/null +++ b/apps/web/src/lib/server/trust-policy.ts @@ -0,0 +1,54 @@ +// Org trust levels (orgs.trust). An org starts `new` at sign-up, becomes +// `verified` once a member signs in with a verified email, and +// `established` after 14 days on a paid plan or an admin bump. `suspended` +// is the kill switch: the content host answers 404 and credentials stop +// resolving. + +export const TRUST_LEVELS = [ + 'new', + 'verified', + 'established', + 'suspended' +] as const; + +export type TrustLevel = (typeof TRUST_LEVELS)[number]; + +export const isTrustLevel = (value: string): value is TrustLevel => + (TRUST_LEVELS as ReadonlyArray).includes(value); + +// An unknown value in the column can only make an org less trusted. +export const parseTrust = (value: string): TrustLevel => + isTrustLevel(value) ? value : 'new'; + +// Whether the org may make anything public: files, and sites (which are +// always public). The free plan shares publicly too once verified; every +// publish by a verified org is scanned before it goes live. +export const canPublish = (trust: TrustLevel) => + trust === 'verified' || trust === 'established'; + +// Verified orgs publish behind the scanner; established ones publish now +// and are scanned after. +export const scanBeforePublish = (trust: TrustLevel) => trust === 'verified'; + +export const publishLimitPerHour = (trust: TrustLevel) => { + switch (trust) { + case 'verified': + return 30; + case 'established': + return 300; + case 'new': + case 'suspended': + return 0; + } +}; + +export const PUBLISH_BLOCKED_MESSAGE = 'Verify your email to share publicly'; + +export const ESTABLISHED_AFTER_DAYS = 14; + +// The cutoff for the verified -> established sweep: orgs created before it +// on a paid plan have been around long enough. +export const establishedCutoff = (now: Date) => + new Date( + now.getTime() - ESTABLISHED_AFTER_DAYS * 24 * 60 * 60 * 1000 + ).toISOString(); diff --git a/apps/web/src/lib/server/trust.ts b/apps/web/src/lib/server/trust.ts new file mode 100644 index 0000000..f1e94df --- /dev/null +++ b/apps/web/src/lib/server/trust.ts @@ -0,0 +1,67 @@ +import type { PgClient } from '@effect/sql-pg'; +import { Effect } from 'effect'; +import { InvalidRequest, StorageError } from './errors'; +import { + PUBLISH_BLOCKED_MESSAGE, + canPublish, + establishedCutoff, + parseTrust, + type TrustLevel +} from './trust-policy'; + +const storageError = (operation: string) => + Effect.mapError((cause: unknown) => new StorageError({ operation, cause })); + +// The org's trust level, read fresh on every publish decision so an admin +// suspension or bump lands on the next request. +export const orgTrust = (sql: PgClient.PgClient, orgId: string) => + sql<{ trust: string }>` + SELECT trust FROM orgs WHERE id = ${orgId} LIMIT 1 + `.pipe( + Effect.map((rows): TrustLevel => parseTrust(rows[0]?.trust ?? 'new')), + storageError('read org trust') + ); + +// The gate in front of every path that makes content public. A `new` org +// gets a 403 that tells the member what unlocks it. +export const requirePublishAllowed = (sql: PgClient.PgClient, orgId: string) => + Effect.gen(function* () { + const trust = yield* orgTrust(sql, orgId); + if (!canPublish(trust)) { + return yield* new InvalidRequest({ + status: 403, + message: PUBLISH_BLOCKED_MESSAGE + }); + } + return trust; + }); + +// Sign-in with a verified email moves the org out of `new`. Never touches +// any other level: a suspension survives a sign-in. +export const promoteVerified = (sql: PgClient.PgClient, orgId: string) => + sql` + UPDATE orgs SET trust = 'verified' + WHERE id = ${orgId} AND trust = 'new' + `.pipe(Effect.asVoid, storageError('promote org to verified')); + +// Maintenance sweep: verified orgs on a paid plan for 14 days become +// established. Returns how many were promoted. +export const promoteEstablished = ( + sql: PgClient.PgClient, + now: Date, + limit = 100 +) => + sql<{ id: string }>` + UPDATE orgs SET trust = 'established' + WHERE id IN ( + SELECT id FROM orgs + WHERE trust = 'verified' AND plan <> 'free' + AND created_at < ${establishedCutoff(now)} + ORDER BY created_at + LIMIT ${Math.max(1, Math.min(limit, 1000))} + ) + RETURNING id + `.pipe( + Effect.map((rows) => rows.length), + storageError('promote orgs to established') + ); diff --git a/apps/web/src/routes/api/internal/maintenance/+server.ts b/apps/web/src/routes/api/internal/maintenance/+server.ts index ee3b3d7..8ae3cc1 100644 --- a/apps/web/src/routes/api/internal/maintenance/+server.ts +++ b/apps/web/src/routes/api/internal/maintenance/+server.ts @@ -6,11 +6,12 @@ import { runAcrossOrgs, runEdgeWithEvent } from '$lib/server/edge'; import { Unauthorized } from '$lib/server/errors'; import { Lifecycle, summarize } from '$lib/server/services/lifecycle'; -// The cron tick: one global pass (device codes), then one bounded pass -// per randomly chosen live org so no tenant starves another. +// The cron tick: one global pass (device codes, trust promotions), then +// one bounded pass per randomly chosen live org so no tenant starves +// another. export const POST: RequestHandler = async (event) => { const { request } = event; - const global = await runEdgeWithEvent( + const { global, trust } = await runEdgeWithEvent( event, Effect.gen(function* () { const config = yield* AppConfig; @@ -31,7 +32,9 @@ export const POST: RequestHandler = async (event) => { message: 'Scheduled request is unauthorized' }); } - return yield* lifecycle.global; + const global = yield* lifecycle.global; + const trust = yield* lifecycle.trust; + return { global, trust }; }) ); if (!event.platform) return new Response(null, { status: 204 }); @@ -43,6 +46,7 @@ export const POST: RequestHandler = async (event) => { JSON.stringify({ message: 'maintenance tick', orgs: perOrg.length, + trustPromotions: trust, ...summarize( global, perOrg.map((entry) => entry.value) From 5de121f5cd0b6e53521ddd94faed8acbaf9fd0f3 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:31:53 -0700 Subject: [PATCH 03/11] FEATURE: Scan published content before it goes live Every version that becomes public gets a scan job. A verified org's publish (visibility change, rename to .html, site commit) is held with publish_pending until the scanner clears it; established orgs publish now and are scanned after. The Scanner runs three checks and records a scan_verdicts row for each: the object's sha256 against blocked_hashes, the first bytes against the declared type (mime-sniff.ts), and the outbound links in HTML through the Cloudflare URL Scanner (UrlReputation; a Null answers clean when URLSCAN_API_KEY is unset, `fake:` selects a fake). Link scans are collected by re-sending the job with the scan ids. Clean publishes a held row and purges the edge cache; malicious quarantines the file, which content routes then 404, and writes a notification; suspicious stays held for review. Co-Authored-By: Claude Fable 5.1 --- apps/web/.dev.vars.example | 15 + apps/web/migrations-pg/0006_scans.sql | 35 ++ apps/web/migrations-pg/0007_abuse.sql | 51 ++ apps/web/src/env.d.ts | 12 + apps/web/src/lib/server/config.ts | 45 ++ apps/web/src/lib/server/edge.ts | 6 + apps/web/src/lib/server/file-rows.ts | 4 + apps/web/src/lib/server/html-links.ts | 65 +++ apps/web/src/lib/server/jobs/consumer.ts | 11 +- apps/web/src/lib/server/layer.ts | 11 + apps/web/src/lib/server/mime-sniff.ts | 185 +++++++ apps/web/src/lib/server/scan-policy.ts | 63 +++ .../src/lib/server/services/cache-purge.ts | 117 ++++ .../lib/server/services/files/internals.ts | 35 +- .../lib/server/services/files/mutations.ts | 32 +- .../src/lib/server/services/files/queries.ts | 4 +- .../src/lib/server/services/files/upload.ts | 4 +- apps/web/src/lib/server/services/scanner.ts | 521 ++++++++++++++++++ .../web/src/lib/server/services/sites/read.ts | 1 + .../src/lib/server/services/sites/sessions.ts | 23 +- .../src/lib/server/services/url-reputation.ts | 191 +++++++ packages/shared/src/index.ts | 10 +- 22 files changed, 1416 insertions(+), 25 deletions(-) create mode 100644 apps/web/migrations-pg/0006_scans.sql create mode 100644 apps/web/migrations-pg/0007_abuse.sql create mode 100644 apps/web/src/lib/server/html-links.ts create mode 100644 apps/web/src/lib/server/mime-sniff.ts create mode 100644 apps/web/src/lib/server/scan-policy.ts create mode 100644 apps/web/src/lib/server/services/cache-purge.ts create mode 100644 apps/web/src/lib/server/services/scanner.ts create mode 100644 apps/web/src/lib/server/services/url-reputation.ts diff --git a/apps/web/.dev.vars.example b/apps/web/.dev.vars.example index cd1b516..eaeffd2 100644 --- a/apps/web/.dev.vars.example +++ b/apps/web/.dev.vars.example @@ -32,3 +32,18 @@ CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE="postgres://adrive:adri # Vite's allowlist is a SHELL setting, not a Worker variable. Start with: # __VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS="siva.otter-hawksbill.ts.net,.100.100.40.20.nip.io" bun --filter @adrive/web dev # Adjust both allowlist entries to match your dashboard/content hostnames. + +# Abuse controls (docs/abuse.md). All optional locally. +# Comma-separated WorkOS user ids allowed on /admin. The WorkOS fake signs +# you in as `user_local`, so that is the id to list here for local work. +ADMIN_USER_IDS="user_local" +# Cloudflare URL Scanner: links inside published HTML are checked against +# it. Leave empty to skip the check (recorded as such on the verdict), or +# set `fake:clean` / `fake:malicious` to exercise the pipeline offline. +# CF_ACCOUNT_ID is the account that owns the token. +URLSCAN_API_KEY="" +CF_ACCOUNT_ID="" +# Zone-level cache purge for the kill switch (Cache Purge permission on the +# content zone). Leave empty to log the purge instead. +CF_API_TOKEN="" +CF_ZONE_ID="" diff --git a/apps/web/migrations-pg/0006_scans.sql b/apps/web/migrations-pg/0006_scans.sql new file mode 100644 index 0000000..40c3dfe --- /dev/null +++ b/apps/web/migrations-pg/0006_scans.sql @@ -0,0 +1,35 @@ +-- migrate:up +-- Content scanning (services/scanner.ts). A publish by a verified org is +-- held (publish_pending) until the scanner clears it; a malicious verdict +-- on any org quarantines the file, which every content route then 404s. +-- One verdict row per (file, version, check). + +ALTER TABLE files ADD COLUMN quarantined boolean NOT NULL DEFAULT false; +ALTER TABLE files ADD COLUMN publish_pending boolean NOT NULL DEFAULT false; + +CREATE INDEX files_held_idx ON files (org_id, updated_at DESC) + WHERE quarantined OR publish_pending; + +CREATE TABLE scan_verdicts ( + file_id text NOT NULL REFERENCES files (id) ON DELETE CASCADE, + org_id text NOT NULL REFERENCES orgs (id), + version integer NOT NULL CHECK (version > 0), + verdict text NOT NULL CHECK (verdict IN ('clean', 'suspicious', 'malicious')), + source text NOT NULL, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (file_id, version, source) +); + +CREATE INDEX scan_verdicts_review_idx ON scan_verdicts (verdict, created_at DESC) + WHERE verdict <> 'clean'; + +ALTER TABLE scan_verdicts ENABLE ROW LEVEL SECURITY; +ALTER TABLE scan_verdicts FORCE ROW LEVEL SECURITY; +CREATE POLICY org_isolation ON scan_verdicts USING (app_org_visible(org_id)); + +-- migrate:down +DROP TABLE IF EXISTS scan_verdicts; +DROP INDEX IF EXISTS files_held_idx; +ALTER TABLE files DROP COLUMN IF EXISTS publish_pending; +ALTER TABLE files DROP COLUMN IF EXISTS quarantined; diff --git a/apps/web/migrations-pg/0007_abuse.sql b/apps/web/migrations-pg/0007_abuse.sql new file mode 100644 index 0000000..4254910 --- /dev/null +++ b/apps/web/migrations-pg/0007_abuse.sql @@ -0,0 +1,51 @@ +-- migrate:up +-- Abuse controls. blocked_hashes is the first scanner check (a nightly +-- mirror of a known-bad list can append to it; so can an admin). +-- notifications tell an org's members what the scanner or an admin did to +-- their file. reports come from anyone on the content host and are worked +-- from /admin, so they are a platform table with no RLS. + +CREATE TABLE blocked_hashes ( + sha256 text PRIMARY KEY CHECK (sha256 ~ '^[0-9a-f]{64}$'), + reason text NOT NULL DEFAULT '', + added_by text, + added_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE notifications ( + id text PRIMARY KEY, + org_id text NOT NULL REFERENCES orgs (id) ON DELETE CASCADE, + kind text NOT NULL, + message text NOT NULL, + file_id text, + created_at timestamptz NOT NULL DEFAULT now(), + read_at timestamptz +); + +CREATE INDEX notifications_unread_idx ON notifications (org_id, created_at DESC) + WHERE read_at IS NULL; + +ALTER TABLE notifications ENABLE ROW LEVEL SECURITY; +ALTER TABLE notifications FORCE ROW LEVEL SECURITY; +CREATE POLICY org_isolation ON notifications USING (app_org_visible(org_id)); + +CREATE TABLE reports ( + id text PRIMARY KEY, + org_id text NOT NULL, + file_id text NOT NULL, + version integer, + reason text NOT NULL, + reporter_ip_hash text NOT NULL, + details text, + created_at timestamptz NOT NULL DEFAULT now(), + resolved_at timestamptz, + resolution text +); + +CREATE INDEX reports_open_idx ON reports (created_at DESC) WHERE resolved_at IS NULL; +CREATE INDEX reports_file_idx ON reports (file_id, created_at DESC); + +-- migrate:down +DROP TABLE IF EXISTS reports; +DROP TABLE IF EXISTS notifications; +DROP TABLE IF EXISTS blocked_hashes; diff --git a/apps/web/src/env.d.ts b/apps/web/src/env.d.ts index 83d54a0..d78c29d 100644 --- a/apps/web/src/env.d.ts +++ b/apps/web/src/env.d.ts @@ -9,6 +9,13 @@ declare global { WORKOS_CLIENT_ID?: string; WORKOS_COOKIE_PASSWORD?: string; WORKOS_WEBHOOK_SECRET?: string; + // Abuse controls (docs/abuse.md). All optional: the Null services + // stand in when unset. + ADMIN_USER_IDS?: string; + URLSCAN_API_KEY?: string; + CF_ACCOUNT_ID?: string; + CF_API_TOKEN?: string; + CF_ZONE_ID?: string; } namespace Cloudflare { @@ -19,6 +26,11 @@ declare global { WORKOS_CLIENT_ID?: string; WORKOS_COOKIE_PASSWORD?: string; WORKOS_WEBHOOK_SECRET?: string; + ADMIN_USER_IDS?: string; + URLSCAN_API_KEY?: string; + CF_ACCOUNT_ID?: string; + CF_API_TOKEN?: string; + CF_ZONE_ID?: string; } } } diff --git a/apps/web/src/lib/server/config.ts b/apps/web/src/lib/server/config.ts index 9b17749..f967c1d 100644 --- a/apps/web/src/lib/server/config.ts +++ b/apps/web/src/lib/server/config.ts @@ -11,6 +11,20 @@ export interface WorkOSConfig { readonly webhookSecret: string; } +// Cloudflare URL Scanner (services/url-reputation.ts). Null when +// URLSCAN_API_KEY is unset; a `fake:` key selects the fake. +export interface UrlScannerConfig { + readonly apiKey: string; + readonly accountId: string; +} + +// Zone-level cache purge (services/cache-purge.ts). Null when either +// CF_API_TOKEN or CF_ZONE_ID is unset. +export interface CloudflareZoneConfig { + readonly apiToken: string; + readonly zoneId: string; +} + export interface AppConfigShape { readonly dashboardOrigin: string; // Tenant content is served from `.` over the @@ -22,6 +36,10 @@ export interface AppConfigShape { // Signs the Worker facade's cron and queue self-requests. readonly maintenanceSecret: string; readonly workos: WorkOSConfig; + readonly urlScanner: UrlScannerConfig | null; + readonly cloudflareZone: CloudflareZoneConfig | null; + // WorkOS user ids allowed on /admin (ADMIN_USER_IDS, comma-separated). + readonly adminUserIds: ReadonlySet; readonly semanticSearch: 'off' | 'auto' | 'required'; readonly embeddingModel: '@cf/baai/bge-small-en-v1.5'; readonly embeddingPooling: 'cls'; @@ -78,6 +96,30 @@ const workosFromEnv = (env: Env): WorkOSConfig => { return { apiKey: rawApiKey, clientId, cookiePassword, webhookSecret }; }; +const urlScannerFromEnv = (env: Env): UrlScannerConfig | null => { + const apiKey = optionalString(env.URLSCAN_API_KEY); + if (apiKey === '') return null; + const accountId = optionalString(env.CF_ACCOUNT_ID); + if (!accountId && !apiKey.startsWith('fake:')) { + throw new Error('CF_ACCOUNT_ID is required alongside URLSCAN_API_KEY'); + } + return { apiKey, accountId }; +}; + +const cloudflareZoneFromEnv = (env: Env): CloudflareZoneConfig | null => { + const apiToken = optionalString(env.CF_API_TOKEN); + const zoneId = optionalString(env.CF_ZONE_ID); + return apiToken && zoneId ? { apiToken, zoneId } : null; +}; + +const adminUserIdsFromEnv = (env: Env) => + new Set( + optionalString(env.ADMIN_USER_IDS) + .split(',') + .map((id) => id.trim()) + .filter((id) => id.length > 0) + ); + export const configFromEnv = (env: Env) => { const origins = normalizeOrigins({ dashboardOrigin: env.DASHBOARD_ORIGIN, @@ -111,6 +153,9 @@ export const configFromEnv = (env: Env) => { maxUploadBytes, maintenanceSecret: env.MAINTENANCE_SECRET, workos: workosFromEnv(env), + urlScanner: urlScannerFromEnv(env), + cloudflareZone: cloudflareZoneFromEnv(env), + adminUserIds: adminUserIdsFromEnv(env), semanticSearch, embeddingModel: '@cf/baai/bge-small-en-v1.5', embeddingPooling: 'cls', diff --git a/apps/web/src/lib/server/edge.ts b/apps/web/src/lib/server/edge.ts index bc6a52f..c62787c 100644 --- a/apps/web/src/lib/server/edge.ts +++ b/apps/web/src/lib/server/edge.ts @@ -34,12 +34,18 @@ import type { GrantSecrets } from './services/grant-secrets'; import type { JobQueue } from './services/jobs'; import type { Org } from './services/org'; import type { RateLimits } from './services/rate-limits'; +import type { CloudflareCachePurge } from './services/cache-purge'; +import type { Scanner } from './services/scanner'; +import type { UrlReputation } from './services/url-reputation'; import type { WorkOSClient } from './services/workos'; export type AppServices = | PgSql | AppConfig | RateLimits + | UrlReputation + | CloudflareCachePurge + | Scanner | AuthGuardStore | Auth | Blobs diff --git a/apps/web/src/lib/server/file-rows.ts b/apps/web/src/lib/server/file-rows.ts index 8b53bce..e4ee369 100644 --- a/apps/web/src/lib/server/file-rows.ts +++ b/apps/web/src/lib/server/file-rows.ts @@ -9,6 +9,8 @@ export const DashboardFileRow = Schema.Struct({ current_version: Schema.Int, size_bytes: Schema.Int, is_public: Schema.Boolean, + quarantined: Schema.Boolean, + publish_pending: Schema.Boolean, has_html: Schema.Boolean, created_at: Schema.String, updated_at: Schema.String, @@ -40,6 +42,8 @@ export const dashboardFileColumns = ` f.current_version, f.size_bytes, f.public AS is_public, + f.quarantined, + f.publish_pending, EXISTS ( SELECT 1 FROM file_versions html_version WHERE html_version.file_id = f.id diff --git a/apps/web/src/lib/server/html-links.ts b/apps/web/src/lib/server/html-links.ts new file mode 100644 index 0000000..6af2f93 --- /dev/null +++ b/apps/web/src/lib/server/html-links.ts @@ -0,0 +1,65 @@ +// The outbound targets inside a published HTML document: ``, +// ` + + + Admin · adrive + + +
+

Admin

+ +
+

+ Reports ({overview.reports.length}) +

+
+ + + + + + + + + + + + + {#each overview.reports as report (report.id)} + + + + + + + + + {:else} + + {/each} + +
FiledReasonFileOrgReporter
{formatDate(report.createdAt)} + {report.reason} + {#if report.details} +
+ {report.details} +
+ {/if} + 		+ {#if report.file} + {report.file.name} +
+ {report.file.quarantined + ? 'quarantined' + : report.file.publishPending + ? 'held' + : report.file.public + ? 'public' + : 'private'} +
+ {:else} + gone + {/if} +
+ {report.fileId} +
+ 		{report.orgSlug ?? report.orgId}{report.reporter} +
+ + + + +
+
None
+
+
+ +
+

+ Held and quarantined files ({overview.held.length}) +

+
+ + + + + + + + + + + + + {#each overview.held as file (file.id)} + + + + + + + + + {:else} + + {/each} + +
UpdatedFileOrgStateVerdicts
{formatDate(file.updatedAt)} + {file.name} +
+ {file.contentType} · v{file.version} +
+
{file.id}
+ 		{file.orgSlug}{file.quarantined ? 'quarantined' : 'held for review'} + {#each file.verdicts as verdict (verdict.source)} +
+ {verdict.source} + {verdict.verdict} +
+ {/each} + 		+
+ + +
+
None
+
+
+ +
+

+ Failed jobs ({overview.failedJobs.length}) +

+
+ + + + + + + + + + + + {#each overview.failedJobs as job (job.id)} + + + + + + + + {:else} + + {/each} + +
FailedKindOrgAttemptsError
{formatDate(job.failedAt)} + {job.kind} +
+ {JSON.stringify(job.payload)} +
+ 		{job.orgId ?? ''}{job.attempts}{job.error}
None
+
+
+ +
+

+ Recent orgs ({overview.orgs.length}) +

+
+ + + + + + + + + + + + + + + {#each overview.orgs as org (org.id)} + + + + + + + + + + + {:else} + + {/each} + +
CreatedSlugIdTrustPlanStoredFiles
{formatDate(org.createdAt)}{org.slug}{org.id}{org.trust}{org.plan}{formatBytes(org.storedBytes)}{org.fileCount} +
+ {#if org.trust === 'suspended'} + + {:else} + {#if org.trust !== 'established'} + + {/if} + + {/if} +
+
None
+
+
+ +
+

Block a hash

+
{ + event.preventDefault(); + void blockHash(); + }} + > + + + +
+
+
diff --git a/apps/web/src/routes/api/admin/files/[id]/+server.ts b/apps/web/src/routes/api/admin/files/[id]/+server.ts new file mode 100644 index 0000000..b1627e3 --- /dev/null +++ b/apps/web/src/routes/api/admin/files/[id]/+server.ts @@ -0,0 +1,26 @@ +import type { RequestHandler } from './$types'; +import { Effect, Schema } from 'effect'; +import { runEdge } from '$lib/server/edge'; +import { decodeJson } from '$lib/server/request-json'; +import { requireAdmin } from '$lib/server/request-auth'; +import { Admin } from '$lib/server/services/admin'; + +const Body = Schema.Struct({ + verdict: Schema.Literals(['clean', 'malicious']) +}); + +// An operator's verdict on a held or quarantined file. +export const PATCH: RequestHandler = (event) => + runEdge( + Effect.gen(function* () { + const admin = yield* Admin; + yield* requireAdmin(event); + const { verdict } = yield* decodeJson( + event.request, + Body, + 'A verdict (clean, malicious) is required' + ); + yield* admin.markFile(event.params.id, verdict); + return Response.json({ ok: true as const }); + }) + ); diff --git a/apps/web/src/routes/api/admin/hashes/+server.ts b/apps/web/src/routes/api/admin/hashes/+server.ts new file mode 100644 index 0000000..b1751ec --- /dev/null +++ b/apps/web/src/routes/api/admin/hashes/+server.ts @@ -0,0 +1,28 @@ +import type { RequestHandler } from './$types'; +import { Effect, Schema } from 'effect'; +import { runEdge } from '$lib/server/edge'; +import { decodeJson } from '$lib/server/request-json'; +import { requireAdmin } from '$lib/server/request-auth'; +import { Admin } from '$lib/server/services/admin'; + +const Body = Schema.Struct({ + sha256: Schema.String, + reason: Schema.optionalKey(Schema.String) +}); + +// Appends to blocked_hashes; the scanner quarantines any upload that +// matches from then on. +export const POST: RequestHandler = (event) => + runEdge( + Effect.gen(function* () { + const admin = yield* Admin; + const auth = yield* requireAdmin(event); + const body = yield* decodeJson( + event.request, + Body, + 'A sha256 is required' + ); + yield* admin.blockHash(body.sha256, body.reason ?? '', auth.userId); + return Response.json({ ok: true as const }, { status: 201 }); + }) + ); diff --git a/apps/web/src/routes/api/admin/orgs/[id]/+server.ts b/apps/web/src/routes/api/admin/orgs/[id]/+server.ts new file mode 100644 index 0000000..ae4056f --- /dev/null +++ b/apps/web/src/routes/api/admin/orgs/[id]/+server.ts @@ -0,0 +1,36 @@ +import type { RequestHandler } from './$types'; +import { Effect, Schema } from 'effect'; +import { runEdge } from '$lib/server/edge'; +import { decodeJson } from '$lib/server/request-json'; +import { requireAdmin } from '$lib/server/request-auth'; +import { Admin } from '$lib/server/services/admin'; + +const Body = Schema.Union([ + Schema.Struct({ action: Schema.Literal('suspend') }), + Schema.Struct({ action: Schema.Literal('restore') }), + Schema.Struct({ + action: Schema.Literal('trust'), + trust: Schema.Literals(['new', 'verified', 'established']) + }) +]); + +export const PATCH: RequestHandler = (event) => + runEdge( + Effect.gen(function* () { + const admin = yield* Admin; + yield* requireAdmin(event); + const body = yield* decodeJson( + event.request, + Body, + 'An org action (suspend, restore, trust) is required' + ); + const orgId = event.params.id; + const org = + body.action === 'suspend' + ? yield* admin.suspendOrg(orgId) + : body.action === 'restore' + ? yield* admin.restoreOrg(orgId) + : yield* admin.setTrust(orgId, body.trust); + return Response.json({ org }); + }) + ); diff --git a/apps/web/src/routes/api/admin/overview/+server.ts b/apps/web/src/routes/api/admin/overview/+server.ts new file mode 100644 index 0000000..92da4c8 --- /dev/null +++ b/apps/web/src/routes/api/admin/overview/+server.ts @@ -0,0 +1,14 @@ +import type { RequestHandler } from './$types'; +import { Effect } from 'effect'; +import { runEdge } from '$lib/server/edge'; +import { requireAdmin } from '$lib/server/request-auth'; +import { Admin } from '$lib/server/services/admin'; + +export const GET: RequestHandler = (event) => + runEdge( + Effect.gen(function* () { + const admin = yield* Admin; + yield* requireAdmin(event); + return Response.json(yield* admin.overview); + }) + ); diff --git a/apps/web/src/routes/api/admin/reports/[id]/+server.ts b/apps/web/src/routes/api/admin/reports/[id]/+server.ts new file mode 100644 index 0000000..e2a1094 --- /dev/null +++ b/apps/web/src/routes/api/admin/reports/[id]/+server.ts @@ -0,0 +1,24 @@ +import type { RequestHandler } from './$types'; +import { Effect, Schema } from 'effect'; +import { runEdge } from '$lib/server/edge'; +import { RESOLUTIONS } from '$lib/server/report-policy'; +import { decodeJson } from '$lib/server/request-json'; +import { requireAdmin } from '$lib/server/request-auth'; +import { Admin } from '$lib/server/services/admin'; + +const Body = Schema.Struct({ resolution: Schema.Literals(RESOLUTIONS) }); + +export const PATCH: RequestHandler = (event) => + runEdge( + Effect.gen(function* () { + const admin = yield* Admin; + yield* requireAdmin(event); + const { resolution } = yield* decodeJson( + event.request, + Body, + `A resolution (${RESOLUTIONS.join(', ')}) is required` + ); + yield* admin.resolveReport(event.params.id, resolution); + return Response.json({ ok: true as const }); + }) + ); From a19886a033c65e6afa7fbde213641ad11130ba8f Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:42:46 -0700 Subject: [PATCH 06/11] DOCS: Add the abuse runbook docs/abuse.md: the zone checklist (WAF managed rules, bot fight mode, hotlink protection off, Netcraft feed, DMCA agent, abuse@ mailbox), the trust levels and what each may do, the scan pipeline check by check and what each verdict does to the row, the report endpoint, the kill switch step by step including the manual purge when the zone API is not configured, the rate limit bindings, and how to work /admin. The admin review list also surfaces live files the scanner flagged after publish until an operator rules on them, and admin verdict rows record who made the call. Co-Authored-By: Claude Fable 5.1 --- apps/web/src/lib/server/services/admin.ts | 33 +++- apps/web/src/routes/admin/+page.svelte | 10 +- .../routes/api/admin/files/[id]/+server.ts | 4 +- docs/abuse.md | 169 ++++++++++++++++++ 4 files changed, 204 insertions(+), 12 deletions(-) create mode 100644 docs/abuse.md diff --git a/apps/web/src/lib/server/services/admin.ts b/apps/web/src/lib/server/services/admin.ts index 489bf99..353fc10 100644 --- a/apps/web/src/lib/server/services/admin.ts +++ b/apps/web/src/lib/server/services/admin.ts @@ -175,11 +175,13 @@ export interface AdminShape { orgId: string, trust: Exclude ) => Effect.Effect; - // An operator's verdict on a held or quarantined file. `clean` clears - // the quarantine and publishes a held row; `malicious` quarantines. + // An operator's verdict on a held, flagged, or quarantined file. `clean` + // clears the quarantine and publishes a held row; `malicious` + // quarantines. Recorded as an `admin` verdict row naming the operator. readonly markFile: ( fileId: string, - verdict: Exclude + verdict: Exclude, + by: string ) => Effect.Effect; readonly blockHash: ( sha256: string, @@ -348,7 +350,23 @@ const makeAdmin = Effect.gen(function* () { ), '[]'::jsonb) AS verdicts FROM files f JOIN orgs o ON o.id = f.org_id - WHERE (f.quarantined OR f.publish_pending) AND f.deleted_at IS NULL + WHERE f.deleted_at IS NULL AND ( + f.quarantined OR f.publish_pending + -- A live file the scanner flagged after publish, until an + -- operator has ruled on that version. + OR ( + EXISTS ( + SELECT 1 FROM scan_verdicts v + WHERE v.file_id = f.id AND v.version = f.current_version + AND v.verdict <> 'clean' AND v.source <> 'admin' + ) + AND NOT EXISTS ( + SELECT 1 FROM scan_verdicts v + WHERE v.file_id = f.id AND v.version = f.current_version + AND v.source = 'admin' + ) + ) + ) ORDER BY f.updated_at DESC LIMIT 200`, sql` @@ -443,16 +461,17 @@ const makeAdmin = Effect.gen(function* () { yield* setOrgTrust(orgId, trust); return yield* loadOrg(orgId); }), - markFile: Effect.fn('Admin.markFile')(function* (fileId, verdict) { + markFile: Effect.fn('Admin.markFile')(function* (fileId, verdict, by) { const file = yield* fileOrgAndVersion(fileId); yield* sql` INSERT INTO scan_verdicts (file_id, org_id, version, verdict, source, details) VALUES ( ${fileId}, ${file.org_id}, ${file.current_version}, ${verdict}, - 'admin', '{}'::jsonb + 'admin', ${JSON.stringify({ by })}::jsonb ) ON CONFLICT (file_id, version, source) DO UPDATE - SET verdict = EXCLUDED.verdict, created_at = now() + SET verdict = EXCLUDED.verdict, details = EXCLUDED.details, + created_at = now() `.pipe(storageError('record admin verdict')); if (verdict === 'malicious') { yield* sql` diff --git a/apps/web/src/routes/admin/+page.svelte b/apps/web/src/routes/admin/+page.svelte index 0792749..2b15bf9 100644 --- a/apps/web/src/routes/admin/+page.svelte +++ b/apps/web/src/routes/admin/+page.svelte @@ -191,9 +191,13 @@
{file.id}
{file.orgSlug} - {file.quarantined ? 'quarantined' : 'held for review'} + + {file.quarantined + ? 'quarantined' + : file.publishPending + ? 'held' + : 'live, flagged'} + {#each file.verdicts as verdict (verdict.source)}
diff --git a/apps/web/src/routes/api/admin/files/[id]/+server.ts b/apps/web/src/routes/api/admin/files/[id]/+server.ts index b1627e3..0d7d7b9 100644 --- a/apps/web/src/routes/api/admin/files/[id]/+server.ts +++ b/apps/web/src/routes/api/admin/files/[id]/+server.ts @@ -14,13 +14,13 @@ export const PATCH: RequestHandler = (event) => runEdge( Effect.gen(function* () { const admin = yield* Admin; - yield* requireAdmin(event); + const auth = yield* requireAdmin(event); const { verdict } = yield* decodeJson( event.request, Body, 'A verdict (clean, malicious) is required' ); - yield* admin.markFile(event.params.id, verdict); + yield* admin.markFile(event.params.id, verdict, auth.userId); return Response.json({ ok: true as const }); }) ); diff --git a/docs/abuse.md b/docs/abuse.md new file mode 100644 index 0000000..aec5dd6 --- /dev/null +++ b/docs/abuse.md @@ -0,0 +1,169 @@ +# Abuse controls + +One bad tenant must not take the content domain down, and an operator +must be able to stop one in under a minute. This is what the code does, +what the Cloudflare zone must be set to by hand, and how to run the +controls. + +## Zone checklist (not code) + +Do these once on the zone that owns `CONTENT_DOMAIN`, and again whenever +the content domain moves. Tenant content is only ever served from +`.`; the dashboard origin is a separate host. + +- **WAF managed rules** on the content zone: enable the Cloudflare + Managed Ruleset and the OWASP Core Ruleset in blocking mode. Content + responses are static files, so false positives are rare; if one appears + add a skip rule scoped to `/f/*`, never to the whole zone. +- **Bot Fight Mode** (or Super Bot Fight Mode on a paid plan): on. Scraper + traffic against tenant hosts otherwise counts against the per-IP rate + limit and R2 egress. +- **Hotlink protection: off.** Serving files from elsewhere is the product. +- **Rate limiting rules** at the zone level are a second layer over the + Worker bindings below; a rule of the shape "more than 1000 requests per + minute from one IP to `*./*` → managed challenge" is + enough. +- **Cache**: leave the default cache level. The Worker sets `Cache-Control` + per file (immutable for pinned versions, revalidate for sites) and + purges by URL and by host when a verdict changes (below), so no zone + cache rule is needed. +- **Netcraft**: register the content domain as a hosting provider in the + Netcraft takedown feed so phishing reports arrive at `abuse@` instead + of the registrar. +- **DMCA agent**: register a designated agent with the US Copyright Office + for the operating entity and publish the contact on the landing site. +- **abuse@ mailbox**: create it, route it to the same people who watch + `/admin`, and list it on the landing site next to the DMCA agent. +- **Reports**: every tenant host answers `GET /report?f=` with a + form; link it from the landing site's abuse page so a reporter who + lands on the bare domain finds it. + +## Trust levels + +`orgs.trust` (`apps/web/src/lib/server/trust-policy.ts`): + +| Level | Reached by | May publish | Scan | +| ------------- | ------------------------------------------------------------ | --------------- | ---------------- | +| `new` | Sign-up | No (403) | n/a | +| `verified` | A sign-in with a verified email | Yes | Before it's live | +| `established` | 14 days on a paid plan (maintenance sweep), or an admin bump | Yes | After it's live | +| `suspended` | The kill switch | No; host is 404 | n/a | + +"Publish" is anything that makes content public: an upload that lands +public (including HTML, which is always public), a visibility change, a +rename to `.html`, a site session or commit. A `new` org gets 403 "Verify +your email to share publicly". The free plan can share publicly once +verified; every publish by a verified org is scanned first. + +Bumping trust is an admin action on `/admin` (or `PATCH +/api/admin/orgs/` with `{ "action": "trust", "trust": "established" }`). + +## Scan pipeline + +`apps/web/src/lib/server/services/scanner.ts`. Every version that becomes +public gets a `scan` job on the queue. For a verified org the row stays +private with `files.publish_pending = true` until the scan clears it; for +an established org it is public at once and scanned after. + +Checks, in order, each writing one `scan_verdicts` row per +`(file, version, source)`: + +1. `hash`: the object's SHA-256 (objects up to 32 MiB; larger ones are + recorded as skipped) against `blocked_hashes`. A hit is `malicious`. + The hash is also stored on `file_versions.sha256`. +2. `sniff`: the first 512 bytes against the declared content type + (`mime-sniff.ts`). HTML, SVG, PE/ELF/Mach-O, or a shell script served + under a type that does not admit it is `suspicious`. +3. `urlscan`: for HTML files and sites, ``, ` @@ -39,7 +39,13 @@ > {/if} {:else} -
{ + if (report.file) + void markFile( + report.fileId, + report.file.version, + 'malicious' + ); + }}>Quarantine - - { + event.preventDefault(); + const value = new FormData(event.currentTarget).get( + 'resolution' + ); + const resolution = resolutions.find( + (item) => item === value + ); + if (resolution) void resolveReport(report.id, resolution); + }} > + + +
@@ -200,10 +246,18 @@ {#each file.verdicts as verdict (verdict.source)} -
- {verdict.source} - {verdict.verdict} -
+
+ + {verdict.source} + {verdict.verdict} + + 
{JSON.stringify(
+												verdict.details,
+												null,
+												2
+											)}
+
{/each} @@ -211,12 +265,13 @@ markFile(file.id, file.version, 'clean')} + >Clean @@ -356,6 +411,7 @@ SHA-256 Reason diff --git a/apps/web/src/routes/api/admin/files/[id]/+server.ts b/apps/web/src/routes/api/admin/files/[id]/+server.ts index 0d7d7b9..c548a69 100644 --- a/apps/web/src/routes/api/admin/files/[id]/+server.ts +++ b/apps/web/src/routes/api/admin/files/[id]/+server.ts @@ -6,7 +6,8 @@ import { requireAdmin } from '$lib/server/request-auth'; import { Admin } from '$lib/server/services/admin'; const Body = Schema.Struct({ - verdict: Schema.Literals(['clean', 'malicious']) + verdict: Schema.Literals(['clean', 'malicious']), + version: Schema.Int }); // An operator's verdict on a held or quarantined file. @@ -15,12 +16,12 @@ export const PATCH: RequestHandler = (event) => Effect.gen(function* () { const admin = yield* Admin; const auth = yield* requireAdmin(event); - const { verdict } = yield* decodeJson( + const { verdict, version } = yield* decodeJson( event.request, Body, 'A verdict (clean, malicious) is required' ); - yield* admin.markFile(event.params.id, verdict, auth.userId); + yield* admin.markFile(event.params.id, version, verdict, auth.userId); return Response.json({ ok: true as const }); }) ); diff --git a/apps/web/src/routes/report/+server.ts b/apps/web/src/routes/report/+server.ts index 5785d92..8a7dbdd 100644 --- a/apps/web/src/routes/report/+server.ts +++ b/apps/web/src/routes/report/+server.ts @@ -10,7 +10,7 @@ import { ReportCreateSchema, hashReporterIp } from '$lib/server/report-policy'; -import { readBoundedJson } from '$lib/server/request-json'; +import { readBoundedJson, readBoundedText } from '$lib/server/request-json'; import { Admin } from '$lib/server/services/admin'; import { RateLimits } from '$lib/server/services/rate-limits'; @@ -77,33 +77,20 @@ const form = (fileId: string) => { `); }; -const FormBody = Schema.Struct({ - fileId: Schema.String, - reason: Schema.String, - details: Schema.optionalKey(Schema.String) -}); - const readForm = (request: Request) => - Effect.tryPromise({ - try: async (): Promise => { - const text = await request.text(); - if (text.length > MAX_BODY_BYTES) throw new Error('too large'); + readBoundedText(request, { + maxBytes: MAX_BODY_BYTES, + invalidLengthMessage: 'Report is too large', + invalidTextMessage: 'Report form is invalid' + }).pipe( + Effect.map((text) => { const params = new URLSearchParams(text); return { fileId: params.get('fileId') ?? '', reason: params.get('reason') ?? '', ...(params.get('details') ? { details: params.get('details') } : {}) }; - }, - catch: () => - new InvalidRequest({ status: 400, message: 'Report form is invalid' }) - }).pipe( - Effect.flatMap(Schema.decodeUnknownEffect(FormBody)), - Effect.mapError((cause) => - cause instanceof InvalidRequest - ? cause - : new InvalidRequest({ status: 400, message: 'Report form is invalid' }) - ) + }) ); const readReport = (request: Request) => diff --git a/docs/abuse.md b/docs/abuse.md index aec5dd6..70fa493 100644 --- a/docs/abuse.md +++ b/docs/abuse.md @@ -42,12 +42,12 @@ the content domain moves. Tenant content is only ever served from `orgs.trust` (`apps/web/src/lib/server/trust-policy.ts`): -| Level | Reached by | May publish | Scan | -| ------------- | ------------------------------------------------------------ | --------------- | ---------------- | -| `new` | Sign-up | No (403) | n/a | -| `verified` | A sign-in with a verified email | Yes | Before it's live | -| `established` | 14 days on a paid plan (maintenance sweep), or an admin bump | Yes | After it's live | -| `suspended` | The kill switch | No; host is 404 | n/a | +| Level | Reached by | May publish | Scan | +| ------------- | ----------------------------------------------------------------------------- | --------------- | ---------------- | +| `new` | Sign-up | No (403) | n/a | +| `verified` | A sign-in with a verified email | Yes | Before it's live | +| `established` | Org over 14 days old and currently paid (maintenance sweep), or an admin bump | Yes | After it's live | +| `suspended` | The kill switch | No; host is 404 | n/a | "Publish" is anything that makes content public: an upload that lands public (including HTML, which is always public), a visibility change, a @@ -85,18 +85,23 @@ The worst verdict decides: - `clean`: a held row is published (`public = true`, `publish_pending = false`), the edge cache for its URLs is purged, and - the org gets a `published` notification. -- `suspicious`: a held row stays held and the org gets a `held` - notification; a scan-after file stays as it was. Either way the file + a `published` record is written to `notifications`. +- `suspicious`: a held row stays held and a `held` notification record is + written; a scan-after file stays as it was. Either way the file appears under "Held and quarantined files" on `/admin` for a person to mark clean or malicious. - `malicious`: `public = false`, `quarantined = true`, the edge is purged, - and the org gets a `quarantined` notification. Every content route - answers 404 for a quarantined file, the owner cannot republish it, and + and a `quarantined` notification record is written. Every content route + reaching the Worker answers 404 for a quarantined file, the owner cannot republish it, and it stays visible in their dashboard so they can delete it. -Nothing is dropped silently: a verdict row exists for every check that -ran, and every hold or quarantine has a notification row. +Verdicts and notification rows are durable database records. Owners see +"Pending review" or "Quarantined" in their dashboard after refreshing; +there is no notification inbox, email delivery, or automatic status polling +in this layer. Quarantined files retain their metadata and deletion controls, +while preview, sharing, rename, and new-version actions are unavailable. +Notify owners through the operator's support process when direct outreach +is needed. Secrets: `URLSCAN_API_KEY` (an API token with the URL Scanner scope) and `CF_ACCOUNT_ID` (the account that owns it). Locally, `URLSCAN_API_KEY=fake:malicious` @@ -117,9 +122,10 @@ Suspending an org (`/admin` → Suspend, or `PATCH /api/admin/orgs/` with `{ "action": "suspend" }`) does, in one request: 1. `orgs.trust = 'suspended'`. -2. Drops the org's slug from the KV host cache, so `.` - answers 404 on every path from the next request (the cache otherwise - holds for up to five minutes). +2. Attempts to drop the org's slug from the KV host cache. Host requests + reaching the Worker check current trust in Postgres, even when the slug + mapping is cached, and return 404 for a suspended org. A failed KV delete + is logged and does not undo suspension. 3. Refuses every API key and browser session for the org with 401 on their next request. Nothing is revoked; restoring the org brings them back. @@ -130,8 +136,12 @@ with `{ "action": "suspend" }`) does, in one request: Cache → Hostname) or wait for the entries to expire (up to a year for pinned file versions, minutes for sites). +Bytes already cached by a browser cannot be revoked. CDN responses that +bypass the Worker remain available until the purge succeeds or they expire; +suspension is not instant global revocation. + Restore (`{ "action": "restore" }`) sets the org back to `verified` and -drops the cache entry again. Suspending is reversible and touches no +attempts to drop the cache entry again. Suspending is reversible and touches no files, so it is the right first move when in doubt. ## Rate limits @@ -153,14 +163,16 @@ errors lets the request through and logs. files with their verdicts, dead-lettered jobs, recent orgs with trust, plan, and usage, and a form to add a hash to `blocked_hashes`. - A report about live content: open the file on its host, decide. Mark - the file malicious (quarantine, notify the owner) and resolve the - report as `quarantined`; or dismiss. Suspend the org when the account - is the problem rather than one file. + the file malicious to quarantine it, then close the report with the + `quarantined` outcome; or choose `dismissed`. Suspend the org when the + account is the problem and choose `suspended`. Closing a report records + the selected outcome; it does not perform the quarantine or suspension. - A held file (`suspicious`): read the verdict details on `/admin`. Mark clean to publish it, malicious to quarantine it. - A takedown notice: quarantine the file, resolve any report as `removed`, - keep the notice with the file id and the `scan_verdicts` row. The - owner's notification tells them it was taken offline after review. + keep the notice with the file id and the `scan_verdicts` row. The owner + sees "Quarantined" after refreshing the dashboard; contact them separately + with the reason and dispute instructions. - A quarantined file the owner disputes: mark clean. The file goes back to private and the owner decides again; the `admin` verdict row records who cleared it. diff --git a/docs/plans/hosted-product.md b/docs/plans/hosted-product.md index cc31b6e..3136ab7 100644 --- a/docs/plans/hosted-product.md +++ b/docs/plans/hosted-product.md @@ -426,22 +426,22 @@ Goal: one bad tenant cannot take the domain down, and you can stop them in under ### E2 Trust levels -- `orgs.trust`: `new` at signup, `verified` after email verification, `established` after 14 days plus a paid plan or an admin bump, `suspended`. -- Gates: `new` cannot make anything public or publish sites. `verified` can, with lower publish limits and every publish scanned before it goes live. `established` publishes immediately and scans after. -- The free plan is private-only by default. Public sharing is a feature flag on paid plans in stack F, or an admin grant. +- `orgs.trust`: `new` at signup, `verified` after email verification, `established` when the org is over 14 days old and currently on a paid plan, or after an admin bump, `suspended`. +- Gates: `new` cannot make anything public or publish sites. `verified` can, with every publish scanned before it goes live (`RL_PUBLISH` is reserved in this layer). `established` publishes immediately and scans after. +- The free plan starts private and can share publicly after email verification; verified orgs wait for scan approval. ### E3 Scan pipeline -- On visibility change to public, and on site commit, send a `scan` job. Content stays private until the verdict arrives for `new` and `verified` orgs. +- On visibility change to public, and on site commit, send a `scan` job. `new` orgs cannot publish; content from `verified` orgs stays private until the verdict arrives. - Scanner does, in order: sha256 against a known-bad hash list mirrored to R2 nightly, MIME sniff versus declared type, and for HTML and sites a Cloudflare URL Scanner submission with the verdict polled by a delayed re-enqueue. -- `scan_verdicts (file_id, version, verdict, source, details, created_at)`. `malicious` flips visibility back to private, sets `files.quarantined`, and notifies the owner. Never silently drop. +- `scan_verdicts (file_id, version, verdict, source, details, created_at)`. `malicious` flips visibility back to private, sets `files.quarantined`, and writes a durable notification record. Owners see the status after refreshing their dashboard; inbox/email delivery is not implemented in this layer. - Web Risk lookup for outbound links inside published HTML is optional. First 100k calls a month are free, then $0.50 per thousand. - ClamAV in a Cloudflare Container is a later PR on this stack. The pattern is a Container running clamd fed by a presigned R2 URL, standard-1 instance, signatures baked into the image. ### E4 Report and kill switch - Report button on every public file page and site 404 page posts to `/report` on the content origin, rate limited by IP, stored in `reports`. -- Kill switch: `UPDATE orgs SET trust = 'suspended'`. The host gate returns 404 for every path on that slug, the KV org cache is purged, API keys stop resolving, and the edge cache for that host is purged via the Cloudflare API. +- Kill switch: `UPDATE orgs SET trust = 'suspended'`. The host gate checks current trust in Postgres and returns 404 for requests reaching the Worker, the KV org cache is invalidated on a best-effort basis, API keys stop resolving, and the edge cache for that host is purged via the Cloudflare API. CDN/browser bytes already cached require successful purge or expiry and are not instantly revoked. - Reversible from the same admin page. ### E5 Admin surface diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 7805997..3bf5f0e 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -96,6 +96,8 @@ export const DashboardFileSchema = Schema.Struct({ version: Schema.Int, sizeBytes: Schema.Int, public: Schema.Boolean, + quarantined: Schema.Boolean, + publishPending: Schema.Boolean, htmlForcedPublic: Schema.Boolean, createdAt: Schema.String, updatedAt: Schema.String, @@ -411,7 +413,9 @@ export const JobSchema = Schema.Union([ urlScan: Schema.optionalKey( Schema.Struct({ ids: Schema.Array(Schema.String), - attempt: Schema.Int + attempt: Schema.Int, + // Binds delayed results to the persisted scan request. + requestedAt: Schema.optionalKey(Schema.String) }) ) }), From f4270f30956f55ec3a4698699bb554bed4dfd82e Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:14:03 -0700 Subject: [PATCH 08/11] Preserve held content previews and resilient device polling --- README.md | 24 +++++++------------ apps/web/src/lib/components/Dashboard.svelte | 2 +- .../src/lib/components/FileDetailView.svelte | 12 ++++++---- .../src/lib/components/files/FileMenu.svelte | 2 +- .../lib/components/files/FileSidebar.svelte | 2 +- .../lib/components/files/VersionList.svelte | 2 +- apps/web/src/lib/server/file-content-link.ts | 2 +- apps/web/src/lib/server/services/scanner.ts | 16 +++++++++++-- .../web/src/lib/server/services/sites/read.ts | 5 +++- .../routes/api/auth/device/token/+server.ts | 13 +++++++--- apps/web/wrangler.jsonc | 5 ++-- docs/abuse.md | 24 +++++++++++++++++-- docs/plans/hosted-product.md | 4 ++-- packages/cli/src/commands/auth.ts | 22 +++++++++++++---- 14 files changed, 94 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f490d64..90d60f1 100644 --- a/README.md +++ b/README.md @@ -120,22 +120,14 @@ __VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS="siva.otter-hawksbill.ts.net,.100.100.40. The content proxy preserves the tenant Host header, so the Worker still enforces its host-routing rules in development. -Production passcode login creates a seven-day, host-only -`__Host-adrive-session` cookie. That cookie is always `Secure`, `HttpOnly`, and -`SameSite=Strict`, so browsers correctly refuse it on plain-HTTP non-localhost -dev URLs (like a Tailscale hostname). Use the generated API key in the -dashboard's “Local HTTP fallback” there. Passcode login works on -`http://localhost` and on any HTTPS dashboard origin. - -Authentication bootstrap endpoints are protected by the `AUTH_GUARD` Workers -KV namespace. Login is limited to ten attempts per client every five minutes, -and five incorrect passcodes within fifteen minutes lock that client out for -thirty minutes. Device authorization creation is limited to five requests per -client every ten minutes, while device polling permits 150 requests per client -every ten minutes so the documented five-second polling interval remains valid. -Rate-limit and lockout responses use HTTP 429 with `Retry-After`. KV is -eventually consistent, so these controls are abuse mitigation rather than an -atomic global security boundary. +The dashboard signs in through WorkOS. Device authorization creation and token +polling share the `RL_AUTH` Workers rate limit: 30 requests per client address +in a 60-second window. A device polls every five seconds, leaving room for +creation and another device behind the same address. Refused token polls return +HTTP 429 with `{ "status": "slow_down" }` and `Retry-After: 60`; the CLI waits +for that delay before polling again. Creation remains rate limited and returns +the usual error response. These counters are approximate and scoped to each +Cloudflare location. `AUTH_GUARD` KV now supports caches, not auth counters. In another shell, configure and exercise the CLI: diff --git a/apps/web/src/lib/components/Dashboard.svelte b/apps/web/src/lib/components/Dashboard.svelte index 9229048..5abb5e7 100644 --- a/apps/web/src/lib/components/Dashboard.svelte +++ b/apps/web/src/lib/components/Dashboard.svelte @@ -158,7 +158,7 @@ session.token, files.list.current.contentOrigin ); - if (file.public) { + if (file.public || file.kind === 'site') { window.open(url, '_blank', 'noopener'); } else { download(url, file.displayName); diff --git a/apps/web/src/lib/components/FileDetailView.svelte b/apps/web/src/lib/components/FileDetailView.svelte index 607e632..6917fd9 100644 --- a/apps/web/src/lib/components/FileDetailView.svelte +++ b/apps/web/src/lib/components/FileDetailView.svelte @@ -161,7 +161,7 @@ const linkFor = async (version?: number) => { const current = detail.current; - if (!current) return { url: '', expiresAt: null }; + if (!current) return { url: '', expiresAt: null, public: false }; if (current.file.quarantined) { throw new Error('Sharing is unavailable while this file is quarantined'); } @@ -172,9 +172,13 @@ const unavailable = current.file.deletedAt !== null || (Number.isFinite(expirationTime) && expirationTime <= Date.now()); + // Historical bytes have their own scan clearance. Resolve those + // through the API so an owner grant can cover an unreviewed version. + const currentVersion = + version === undefined || version === current.file.version; const link = - current.file.public && !unavailable - ? { url: fileUrl(version), expiresAt: null } + current.file.public && !unavailable && currentVersion + ? { url: fileUrl(version), expiresAt: null, public: true } : getContentLink( session.token, current.file.id, @@ -198,7 +202,7 @@ const fileId = current.file.id; const link = await linkFor(version); if (detail.current?.file.id !== fileId) return; - if (current.file.public) { + if (link.public || current.file.kind === 'site') { window.open(link.url, '_blank', 'noopener'); } else { const anchor = document.createElement('a'); diff --git a/apps/web/src/lib/components/files/FileMenu.svelte b/apps/web/src/lib/components/files/FileMenu.svelte index e0955aa..5331a66 100644 --- a/apps/web/src/lib/components/files/FileMenu.svelte +++ b/apps/web/src/lib/components/files/FileMenu.svelte @@ -46,7 +46,7 @@ disabled={file.quarantined} onclick={onopen} > - {file.public ? 'Open' : 'Download'} + {file.public || file.kind === 'site' ? 'Open' : 'Download'} - {file.public ? 'Open' : 'Download'} + {file.public || file.kind === 'site' ? 'Open' : 'Download'}
diff --git a/apps/web/src/lib/components/files/VersionList.svelte b/apps/web/src/lib/components/files/VersionList.svelte index 710b002..5b70317 100644 --- a/apps/web/src/lib/components/files/VersionList.svelte +++ b/apps/web/src/lib/components/files/VersionList.svelte @@ -60,7 +60,7 @@ disabled={file.quarantined} onclick={() => onopen(version.version)} > - {file.public ? 'Open' : 'Download'} + {file.public || file.kind === 'site' ? 'Open' : 'Download'} {/if}
diff --git a/apps/web/src/lib/server/file-content-link.ts b/apps/web/src/lib/server/file-content-link.ts index cc66fd4..e30abaf 100644 --- a/apps/web/src/lib/server/file-content-link.ts +++ b/apps/web/src/lib/server/file-content-link.ts @@ -73,7 +73,7 @@ export const resolveFileContentLink = ( message: 'Only the current site version can be previewed' }); } - if (!includeUnavailable && !requireGrant) { + if (resolved.file.public && !includeUnavailable && !requireGrant) { return { url: new URL( `/s/${encodeURIComponent(resolved.file.id)}/`, diff --git a/apps/web/src/lib/server/services/scanner.ts b/apps/web/src/lib/server/services/scanner.ts index 230cf93..86a9fdf 100644 --- a/apps/web/src/lib/server/services/scanner.ts +++ b/apps/web/src/lib/server/services/scanner.ts @@ -515,12 +515,24 @@ const makeScanner = Effect.gen(function* () { ); const submitted = ids.flatMap((id) => (id === null ? [] : [id])); if (submitted.length < ids.length) { - // A link that could not be submitted is not cleared. - yield* record(row.id, job.version, 'urlscan', 'suspicious', { + // Keep submission failures separate from poll results: successful + // submissions can still reveal malicious links, while clean results + // cannot clear the links the provider never accepted. + yield* record(row.id, job.version, 'urlscan-submit', 'suspicious', { links: [...links], submitted: submitted.length, failed: ids.length - submitted.length }); + } else { + // A later explicit scan can recover from a provider outage. Only a + // complete submission supersedes the earlier incomplete attempt. + yield* sql` + DELETE FROM scan_verdicts + WHERE file_id = ${row.id} AND org_id = ${org.id} + AND version = ${job.version} AND source = 'urlscan-submit' + `.pipe(storageError('clear recovered URL submissions')); + } + if (submitted.length === 0) { return { _tag: 'Settled' as const, verdict: yield* finalize(row, job.version) diff --git a/apps/web/src/lib/server/services/sites/read.ts b/apps/web/src/lib/server/services/sites/read.ts index 35d719c..29bcf11 100644 --- a/apps/web/src/lib/server/services/sites/read.ts +++ b/apps/web/src/lib/server/services/sites/read.ts @@ -27,6 +27,8 @@ export const readOps = ( message: 'Site asset path is unsafe' }) }); + // Only signed owner routes set this; the caller verifies the bound + // grant before serving bytes. Held sites remain private anonymously. const includeUnavailable = options.includeUnavailable === true; const pinVersion = options.version !== undefined; // Content requests carry the org their host names, so a site id @@ -38,7 +40,8 @@ export const readOps = ( FROM files f JOIN site_assets a ON a.file_id = f.id AND a.version = f.current_version - WHERE f.id = ${fileId} AND f.is_site = true AND f.public = true + WHERE f.id = ${fileId} AND f.is_site = true + AND (f.public = true OR ${includeUnavailable}::boolean) AND f.quarantined = false AND (${orgId}::text IS NULL OR f.org_id = ${orgId}) AND ( diff --git a/apps/web/src/routes/api/auth/device/token/+server.ts b/apps/web/src/routes/api/auth/device/token/+server.ts index 50c5d14..9e85869 100644 --- a/apps/web/src/routes/api/auth/device/token/+server.ts +++ b/apps/web/src/routes/api/auth/device/token/+server.ts @@ -1,7 +1,7 @@ import { DeviceTokenRequestSchema } from '@adrive/shared'; import type { RequestHandler } from './$types'; import { Effect } from 'effect'; -import { rateLimitResponse } from '$lib/server/auth-rate-limit-response'; +import { RATE_LIMIT_PERIOD_SECONDS } from '$lib/server/auth-rate-limit-response'; import { runEdge } from '$lib/server/edge'; import { decodeJson } from '$lib/server/request-json'; import { Auth } from '$lib/server/services/auth'; @@ -14,8 +14,15 @@ export const POST: RequestHandler = ({ request, getClientAddress }) => const rateLimits = yield* RateLimits; const rateLimit = yield* rateLimits.auth(getClientAddress()); if (!rateLimit.allowed) { - return rateLimitResponse( - 'Too many authentication requests. Try again later.' + return Response.json( + { status: 'slow_down' }, + { + status: 429, + headers: { + 'Cache-Control': 'private, no-store', + 'Retry-After': String(RATE_LIMIT_PERIOD_SECONDS) + } + } ); } const input = yield* decodeJson( diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index 4ef8e98..cecd9a3 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -63,7 +63,8 @@ { "name": "RL_AUTH", "namespace_id": "1003", - "simple": { "limit": 10, "period": 60 } + // Shared creation + polling budget; a device polls 12 times/minute. + "simple": { "limit": 30, "period": 60 } }, { "name": "RL_ANON", @@ -194,7 +195,7 @@ { "name": "RL_AUTH", "namespace_id": "2003", - "simple": { "limit": 10, "period": 60 } + "simple": { "limit": 30, "period": 60 } }, { "name": "RL_ANON", diff --git a/docs/abuse.md b/docs/abuse.md index 70fa493..f1a145c 100644 --- a/docs/abuse.md +++ b/docs/abuse.md @@ -65,6 +65,21 @@ public gets a `scan` job on the queue. For a verified org the row stays private with `files.publish_pending = true` until the scan clears it; for an established org it is public at once and scanned after. +Each version write records its scan obligation in Postgres before enqueueing. +The lifecycle sweep requeues overdue obligations in bounded batches, so a queue +outage cannot silently lose the initial scan. A completed suspicious verdict +ends automatic scanning and leaves the file for review. When only some URL +submissions succeed, their results are still collected; failed submissions +keep a suspicious minimum verdict, and any malicious result quarantines. + +Publishing the current version does not expose unscanned older bytes. Older +versions need their own completed clean scan or operator clearance for anonymous +access. Owners can still request signed links for pending/private versions; +a malicious version stays blocked even with an existing grant. Clearing the +current version does not clear older versions. Admin actions include the +displayed current version and reject an outdated overview with a +refresh-required response. + Checks, in order, each writing one `scan_verdicts` row per `(file, version, source)`: @@ -149,10 +164,15 @@ files, so it is the right first move when in doubt. Workers rate limit bindings (`wrangler.jsonc` `ratelimits`, per colo, approximate, 60 second windows): `RL_UPLOAD` 60/min per org (uploads, site sessions), `RL_PUBLISH` 10/min per org (reserved for publish -counting), `RL_AUTH` 10/min per client address (device auth), `RL_ANON` +counting), `RL_AUTH` 30/min per client address (shared device creation and +token polling), `RL_ANON` 300/min per client address (content fetches past the edge cache, and reports). A refused request is 429 with `Retry-After: 60`. A binding that -errors lets the request through and logs. +errors lets the request through and logs. The auth budget accommodates the +advertised five-second polling interval. Refused token polls return +`{ "status": "slow_down" }`; the CLI honors `Retry-After` before polling +again. Device creation still consumes the shared limit and returns a normal +error response when refused. ## Operating diff --git a/docs/plans/hosted-product.md b/docs/plans/hosted-product.md index 3136ab7..02a8681 100644 --- a/docs/plans/hosted-product.md +++ b/docs/plans/hosted-product.md @@ -415,12 +415,12 @@ Goal: one bad tenant cannot take the domain down, and you can stop them in under "ratelimits": [ { "name": "RL_UPLOAD", "namespace_id": "1001", "simple": { "limit": 60, "period": 60 } }, { "name": "RL_PUBLISH", "namespace_id": "1002", "simple": { "limit": 10, "period": 60 } }, - { "name": "RL_AUTH", "namespace_id": "1003", "simple": { "limit": 10, "period": 60 } }, + { "name": "RL_AUTH", "namespace_id": "1003", "simple": { "limit": 30, "period": 60 } }, { "name": "RL_ANON", "namespace_id": "1004", "simple": { "limit": 300, "period": 60 } } ] ``` -- Keyed by `orgId` for upload and publish, by `userId` for auth, by client IP for anonymous content fetches. +- Keyed by `orgId` for upload and publish, by client IP for device auth and anonymous content fetches. Device creation and five-second token polling share the 30/min auth budget; refused polls return `slow_down` with `Retry-After: 60`. - Replaces the KV-based `auth-guard.ts` counters. Passcode lockout goes away with the passcode. - Bindings are per-colo and approximate. That is fine for abuse, not for billing. diff --git a/packages/cli/src/commands/auth.ts b/packages/cli/src/commands/auth.ts index 39fe930..89e5da0 100644 --- a/packages/cli/src/commands/auth.ts +++ b/packages/cli/src/commands/auth.ts @@ -135,8 +135,9 @@ export const login = Command.make( } let apiKey = ''; + let pollDelaySeconds = authorization.interval; while (!apiKey) { - yield* Effect.sleep(`${authorization.interval} seconds`); + yield* Effect.sleep(`${pollDelaySeconds} seconds`); const pollResponse = yield* Effect.tryPromise({ try: async () => { const response = await fetch(`${endpoint}/api/auth/device/token`, { @@ -148,7 +149,15 @@ export const login = Command.make( }); const body: unknown = await response.json(); if (response.status === 202 || response.status === 429) { - return { kind: 'pending' as const, body }; + const retryAfter = Number(response.headers.get('Retry-After')); + return { + kind: 'pending' as const, + body, + retryAfterSeconds: + Number.isFinite(retryAfter) && retryAfter >= 0 + ? retryAfter + : 0 + }; } if (response.ok) return { kind: 'complete' as const, body }; throw new Error(`Device authorization failed (${response.status})`); @@ -189,8 +198,13 @@ export const login = Command.make( ); if ('apiKey' in poll) { apiKey = poll.apiKey; - } else if (poll.status === 'slow_down') { - yield* Effect.sleep(`${authorization.interval} seconds`); + } else if (pollResponse.kind === 'pending') { + // A shared limiter can need longer than this device's own + // interval. Honor its delay before issuing another poll. + pollDelaySeconds = Math.max( + authorization.interval * (poll.status === 'slow_down' ? 2 : 1), + pollResponse.retryAfterSeconds + ); } } // Record the deployment's content origin so later commands can From 2a5cbdcb82d367dd40631feb303612ab96962965 Mon Sep 17 00:00:00 2001 From: Ben Davis <45952064+bmdavis419@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:26:54 -0700 Subject: [PATCH 09/11] Fence scan persistence and decode outbound HTML attributes --- apps/web/package.json | 1 + apps/web/src/lib/server/html-links.ts | 18 +-- apps/web/src/lib/server/services/scanner.ts | 160 ++++++++++++++++---- bun.lock | 1 + 4 files changed, 137 insertions(+), 43 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index 7589b52..2578c89 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -20,6 +20,7 @@ "@modelcontextprotocol/server": "2.0.0", "agents": "^0.20.1", "effect": "4.0.0-beta.102", + "entities": "4.5.0", "markdown-it": "^14.3.0", "mime": "^4.1.0", "pg": "^8.22.0", diff --git a/apps/web/src/lib/server/html-links.ts b/apps/web/src/lib/server/html-links.ts index 6af2f93..1b26cb5 100644 --- a/apps/web/src/lib/server/html-links.ts +++ b/apps/web/src/lib/server/html-links.ts @@ -1,3 +1,5 @@ +import { decodeHTMLAttribute } from 'entities'; + // The outbound targets inside a published HTML document: ``, // `