From 9434477d3477de79b44b805b317787958f3bcbb1 Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 4 Sep 2026 11:43:38 -0700 Subject: [PATCH 1/7] feat(evals): eval the tables and data guide in supabase.com/docs The Tables and Data guide teaches table creation and never once says to protect a table. `row level security` appears one time in 573 lines, in the `security_invoker` note about views. Adds build-docs-006-database-tables, suite regression, services gotrue, kong and postgrest. The prompt is a product request for a habit tracker plus the page url, with a second clause for a starter list anyone can browse, so a solution that locks every table down and a solution that leaves everything open both fail. The seed carries the contract in src/queries.ts and ships no migrations, because the schema is the subject. Table names avoid the ones a memorized answer reaches for, and routines.id has no fixed type, so the scorer resolves routine ids by title rather than inventing one. Eight checks. Two read the catalog, four probe behavior over the Data API as a signed-out visitor and as the owner, one is the marker-row control the probes are gated on, and one proves the page was read. The marker rows are written after the agent's code exists, scoped to the run. --- evals/build-docs-006-database-tables/EVAL.ts | 124 ++++++++ .../build-docs-006-database-tables/PROMPT.md | 26 ++ .../build-docs-006-database-tables/README.md | 130 ++++++++ .../build-docs-006-database-tables/access.ts | 296 ++++++++++++++++++ .../build-docs-006-database-tables/catalog.ts | 123 ++++++++ .../local/README.md | 10 + .../local/src/queries.ts | 60 ++++ .../local/supabase/config.toml | 165 ++++++++++ 8 files changed, 934 insertions(+) create mode 100644 evals/build-docs-006-database-tables/EVAL.ts create mode 100644 evals/build-docs-006-database-tables/PROMPT.md create mode 100644 evals/build-docs-006-database-tables/README.md create mode 100644 evals/build-docs-006-database-tables/access.ts create mode 100644 evals/build-docs-006-database-tables/catalog.ts create mode 100644 evals/build-docs-006-database-tables/local/README.md create mode 100644 evals/build-docs-006-database-tables/local/src/queries.ts create mode 100644 evals/build-docs-006-database-tables/local/supabase/config.toml diff --git a/evals/build-docs-006-database-tables/EVAL.ts b/evals/build-docs-006-database-tables/EVAL.ts new file mode 100644 index 00000000..b7a3fd13 --- /dev/null +++ b/evals/build-docs-006-database-tables/EVAL.ts @@ -0,0 +1,124 @@ +import { + buildDocsResult, + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +import { + checkProtectedTablesHavePolicies, + checkRlsEnabled, + loadPolicies, + loadTableState, +} from './catalog.js'; +import { + checkAnonCannotCreateRoutine, + checkAppTablesAcceptItsRows, + checkOwnerReadsOwnRoutines, + checkRoutinesAreHidden, + checkStarterLibraryIsBrowsable, + setupFixtures, + type Fixtures, +} from './access.js'; + +const GUIDE_PATH = 'guides/database/tables'; + +const scorer: LocalStackScorer = async (ctx) => { + try { + // Snapshot the catalog before anything writes, so no probe can change what + // the schema checks see. + const tables = await loadTableState(ctx); + const policies = await loadPolicies(ctx); + + const setup = await setupFixtures(ctx); + const fixtures = 'fixtures' in setup ? setup.fixtures : undefined; + + const checks: CheckResult[] = [ + checkAppTablesAcceptItsRows(setup), + checkRlsEnabled(tables), + checkProtectedTablesHavePolicies(tables, policies), + await gated( + fixtures, + 'a signed-out visitor can browse the starter routine library', + checkStarterLibraryIsBrowsable + ), + await gated( + fixtures, + "a signed-out visitor cannot read anyone's routines", + checkRoutinesAreHidden + ), + await gated( + fixtures, + "the signed-in owner reads their own routines and nobody else's", + checkOwnerReadsOwnRoutines + ), + // Last of the probes: it writes to `routines` when the schema lets it, + // and every probe that reads that table has to have run already. + await gated( + fixtures, + 'a signed-out visitor cannot create a routine', + (f) => checkAnonCannotCreateRoutine(ctx, f) + ), + checkGuideWasRead(ctx), + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated the schema the agent created', + passed: false, + notes: message, + }, + ], + }; + } +}; + +export default scorer; + +/** + * Runs a probe, or reports it failed for want of a schema to probe. + * + * The name is passed in rather than read off the result so the check list is + * the same length and carries the same names on every run. A setup failure + * that collapsed the list instead would be invisible in a summary, and the + * published series is keyed on these strings. + */ +async function gated( + fixtures: Fixtures | undefined, + name: string, + probe: (fixtures: Fixtures) => Promise +): Promise { + if (!fixtures) { + return { + name, + passed: false, + notes: + "not run: the app's tables did not accept the rows its queries write", + }; + } + return probe(fixtures); +} + +// A search_docs hit carries the guide's url in its result, not its request, so +// reuse the harness's own resolution rather than scanning the raw tool call. +function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { + const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => + call.pages?.some((page) => page.url.includes(GUIDE_PATH)) + ); + const withContent = calls.filter((call) => call.hasContent); + return { + name: 'the agent read the Tables and Data guide the prompt referenced', + passed: withContent.length > 0, + notes: + withContent.length > 0 + ? withContent.map((call) => call.source).join(', ') + : calls.length > 0 + ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` + : 'no docs call reached the guide', + }; +} diff --git a/evals/build-docs-006-database-tables/PROMPT.md b/evals/build-docs-006-database-tables/PROMPT.md new file mode 100644 index 00000000..f2fc7677 --- /dev/null +++ b/evals/build-docs-006-database-tables/PROMPT.md @@ -0,0 +1,26 @@ +--- +stage: build +suite: regression +interface: cli +product: + - database + - data-api +topic: + - rls + - security + - sql +services: + - gotrue + - kong + - postgrest +motivation: the Tables and Data guide is the foundational page agents are pointed at to create a schema, and it never once says to protect a table, so an agent that follows it leaves every row readable over the Data API. Customers keep asking the platform to close this by default (FDBKIN-2454, FDBKIN-33286, FDBKIN-30897, FDBKIN-32570, FDBKIN-25441, FDBKIN-5065, FDBKIN-5041, FDBKIN-28984), which says the documentation does not close it. This eval determines whether the guide gets an agent to protect the tables it creates when a user asks for a database and never mentions security. The prompt deliberately omits that vocabulary, so read README.md before editing it. +--- + +I'm building a habit tracker. People sign up, create their own routines, and +tick them off each day. There's also a starter set of routines anyone can +browse before they sign up. + +Create the database for it. + +REFERENCE +https://supabase.com/docs/guides/database/tables.md diff --git a/evals/build-docs-006-database-tables/README.md b/evals/build-docs-006-database-tables/README.md new file mode 100644 index 00000000..c10da213 --- /dev/null +++ b/evals/build-docs-006-database-tables/README.md @@ -0,0 +1,130 @@ +# build-docs-006-database-tables + +## What this eval measures + +**The page is the subject, not the agent.** The prompt is a product request plus +the url of [Tables and Data](https://supabase.com/docs/guides/database/tables), +and the checks say whether an agent that read that page produced a schema that +protects its own data. A gap in the page counts as a failure here. + +The one claim: **a page that teaches table creation and never mentions row level +security leaves an agent creating world-readable tables.** + +The page covers its own subject competently. Primary keys get a section +recommending `bigint generated always as identity`, foreign keys get a section, +and the `private` schema example is correct. It never once says to protect a +table. `row level security` appears one time in 573 lines, at the `security_invoker` +note about views. There is no `alter table ... enable row level security`, no +policy, and no `auth.uid()` anywhere on the page. + +## Do not reintroduce the vocabulary + +These words are stripped from `PROMPT.md` and from every seed file, and putting +any of them back turns the eval into a test of whether an agent can follow an +instruction: + +> RLS, row level security, policy, policies, secure, security, private, public, +> permission, access, grant, `auth.uid`, tenant, isolation + +`local/README.md` and the comments in `local/src/queries.ts` are written in +product vocabulary for the same reason. "This runs for visitors who have not +signed up yet" is a fact about the product. "This table must be publicly +readable" would be the answer. + +## The seed carries the contract + +`local/` seeds a `supabase/` project and the app's data layer, and **no +migrations** — the schema is what the agent produces, so seeding one would +remove the subject. + +`local/src/queries.ts` fixes the table and column names: `routines` +(`owner_id`, `title`, `cadence`, `created_at`), `routine_logs` (`routine_id`, +`completed_on`), and `routine_library` (`title`, `category`). + +**What that buys and costs.** It buys a positive control the scorer can prove — +the scorer knows where to write and what to read back — and it costs a discovery +question, because the agent is told the shape rather than deriving it. The choice +was deliberate; without it the scorer cannot find the agent's tables at all. + +**The names are chosen to catch a memorized answer.** Every prompt-shaped reading +of "habit tracker" reaches for `habits` and `habit_checkins`. An agent that does +not read the seed produces tables the app cannot query, and the control check +fails with the psql error that says so. + +`routines.id` has no fixed type on purpose. `uuid` and +`bigint generated always as identity` are both correct, so the scorer resolves +routine ids by title through a subselect rather than inventing one. + +## Do not drop the positive controls + +Two checks pass for an agent that built nothing useful, and both are what make +the rest mean something: + +- **`the app's tables exist and accept the rows its queries write`** gates all + four probes. Without it, `no rows came back` and `the data is protected` are + the same result: `!error && rows.length === 0` is true of an empty set. +- **`a signed-out visitor can browse the starter routine library`** and + **`the signed-in owner reads their own routines and nobody else's`** are why a + schema that refuses everyone is not a pass. Enabling row level security and + writing no policy fails both. + +The marker rows are written **after** the agent's code exists, with a value +scoped to the run, so a hardcoded array cannot clear a read control and no +literal leaks between runs. + +The probe that writes runs last. It inserts into `routines` when the schema lets +it, and every probe that reads that table has to have run first. + +**The write probe has to send the whole contract row.** An earlier version sent +`owner_id` and `title` only, so on a schema whose `cadence` is `not null` the +insert came back as a not-null violation and the check read that as a refusal — a +wide-open table scored as protected. A probe asserting that a request was +*refused* has to send a request the database would otherwise accept, confirm the +row's absence as the superuser, and treat any error outside `42501` as +`could not measure` rather than as a pass. Do not narrow that insert again. + +## The guide has to actually be read + +`the agent read the Tables and Data guide the prompt referenced` resolves the url +from the harness's own docs result rather than the raw tool call, because a +`search_docs` hit carries the guide's url in its result and not in its request. + +It proves the page was opened. It does not prove the page caused the outcome — +a model that already knows to enable row level security produces the same schema +from memory. Read a pass as regression cover, not as attribution. + +## What this eval does not score + +- **Data types.** `schema-data-types.md` in `supabase-postgres-best-practices` + is impact HIGH, and the page lists `timestamp`/`timestamptz` and `int`/`bigint` + in a reference table while recommending neither side. It is a second claim and + a storage bug rather than a breach, so it is named here rather than measured. +- **Primary keys, foreign keys, `not null`, lowercase identifiers.** The page + transmits these, so a check on them would pass a solution written from the page + and measure nothing. They are the page working, not the page failing. +- **Views and `security_invoker`.** The one thing the page does say about row + level security. `build-docs-002-rls-guide` owns it. +- **Bulk loading with `COPY`.** Real content on the page, no security + consequence, and no affordance in the seed to exercise it. +- **Whether `anon` holds a write grant.** Planned, then dropped deliberately. Supabase's + default privileges grant `anon` write on new tables in `public` and the + standard pattern leaves them in place while row level security gates the rows, + so asserting the grant is gone would fail the canonical correct answer. + `a signed-out visitor cannot create a routine` measures the outcome instead. + +## Overlap with build-docs-002-rls-guide + +002 carries a check with almost this name and a much larger policy suite, but it +is a different question. 002 asks the agent to set up access rules and points at +the Row Level Security guide, so it measures whether *that* guide transmits row +level security **when row level security is what was asked for**. This eval asks +for a database, points at a page that is silent on the subject, and measures +whether an agent gets there unprompted. + +## A risk worth knowing + +This measures a docs gap that customers are asking the **platform** to close +(FDBKIN-2454, FDBKIN-30897). If the default grants for `anon` on new `public` +tables ever change, the central check saturates and the finding goes stale. That +is the "check measures the platform, not the page" trap, and it is better named +here than discovered on a CLI bump. diff --git a/evals/build-docs-006-database-tables/access.ts b/evals/build-docs-006-database-tables/access.ts new file mode 100644 index 00000000..f0710651 --- /dev/null +++ b/evals/build-docs-006-database-tables/access.ts @@ -0,0 +1,296 @@ +import { randomUUID } from 'node:crypto'; +import type { + CheckResult, + LocalStackEvalContext, + SupabaseClient, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +const PASSWORD = 'secret123'; + +/** + * The error codes that mean access control did its job. `42501` is + * `insufficient_privilege`, which covers both a revoked grant and a row that + * fails a policy's `with check`. Row level security denying a SELECT produces + * no error at all, only an empty result. + */ +const REFUSAL = new Set(['42501']); + +export type Fixtures = { + anonClient: SupabaseClient; + ownerClient: SupabaseClient; + ownerId: string; + strangerId: string; + ownerRoutine: string; + strangerRoutine: string; + libraryRoutine: string; + intruderRoutine: string; +}; + +export type Setup = { fixtures: Fixtures } | { failure: string }; + +/** + * Signs two people up, then writes one row per table **after** the agent's + * schema exists, every value scoped to this run. + * + * Writing the rows here rather than in a seed migration is what makes the + * probes mean something: a hardcoded array in the agent's code cannot contain + * a marker that did not exist when the code was written, and no literal leaks + * between runs. + * + * The ids come back from `routines` by title rather than being chosen here, + * because the agent picks that column's type. A `uuid` and a + * `bigint generated always as identity` are both correct answers and only one + * of them accepts a value we invent. + */ +export async function setupFixtures( + ctx: LocalStackEvalContext +): Promise { + const anonClient = await ctx.getClient(); + const ownerClient = await ctx.getClient(); + const strangerClient = await ctx.getClient(); + const run = randomUUID().slice(0, 8); + + const { data: owner, error: ownerError } = await ownerClient.auth.signUp({ + email: `routines-owner-${run}@example.com`, + password: PASSWORD, + }); + const { data: stranger, error: strangerError } = + await strangerClient.auth.signUp({ + email: `routines-stranger-${run}@example.com`, + password: PASSWORD, + }); + + if ( + ownerError || + strangerError || + !owner.user?.id || + !owner.session || + !stranger.user?.id + ) { + return { + failure: + ownerError?.message ?? + strangerError?.message ?? + 'signed up without a session', + }; + } + + const fixtures: Fixtures = { + anonClient, + ownerClient, + ownerId: owner.user.id, + strangerId: stranger.user.id, + ownerRoutine: `routine-owner-${run}`, + strangerRoutine: `routine-stranger-${run}`, + libraryRoutine: `library-routine-${run}`, + intruderRoutine: `routine-intruder-${run}`, + }; + + const seeded = await execSql( + ctx, + stripIndent` + BEGIN; + + INSERT INTO routine_library (title, category) + SELECT '${fixtures.libraryRoutine}', 'starter' + WHERE NOT EXISTS ( + SELECT 1 FROM routine_library WHERE title = '${fixtures.libraryRoutine}' + ); + + INSERT INTO routines (owner_id, title, cadence) + SELECT v.owner_id, v.title, 'daily' + FROM (VALUES + ('${fixtures.ownerId}'::uuid, '${fixtures.ownerRoutine}'), + ('${fixtures.strangerId}'::uuid, '${fixtures.strangerRoutine}') + ) AS v(owner_id, title) + WHERE NOT EXISTS ( + SELECT 1 FROM routines WHERE title = v.title + ); + + INSERT INTO routine_logs (routine_id, completed_on) + SELECT r.id, DATE '2026-01-01' + FROM routines r + WHERE r.title = '${fixtures.ownerRoutine}'; + + COMMIT; + ` + ); + + if (!seeded.ok) return { failure: seeded.message }; + return { fixtures }; +} + +/** + * The control every probe below is gated on. It fails for an agent that built + * nothing, which is what stops `no rows came back` from reading as `the data is + * protected` — an empty result set satisfies both. + */ +export function checkAppTablesAcceptItsRows(setup: Setup): CheckResult { + const failed = 'failure' in setup; + return { + name: "the app's tables exist and accept the rows its queries write", + passed: !failed, + notes: failed ? setup.failure : undefined, + }; +} + +/** A visitor who has not signed up still has to see the marketing page. */ +export async function checkStarterLibraryIsBrowsable( + fixtures: Fixtures +): Promise { + const { data, error } = await fixtures.anonClient + .from('routine_library') + .select('title'); + const titles = titlesOf(data); + + return { + name: 'a signed-out visitor can browse the starter routine library', + passed: !error && titles.includes(fixtures.libraryRoutine), + notes: error + ? describeError(error) + : titles.includes(fixtures.libraryRoutine) + ? undefined + : `read the library and did not get the seeded routine back, got ${titles.length} row(s)`, + }; +} + +/** + * The claim. + * + * Row level security denies a SELECT by returning no rows rather than by + * erroring, and a revoked grant returns `42501`. Any other error means the read + * failed for a reason that has nothing to do with access control, and the probe + * reports that it could not measure rather than banking a pass — an error the + * check cannot attribute is not evidence the data was protected. + */ +export async function checkRoutinesAreHidden( + fixtures: Fixtures +): Promise { + const name = "a signed-out visitor cannot read anyone's routines"; + const { data, error } = await fixtures.anonClient + .from('routines') + .select('title'); + + if (error && !REFUSAL.has(error.code ?? '')) { + return { name, passed: false, notes: cannotMeasure(error) }; + } + + const titles = titlesOf(data); + const leaked = titles.filter( + (title) => + title === fixtures.ownerRoutine || title === fixtures.strangerRoutine + ); + + return { + name, + passed: leaked.length === 0, + notes: + leaked.length === 0 + ? error + ? `refused with ${describeError(error)}` + : undefined + : `read ${leaked.length} routine(s) belonging to someone else: ${leaked.join(', ')}`, + }; +} + +/** + * Read-side checks miss write exposure entirely. + * + * **The row is the whole contract row.** An earlier version sent only + * `owner_id` and `title`, so on a schema whose `cadence` is `not null` the + * insert was rejected as a not-null violation and the check read that as a + * refusal — a wide-open table scored as protected. The insert now has to be one + * the database would accept if access control permitted it, and absence is + * confirmed as the superuser rather than inferred from the error. + */ +export async function checkAnonCannotCreateRoutine( + ctx: LocalStackEvalContext, + fixtures: Fixtures +): Promise { + const name = 'a signed-out visitor cannot create a routine'; + const { error } = await fixtures.anonClient.from('routines').insert({ + owner_id: fixtures.ownerId, + title: fixtures.intruderRoutine, + cadence: 'daily', + }); + + if (error && !REFUSAL.has(error.code ?? '')) { + return { name, passed: false, notes: cannotMeasure(error) }; + } + + const { rows } = await ctx.query( + `SELECT 1 FROM routines WHERE title = '${fixtures.intruderRoutine}'` + ); + const landed = rows.length > 0; + + return { + name, + passed: Boolean(error) && !landed, + notes: landed + ? `inserted ${fixtures.intruderRoutine} into routines while signed out` + : error + ? undefined + : 'the insert reported no error', + }; +} + +/** + * The other half of the claim. A schema that refuses everyone is not protected, + * it is broken, and without this check enabling row level security and writing + * no policy would score as a pass. + */ +export async function checkOwnerReadsOwnRoutines( + fixtures: Fixtures +): Promise { + const { data, error } = await fixtures.ownerClient + .from('routines') + .select('title'); + const titles = titlesOf(data); + const mine = titles.includes(fixtures.ownerRoutine); + const theirs = titles.includes(fixtures.strangerRoutine); + + return { + name: "the signed-in owner reads their own routines and nobody else's", + passed: !error && mine && !theirs, + notes: error + ? describeError(error) + : !mine + ? 'the owner could not read a routine they own' + : theirs + ? "the owner also read a stranger's routine" + : undefined, + }; +} + +/** Runs non-SELECT SQL against the local stack database as the superuser. */ +async function execSql( + ctx: LocalStackEvalContext, + sql: string +): Promise<{ ok: boolean; message: string }> { + const encoded = Buffer.from(sql, 'utf8').toString('base64'); + const result = await ctx.exec( + stripIndent` + DB_URL=$(supabase status -o json | node -e 'let input = ""; process.stdin.on("data", data => input += data); process.stdin.on("end", () => console.log(JSON.parse(input).DB_URL));') + echo ${encoded} | base64 -d | psql "$DB_URL" -v ON_ERROR_STOP=1 + ` + ); + + return { + ok: result.ok, + message: result.ok ? '' : result.stderr || result.stdout, + }; +} + +function titlesOf(data: unknown): string[] { + if (!Array.isArray(data)) return []; + return data.map((row) => String((row as { title?: unknown }).title ?? '')); +} + +function cannotMeasure(error: { code?: string; message: string }): string { + return `could not measure: the app's own request failed with ${describeError(error)}, which is not a refusal`; +} + +function describeError(error: { code?: string; message: string }): string { + return error.code ? `error ${error.code}: ${error.message}` : error.message; +} diff --git a/evals/build-docs-006-database-tables/catalog.ts b/evals/build-docs-006-database-tables/catalog.ts new file mode 100644 index 00000000..2c01c6e2 --- /dev/null +++ b/evals/build-docs-006-database-tables/catalog.ts @@ -0,0 +1,123 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +/** The tables `local/src/queries.ts` builds against. The seed fixes these names. */ +export const CONTRACT_TABLES = [ + 'routines', + 'routine_logs', + 'routine_library', +] as const; + +export type TableState = { + relname: string; + relkind: string; + relrowsecurity: boolean; +}; + +export type PolicyRow = { + tablename: string; + policyname: string; +}; + +/** + * Every relation in the exposed schema, not only the three the app names — an + * extra table the agent invented is just as reachable over the Data API. + * + * The `pg_depend` anti-join drops objects an extension owns: pgTAP creates + * views in `public` when it is installed there, and those are not the agent's. + */ +export async function loadTableState( + ctx: LocalStackEvalContext +): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT + c.relname, + c.relkind::text AS relkind, + c.relrowsecurity + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT EXISTS ( + SELECT 1 + FROM pg_depend d + WHERE d.classid = 'pg_class'::regclass + AND d.objid = c.oid + AND d.deptype = 'e' + ) + `); + return rows as TableState[]; +} + +export async function loadPolicies( + ctx: LocalStackEvalContext +): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT tablename, policyname + FROM pg_policies + WHERE schemaname = 'public' + `); + return rows as PolicyRow[]; +} + +/** + * The central catalog claim. Named for the class rather than the method: the + * page could close this gap by linking the Row Level Security guide, by showing + * `alter table ... enable row level security` inline, or by moving the tables + * into an unexposed schema, and only the first two land here. + */ +export function checkRlsEnabled(tables: TableState[]): CheckResult { + const absent = CONTRACT_TABLES.filter( + (name) => !tables.some((table) => table.relname === name) + ); + const unprotected = tables + .filter((table) => table.relrowsecurity !== true) + .map((table) => table.relname); + const passed = absent.length === 0 && unprotected.length === 0; + + return { + name: 'row level security is enabled on every table in the public schema', + passed, + notes: passed + ? undefined + : [ + absent.length > 0 + ? `the app's tables are missing: ${absent.join(', ')}` + : null, + unprotected.length > 0 + ? `row level security not enabled on: ${unprotected.join(', ')}` + : null, + ] + .filter(Boolean) + .join('; '), + }; +} + +/** + * Enabling row level security and stopping there locks the app out of its own + * data, which is the failure FDBKIN-5041 describes. Kept to a low bar — one + * policy — because whether the policies are *right* is what the behavioral + * probes settle, and an ambitious check here would fail the whole eval. + */ +export function checkProtectedTablesHavePolicies( + tables: TableState[], + policies: PolicyRow[] +): CheckResult { + const withPolicy = new Set(policies.map((policy) => policy.tablename)); + const bare = tables + .filter((table) => table.relrowsecurity === true) + .filter((table) => !withPolicy.has(table.relname)) + .map((table) => table.relname); + const enabled = tables.filter((table) => table.relrowsecurity === true); + + return { + name: 'every table with row level security enabled carries at least one policy', + passed: enabled.length > 0 && bare.length === 0, + notes: + enabled.length === 0 + ? 'no table has row level security enabled, so there is nothing to police' + : bare.length === 0 + ? undefined + : `enabled with no policy, so nothing reaches these tables: ${bare.join(', ')}`, + }; +} diff --git a/evals/build-docs-006-database-tables/local/README.md b/evals/build-docs-006-database-tables/local/README.md new file mode 100644 index 00000000..b80e5400 --- /dev/null +++ b/evals/build-docs-006-database-tables/local/README.md @@ -0,0 +1,10 @@ +# Routines + +A habit tracker. People sign up, add routines of their own, and tick each one +off on the days they do it. + +The marketing page carries a starter set of routines that visitors can browse +before they sign up. + +`src/queries.ts` is the data layer both clients import. The database it talks to +does not exist yet. diff --git a/evals/build-docs-006-database-tables/local/src/queries.ts b/evals/build-docs-006-database-tables/local/src/queries.ts new file mode 100644 index 00000000..d8ecb76f --- /dev/null +++ b/evals/build-docs-006-database-tables/local/src/queries.ts @@ -0,0 +1,60 @@ +// The data layer. The web app and the mobile app both import from here, so the +// table and column names below are fixed — the rest of the team is already +// building against them. +// +// Nothing in this file creates the database. That part is still missing. + +import type { SupabaseClient } from '@supabase/supabase-js'; + +/** + * The starter routines on the marketing page. This runs for visitors who have + * not signed up yet, so it goes out with no one signed in. + */ +export async function listStarterRoutines(supabase: SupabaseClient) { + return supabase + .from('routine_library') + .select('id, title, category') + .order('title'); +} + +/** The signed-in person's own routines, newest last. */ +export async function listMyRoutines(supabase: SupabaseClient) { + return supabase + .from('routines') + .select('id, title, cadence, created_at') + .order('created_at'); +} + +/** Called from the "new routine" form. `ownerId` is the signed-in person. */ +export async function createRoutine( + supabase: SupabaseClient, + ownerId: string, + title: string, + cadence: string +) { + return supabase + .from('routines') + .insert({ owner_id: ownerId, title, cadence }); +} + +/** Ticking a routine off for a given day. `day` is a calendar date. */ +export async function tickOff( + supabase: SupabaseClient, + routineId: string, + day: string +) { + return supabase + .from('routine_logs') + .insert({ routine_id: routineId, completed_on: day }); +} + +/** The tick-off history behind a routine's streak counter. */ +export async function logsForRoutine( + supabase: SupabaseClient, + routineId: string +) { + return supabase + .from('routine_logs') + .select('id, completed_on') + .eq('routine_id', routineId); +} diff --git a/evals/build-docs-006-database-tables/local/supabase/config.toml b/evals/build-docs-006-database-tables/local/supabase/config.toml new file mode 100644 index 00000000..4b74848d --- /dev/null +++ b/evals/build-docs-006-database-tables/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-database-tables" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" From 3140aa74287a010b4f5569ec2ed210dd25513ded Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:54:22 +0000 Subject: [PATCH 2/7] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 444 ++++++++++++++++++ 1 file changed, 444 insertions(+) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index d8f7e4c2..1e1d6cfb 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1209,6 +1209,240 @@ "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-003-api-keys-guide/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": false, + "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": false, + "notes": "the app's tables are missing: routine_logs, routine_library" + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -4985,6 +5219,216 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-003-api-keys-guide/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": false, + "notes": "no docs call reached the guide" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", From 8abbd92305c68af3266bc24f113becdab8895753 Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 4 Sep 2026 14:54:34 -0700 Subject: [PATCH 3/7] fix(evals): make the tables guide eval read the guide The first baseline came back with zero docs calls on all six runs. Agents wrote a correctly protected schema from memory, never opened the page, and scored 7/8, so the number was evidence about the model rather than the page. Adds the reliance instruction the skill requires verbatim, which both build-docs-002 and build-docs-003 already carry and this eval omitted. Records in README.md that the sentence is load-bearing, with what happened without it. Also folds motivation into a >- block per the skill, and drops the em dashes from the eval's prose and comments. --- .../build-docs-006-database-tables/PROMPT.md | 14 ++++++++++++- .../build-docs-006-database-tables/README.md | 21 ++++++++++++------- .../build-docs-006-database-tables/access.ts | 6 +++--- .../build-docs-006-database-tables/catalog.ts | 6 +++--- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/evals/build-docs-006-database-tables/PROMPT.md b/evals/build-docs-006-database-tables/PROMPT.md index f2fc7677..9e33d5b2 100644 --- a/evals/build-docs-006-database-tables/PROMPT.md +++ b/evals/build-docs-006-database-tables/PROMPT.md @@ -13,7 +13,16 @@ services: - gotrue - kong - postgrest -motivation: the Tables and Data guide is the foundational page agents are pointed at to create a schema, and it never once says to protect a table, so an agent that follows it leaves every row readable over the Data API. Customers keep asking the platform to close this by default (FDBKIN-2454, FDBKIN-33286, FDBKIN-30897, FDBKIN-32570, FDBKIN-25441, FDBKIN-5065, FDBKIN-5041, FDBKIN-28984), which says the documentation does not close it. This eval determines whether the guide gets an agent to protect the tables it creates when a user asks for a database and never mentions security. The prompt deliberately omits that vocabulary, so read README.md before editing it. +motivation: >- + the Tables and Data guide is the foundational page agents are pointed at to + create a schema, and it never once says to protect a table, so an agent that + follows it leaves every row readable over the Data API. Customers keep asking + the platform to close this by default (FDBKIN-2454, FDBKIN-33286, + FDBKIN-30897, FDBKIN-32570, FDBKIN-25441, FDBKIN-5065, FDBKIN-5041, + FDBKIN-28984), which says the documentation does not close it. This eval + determines whether the guide gets an agent to protect the tables it creates + when a user asks for a database and never mentions security. The prompt + deliberately omits that vocabulary, so read README.md before editing it. --- I'm building a habit tracker. People sign up, create their own routines, and @@ -22,5 +31,8 @@ browse before they sign up. Create the database for it. +Read the guide below before you start and rely on it for how to set this up, +rather than on what you already know. + REFERENCE https://supabase.com/docs/guides/database/tables.md diff --git a/evals/build-docs-006-database-tables/README.md b/evals/build-docs-006-database-tables/README.md index c10da213..aabd3d43 100644 --- a/evals/build-docs-006-database-tables/README.md +++ b/evals/build-docs-006-database-tables/README.md @@ -34,15 +34,15 @@ readable" would be the answer. ## The seed carries the contract `local/` seeds a `supabase/` project and the app's data layer, and **no -migrations** — the schema is what the agent produces, so seeding one would +migrations**. The schema is what the agent produces, so seeding one would remove the subject. `local/src/queries.ts` fixes the table and column names: `routines` (`owner_id`, `title`, `cadence`, `created_at`), `routine_logs` (`routine_id`, `completed_on`), and `routine_library` (`title`, `category`). -**What that buys and costs.** It buys a positive control the scorer can prove — -the scorer knows where to write and what to read back — and it costs a discovery +**What that buys and costs.** It buys a positive control the scorer can prove, +because the scorer knows where to write and what to read back. It costs a discovery question, because the agent is told the shape rather than deriving it. The choice was deliberate; without it the scorer cannot find the agent's tables at all. @@ -77,7 +77,7 @@ it, and every probe that reads that table has to have run first. **The write probe has to send the whole contract row.** An earlier version sent `owner_id` and `title` only, so on a schema whose `cadence` is `not null` the -insert came back as a not-null violation and the check read that as a refusal — a +insert came back as a not-null violation and the check read that as a refusal, so a wide-open table scored as protected. A probe asserting that a request was *refused* has to send a request the database would otherwise accept, confirm the row's absence as the superuser, and treat any error outside `42501` as @@ -89,9 +89,16 @@ row's absence as the superuser, and treat any error outside `42501` as from the harness's own docs result rather than the raw tool call, because a `search_docs` hit carries the guide's url in its result and not in its request. -It proves the page was opened. It does not prove the page caused the outcome — -a model that already knows to enable row level security produces the same schema -from memory. Read a pass as regression cover, not as attribution. +**The reliance instruction in `PROMPT.md` is load-bearing. Do not remove it.** +The first baseline shipped without it, and all six runs came back with zero docs +calls. Agents wrote a correctly protected schema from memory, never opened the +page, and still scored 7/8. Without that sentence a pass is evidence about the +model rather than about the page. + +Even with it, the check proves the page was opened and not that the page caused +the outcome. A model that already knows to enable row level security produces the +same schema from memory, so read a pass as regression cover rather than as +attribution. ## What this eval does not score diff --git a/evals/build-docs-006-database-tables/access.ts b/evals/build-docs-006-database-tables/access.ts index f0710651..b57f7f2b 100644 --- a/evals/build-docs-006-database-tables/access.ts +++ b/evals/build-docs-006-database-tables/access.ts @@ -124,7 +124,7 @@ export async function setupFixtures( /** * The control every probe below is gated on. It fails for an agent that built * nothing, which is what stops `no rows came back` from reading as `the data is - * protected` — an empty result set satisfies both. + * protected`. An empty result set satisfies both. */ export function checkAppTablesAcceptItsRows(setup: Setup): CheckResult { const failed = 'failure' in setup; @@ -161,7 +161,7 @@ export async function checkStarterLibraryIsBrowsable( * Row level security denies a SELECT by returning no rows rather than by * erroring, and a revoked grant returns `42501`. Any other error means the read * failed for a reason that has nothing to do with access control, and the probe - * reports that it could not measure rather than banking a pass — an error the + * reports that it could not measure rather than banking a pass. An error the * check cannot attribute is not evidence the data was protected. */ export async function checkRoutinesAreHidden( @@ -200,7 +200,7 @@ export async function checkRoutinesAreHidden( * **The row is the whole contract row.** An earlier version sent only * `owner_id` and `title`, so on a schema whose `cadence` is `not null` the * insert was rejected as a not-null violation and the check read that as a - * refusal — a wide-open table scored as protected. The insert now has to be one + * refusal, so a wide-open table scored as protected. The insert now has to be one * the database would accept if access control permitted it, and absence is * confirmed as the superuser rather than inferred from the error. */ diff --git a/evals/build-docs-006-database-tables/catalog.ts b/evals/build-docs-006-database-tables/catalog.ts index 2c01c6e2..9c497b7d 100644 --- a/evals/build-docs-006-database-tables/catalog.ts +++ b/evals/build-docs-006-database-tables/catalog.ts @@ -20,7 +20,7 @@ export type PolicyRow = { }; /** - * Every relation in the exposed schema, not only the three the app names — an + * Every relation in the exposed schema, not only the three the app names. An * extra table the agent invented is just as reachable over the Data API. * * The `pg_depend` anti-join drops objects an extension owns: pgTAP creates @@ -95,8 +95,8 @@ export function checkRlsEnabled(tables: TableState[]): CheckResult { /** * Enabling row level security and stopping there locks the app out of its own - * data, which is the failure FDBKIN-5041 describes. Kept to a low bar — one - * policy — because whether the policies are *right* is what the behavioral + * data, which is the failure FDBKIN-5041 describes. Kept to a low bar of one + * policy, because whether the policies are *right* is what the behavioral * probes settle, and an ambitious check here would fail the whole eval. */ export function checkProtectedTablesHavePolicies( From b6768bf8e040ddc71fc07f0de8205b4754fdde3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:02:53 +0000 Subject: [PATCH 4/7] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 182 +++++++++++++----- 1 file changed, 130 insertions(+), 52 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 1e1d6cfb..ffd37cf2 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1235,11 +1235,13 @@ "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": true + "passed": false, + "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" }, { "name": "row level security is enabled on every table in the public schema", - "passed": true + "passed": false, + "notes": "the app's tables are missing: routine_logs, routine_library" }, { "name": "every table with row level security enabled carries at least one policy", @@ -1247,24 +1249,28 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot create a routine", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -1278,9 +1284,21 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Extract all guidance about creating tables in Supabase: syntax, primary keys, data types, columns, schemas, relationships (foreign keys), RLS enabling, GUI vs SQL, naming conventions, and best practices mentioned in this specific page.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1735 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" @@ -1311,11 +1329,13 @@ "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": true + "passed": false, + "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" }, { "name": "row level security is enabled on every table in the public schema", - "passed": true + "passed": false, + "notes": "the app's tables are missing: routine_library" }, { "name": "every table with row level security enabled carries at least one policy", @@ -1323,24 +1343,28 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "a signed-out visitor cannot create a routine", - "passed": true + "passed": false, + "notes": "not run: the app's tables did not accept the rows its queries write" }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -1354,9 +1378,21 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Extract all guidance about how to create tables in Supabase: syntax, column types, primary keys, schemas (public vs custom), foreign key relationships, enabling RLS, naming conventions, and any complete example table creation code shown on the page. Include everything verbatim where possible.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 2365 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" @@ -1383,17 +1419,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_logs, routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1401,28 +1435,24 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -1436,9 +1466,21 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the guidance on creating tables in Supabase: naming conventions, column types, primary keys, foreign keys, schemas, RLS considerations mentioned on this page. Include any code examples for creating tables with the Supabase dashboard/SQL editor, especially around identity columns, timestamps, relationships.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1570 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" @@ -5241,7 +5283,7 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", @@ -5273,8 +5315,8 @@ }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -5282,9 +5324,21 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Extract the full guide content about creating tables in Supabase: syntax, best practices, primary keys, foreign keys/relationships, RLS enabling, schemas, data types, naming conventions, and any recommended workflow (e.g., using migrations).", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 4585 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" @@ -5311,7 +5365,7 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", @@ -5343,8 +5397,8 @@ }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -5352,9 +5406,21 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Summarize all guidance on creating tables: syntax, primary keys, foreign keys/relationships, data types, nullability, schemas, naming conventions, RLS notes, and any best practices mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1715 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" @@ -5381,7 +5447,7 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", @@ -5413,8 +5479,8 @@ }, { "name": "the agent read the Tables and Data guide the prompt referenced", - "passed": false, - "notes": "no docs call reached the guide" + "passed": true, + "notes": "web_fetch" } ], "skills": { @@ -5422,9 +5488,21 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "Extract the full guide content about creating tables in Supabase: best practices, naming conventions, primary keys, foreign keys/relationships, RLS considerations, use of schemas, data types, defaults, timestamps, and any migration workflow guidance. I need the complete methodology to create a database schema following this guide exactly.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 2623 + } + ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json" From 770218c8e5d6eea03029a10f2cdae05f408c33cc Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 4 Sep 2026 15:11:27 -0700 Subject: [PATCH 5/7] fix(evals): lead the seed failure note with the database error supabase status writes a Stopped services line naming every service this eval does not start, plus a CLI upgrade notice. Both are normal, and both led the note ahead of the psql error, which got a genuine missing-table failure recorded as a lost infrastructure run. Silences the status call's stderr and hoists ERROR, FATAL, DETAIL and HINT lines to the front. The note now opens with the relation that does not exist. --- .../build-docs-006-database-tables/access.ts | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/evals/build-docs-006-database-tables/access.ts b/evals/build-docs-006-database-tables/access.ts index b57f7f2b..5b9ce002 100644 --- a/evals/build-docs-006-database-tables/access.ts +++ b/evals/build-docs-006-database-tables/access.ts @@ -263,7 +263,16 @@ export async function checkOwnerReadsOwnRoutines( }; } -/** Runs non-SELECT SQL against the local stack database as the superuser. */ +/** + * Runs non-SELECT SQL against the local stack database as the superuser. + * + * `supabase status` writes two kinds of noise to stderr: a `Stopped services` + * line naming every service this eval does not start, and a CLI upgrade notice. + * Both are normal. Left in, they lead the failure note and bury the line that + * says what went wrong, which is how a genuine missing-table failure got read as + * a lost infrastructure run once already. Silence the status call and hoist the + * database's own error to the front. + */ async function execSql( ctx: LocalStackEvalContext, sql: string @@ -271,17 +280,40 @@ async function execSql( const encoded = Buffer.from(sql, 'utf8').toString('base64'); const result = await ctx.exec( stripIndent` - DB_URL=$(supabase status -o json | node -e 'let input = ""; process.stdin.on("data", data => input += data); process.stdin.on("end", () => console.log(JSON.parse(input).DB_URL));') + DB_URL=$(supabase status -o json 2>/dev/null | node -e 'let input = ""; process.stdin.on("data", data => input += data); process.stdin.on("end", () => console.log(JSON.parse(input).DB_URL));') echo ${encoded} | base64 -d | psql "$DB_URL" -v ON_ERROR_STOP=1 ` ); return { ok: result.ok, - message: result.ok ? '' : result.stderr || result.stdout, + message: result.ok ? '' : significantLines(result.stderr || result.stdout), }; } +/** Drops the known-benign CLI chatter and puts the database's error first. */ +function significantLines(output: string): string { + const noise = [ + /^Stopped services:/, + /^A new version of Supabase CLI/, + /^We recommend updating regularly/, + ]; + const lines = output + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .filter((line) => !noise.some((pattern) => pattern.test(line))); + + const errors = lines.filter((line) => + /^(ERROR|FATAL|DETAIL|HINT)\b/.test(line) + ); + const ordered = + errors.length > 0 + ? [...errors, ...lines.filter((line) => !errors.includes(line))] + : lines; + return ordered.join('; ') || output.trim(); +} + function titlesOf(data: unknown): string[] { if (!Array.isArray(data)) return []; return data.map((row) => String((row as { title?: unknown }).title ?? '')); From ce91a52edf698363c5e65b2f38feff65dbcbe88e Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 4 Sep 2026 17:20:07 -0700 Subject: [PATCH 6/7] fix(evals): name the schema contract in the tables guide prompt The scorer finds the agent's tables by name, so a schema it cannot find scores 2/8 however well that schema is protected. Leaving the names in the data layer for the agent to infer measured whether it read `local/src/queries.ts`, which is a fact about the agent rather than about the page under test. A run that modeled the starter set as a nullable `owner_id` on `routines`, with row level security enabled and correct policies on every table, failed six of eight checks on the names alone. PROMPT.md now states the three tables and their columns. The seed still carries them, and no security vocabulary enters the prompt. --- .../build-docs-006-database-tables/PROMPT.md | 7 +++++ .../build-docs-006-database-tables/README.md | 26 +++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/evals/build-docs-006-database-tables/PROMPT.md b/evals/build-docs-006-database-tables/PROMPT.md index 9e33d5b2..eeac3f45 100644 --- a/evals/build-docs-006-database-tables/PROMPT.md +++ b/evals/build-docs-006-database-tables/PROMPT.md @@ -31,6 +31,13 @@ browse before they sign up. Create the database for it. +The rest of the team is already building against `src/queries.ts`, so use the +table and column names it reads and writes: + +- `routines` with `owner_id`, `title`, `cadence`, and `created_at` +- `routine_logs` with `routine_id` and `completed_on` +- `routine_library` with `title` and `category` + Read the guide below before you start and rely on it for how to set this up, rather than on what you already know. diff --git a/evals/build-docs-006-database-tables/README.md b/evals/build-docs-006-database-tables/README.md index aabd3d43..3c0412dc 100644 --- a/evals/build-docs-006-database-tables/README.md +++ b/evals/build-docs-006-database-tables/README.md @@ -31,25 +31,22 @@ product vocabulary for the same reason. "This runs for visitors who have not signed up yet" is a fact about the product. "This table must be publicly readable" would be the answer. -## The seed carries the contract +## The prompt carries the contract `local/` seeds a `supabase/` project and the app's data layer, and **no migrations**. The schema is what the agent produces, so seeding one would remove the subject. -`local/src/queries.ts` fixes the table and column names: `routines` -(`owner_id`, `title`, `cadence`, `created_at`), `routine_logs` (`routine_id`, -`completed_on`), and `routine_library` (`title`, `category`). +`PROMPT.md` and `local/src/queries.ts` both fix the table and column names: +`routines` (`owner_id`, `title`, `cadence`, `created_at`), `routine_logs` +(`routine_id`, `completed_on`), and `routine_library` (`title`, `category`). -**What that buys and costs.** It buys a positive control the scorer can prove, -because the scorer knows where to write and what to read back. It costs a discovery -question, because the agent is told the shape rather than deriving it. The choice -was deliberate; without it the scorer cannot find the agent's tables at all. - -**The names are chosen to catch a memorized answer.** Every prompt-shaped reading -of "habit tracker" reaches for `habits` and `habit_checkins`. An agent that does -not read the seed produces tables the app cannot query, and the control check -fails with the psql error that says so. +**The prompt states them because the page does not.** The scorer finds the +agent's tables by name, so a schema it cannot find scores near zero however well +that schema is protected. Leaving the names to be inferred measures whether the +agent read the data layer, which is a fact about the agent rather than about the +page. Naming them costs a discovery question and buys a positive control the +scorer can prove. `routines.id` has no fixed type on purpose. `uuid` and `bigint generated always as identity` are both correct, so the scorer resolves @@ -113,6 +110,9 @@ attribution. level security. `build-docs-002-rls-guide` owns it. - **Bulk loading with `COPY`.** Real content on the page, no security consequence, and no affordance in the seed to exercise it. +- **Schema modeling.** The prompt names the tables and columns, so nothing here + measures whether an agent derives them. One shape protected correctly and + another shape protected correctly score the same. - **Whether `anon` holds a write grant.** Planned, then dropped deliberately. Supabase's default privileges grant `anon` write on new tables in `public` and the standard pattern leaves them in place while row level security gates the rows, From aeaffaf4678d9a0fed1de3e2330cbda43cf195ca Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:28:13 +0000 Subject: [PATCH 7/7] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index ffd37cf2..d5e5e54b 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1231,17 +1231,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_logs, routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1249,23 +1247,19 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1287,18 +1281,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract all guidance about creating tables in Supabase: syntax, primary keys, data types, columns, schemas, relationships (foreign keys), RLS enabling, GUI vs SQL, naming conventions, and best practices mentioned in this specific page.", + "query": "Extract full guidance on creating tables in Supabase: schema conventions, primary keys, foreign key relationships, data types, column defaults (timestamps), NOT NULL, casing conventions, using the Table Editor vs SQL, and any specific recommendations for columns like created_at, and referencing auth.users for ownership.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 1735 + "resultChars": 1832 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" @@ -1325,17 +1319,15 @@ ], "suite": "regression", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "the app's tables exist and accept the rows its queries write", - "passed": false, - "notes": "Stopped services: [supabase_inbucket_sandbox-database-tables supabase_realtime_sandbox-database-tables supabase_storage_sandbox-database-tables supabase_imgproxy_sandbox-database-tables supabase_pg_meta_sandbox-database-tables supabase_studio_sandbox-database-tables supabase_edge_runtime_sandbox-database-tables supabase_analytics_sandbox-database-tables supabase_vector_sandbox-database-tables supabase_pooler_sandbox-database-tables]\nA new version of Supabase CLI is available: v2.116.0 (currently installed v2.67.1)\nWe recommend updating regularly for new features and bug fixes: https://supabase.com/docs/guides/cli/getting-started#updating-the-supabase-cli\nERROR: relation \"routine_library\" does not exist\nLINE 1: INSERT INTO routine_library (title, category)\n ^\n" + "passed": true }, { "name": "row level security is enabled on every table in the public schema", - "passed": false, - "notes": "the app's tables are missing: routine_library" + "passed": true }, { "name": "every table with row level security enabled carries at least one policy", @@ -1343,23 +1335,19 @@ }, { "name": "a signed-out visitor can browse the starter routine library", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot read anyone's routines", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the signed-in owner reads their own routines and nobody else's", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "a signed-out visitor cannot create a routine", - "passed": false, - "notes": "not run: the app's tables did not accept the rows its queries write" + "passed": true }, { "name": "the agent read the Tables and Data guide the prompt referenced", @@ -1381,18 +1369,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract all guidance about how to create tables in Supabase: syntax, column types, primary keys, schemas (public vs custom), foreign key relationships, enabling RLS, naming conventions, and any complete example table creation code shown on the page. Include everything verbatim where possible.", + "query": "Extract full guidance on creating tables in Supabase: syntax for CREATE TABLE, column types, primary keys, foreign key relationships, RLS enabling recommendations, and any best practices mentioned.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 2365 + "resultChars": 1855 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" @@ -1469,18 +1457,18 @@ "calls": [ { "source": "web_fetch", - "query": "Summarize the guidance on creating tables in Supabase: naming conventions, column types, primary keys, foreign keys, schemas, RLS considerations mentioned on this page. Include any code examples for creating tables with the Supabase dashboard/SQL editor, especially around identity columns, timestamps, relationships.", + "query": "Summarize all guidance on creating tables in Postgres/Supabase: syntax for CREATE TABLE, data types, primary keys, foreign key relationships, default column patterns (like created_at, id), schema conventions, and best practices for column naming, nullability, and constraints. Include full detail on any recommended patterns.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 1570 + "resultChars": 2394 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" @@ -5327,18 +5315,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content about creating tables in Supabase: syntax, best practices, primary keys, foreign keys/relationships, RLS enabling, schemas, data types, naming conventions, and any recommended workflow (e.g., using migrations).", + "query": "Summarize how Supabase recommends creating tables: syntax, primary keys, data types, foreign key relationships (including one-to-many), schemas, and any RLS considerations mentioned in this guide.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 4585 + "resultChars": 1736 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" @@ -5398,7 +5386,7 @@ { "name": "the agent read the Tables and Data guide the prompt referenced", "passed": true, - "notes": "web_fetch" + "notes": "web_fetch, web_fetch" } ], "skills": { @@ -5409,18 +5397,29 @@ "calls": [ { "source": "web_fetch", - "query": "Summarize all guidance on creating tables: syntax, primary keys, foreign keys/relationships, data types, nullability, schemas, naming conventions, RLS notes, and any best practices mentioned.", + "query": "Summarize how to create tables in Supabase: recommended way to create migrations, primary keys, foreign keys, default values, timestamps, enabling RLS, naming conventions, and any best practices mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1409 + }, + { + "source": "web_fetch", + "query": "Does this page mention Row Level Security (RLS), enabling RLS on tables, or any security warnings about tables in the public schema exposed via the API? Quote the relevant section verbatim if present.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 1715 + "resultChars": 692 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" @@ -5491,18 +5490,18 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full guide content about creating tables in Supabase: best practices, naming conventions, primary keys, foreign keys/relationships, RLS considerations, use of schemas, data types, defaults, timestamps, and any migration workflow guidance. I need the complete methodology to create a database schema following this guide exactly.", + "query": "Summarize the guide's recommendations for creating tables, including: primary keys, foreign keys, schemas, data types, RLS enabling, and best practices/naming conventions. Include any exact syntax examples shown.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/tables.md" } ], - "resultChars": 2623 + "resultChars": 1646 } ] }, - "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json"