diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 344b29de..d9842f1d 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,127 +1,11 @@ [ { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user d7fb398a-0a59-40f5-aaa8-bc0b26722470, signUp returned {\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password options data user metadata javascript\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 22309 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3161 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-opus-5", + "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" }, "eval": "build-cli-001-bootstrap-app", "stage": "build", @@ -148,7 +32,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -161,160 +45,12 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "error 42501: permission denied for table todos" + "notes": "0 rows" }, { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | grep -iE 'breaking|rls|policy|grant' | head -30", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5391 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>&1 | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md" - } - ], - "resultChars": 3976 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" - } - ], - "resultChars": 4170 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 83) from the queue" + "notes": "3 rows" } ], "skills": { @@ -329,22989 +65,22 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job invoke edge function pgmq queue\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - } - ], - "resultChars": 62989 - }, - { - "source": "web_fetch", - "query": "How do you create a cron job that runs every minute using SQL (cron.schedule)? Show the exact SQL syntax, including how to name a job, how to run SQL vs invoke an edge function, and how to unschedule/replace an existing job. Include any notes about job names and idempotency.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" - } - ], - "resultChars": 1310 - }, - { - "source": "web_fetch", - "query": "How do you create a pgmq queue and send/read/delete messages? Show exact SQL function signatures for pgmq.create, pgmq.send, pgmq.read, pgmq.delete, pgmq.pop. Also note anything about queue types (basic/unlogged/partitioned) and the pgmq_public wrappers for the Data API.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart.md" - } - ], - "resultChars": 1391 - }, { "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise notable that relate to: Queues / pgmq, Cron / pg_cron, or Edge Functions (deployment, config.toml verify_jwt, serving). Include dates and links.", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3048 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key service role server-side supabase-js createClient API keys\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", - "title": "Use Supabase with SvelteKit" - } - ], - "resultChars": 94242 + "resultChars": 94610 } ] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth user Authorization header RLS createClient\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - } - ], - "resultChars": 73347 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 5299 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth-headers.md", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" - } - ], - "resultChars": 3330 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md | head -60", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 2599 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"831c5088-6bc6-4c0e-8c1f-3fe53e8a0852\",\"metric\":\"steps_b_msj188pa\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60 || echo \"changelog fetch failed\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3161 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45702-developer-update-may-2026.md 2>/dev/null | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog/45702-developer-update-may-2026.md" - } - ], - "resultChars": 6160 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions auth service role\", limit: 12) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/auth0", - "title": "Auth0" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/aws-cognito", - "title": "Amazon Cognito (Amplify)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/workos", - "title": "WorkOS" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - } - ], - "resultChars": 75829 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-alpha.pdf, 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all required criteria: private user-files bucket, owner-scoped SELECT and INSERT RLS policies on storage.objects for authenticated users with RLS kept enabled, and supabase-js createSignedUrl code with expiry for temporary sharing. No disqualifying public bucket, permissive policies, public URL sharing, or client service-role usage." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies private bucket user folder owner uid\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 36724 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Storage buckets, storage RLS policies, signed URLs, or the storage.objects schema (e.g. owner/owner_id columns, new bucket types). Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8117 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expires in temporary link download share\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" - } - ], - "resultChars": 3099 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "3 file(s): supabase/tests/002_tenant_isolation_writes.test.sql, supabase/tests/001_tenant_isolation_select.test.sql, supabase/tests/003_tenant_isolation_baseline.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "7 passed, 5 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, explicitly noting authenticated users in any org can read every post because the policy is missing `m.org_id = posts.org_id`. It grounds this in pgTAP results, distinguishes `notes` as passing read isolation, and treats test results as authoritative rather than trusting migration comments." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector gte-small edge function embeddings match_document_sections RLS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - } - ], - "resultChars": 84107 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape for evalshostedprojectxy.supabase.co at /customer/v1/privileged/metrics, uses HTTP Basic Auth with password_file, preserves existing app scrape, and docker-compose mounts the secrets directory matching the password_file path." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching password_file setup, reload/Compose rollout steps, and concrete verification via curl, Prometheus targets/query API, and Grafana dashboard." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 29057 - }, - { - "source": "web_fetch", - "query": "What is the exact metrics endpoint URL format, what authentication does it use (username/password), and what is the recommended Prometheus scrape config and scrape interval? Include any notes about rate limits or which key to use.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" - } - ], - "resultChars": 1186 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions secrets environment variables managing\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 71163 - }, - { - "source": "web_fetch", - "query": "List any breaking changes or notable changes related to Edge Functions, function deployment, function secrets/environment variables, or the CLI's secrets/functions commands.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1947 - }, - { - "source": "web_fetch", - "query": "How do you set and access environment variables/secrets in Supabase Edge Functions? Include the exact CLI commands for setting production secrets from a .env file, local .env handling, and reserved secret names. Quote commands verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 1274 - }, - { - "source": "web_fetch", - "query": "Show the current recommended boilerplate for a Supabase Edge Function index.ts (Deno.serve vs std serve import), the deploy command, and any config.toml settings for functions like verify_jwt. Quote code verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" - } - ], - "resultChars": 879 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/quickstart.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" - } - ], - "resultChars": 5029 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "Extract the full self-hosting with Docker guide verbatim-ish: the exact steps to get the docker setup, all the secrets/env vars that must be changed before going to production (JWT secret, anon/service keys, postgres password, dashboard user/pass, secret_key_base, vault enc key, SITE_URL, API_EXTERNAL_URL, SUPABASE_PUBLIC_URL, pooler tenant id, etc.), securing services, and any notes about restarting/regenerating keys.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 3113 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise relevant to self-hosting, Docker, JWT signing keys / API key format (publishable/secret vs anon/service_role), Supavisor/pooler, or Studio, from 2025 and 2026. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3915 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the original soft-delete-only bug and lack of session/refresh-token revocation, implements real auth/session revocation via banning and deleting sessions plus RLS checks, and explains the remaining stateless JWT caveat consistently: Data API notes/profiles are closed by RLS, but purely local JWT validation remains valid until exp with mitigations. It also correctly distinguishes publishable frontend keys with RLS from server-only secret/service_role keys that bypass RLS and must not ship to clients." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key new API keys anon service_role RLS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 190908 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete user does not invalidate access token revoke sessions sign out\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - } - ], - "resultChars": 66489 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED succeeding, fixed it with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations. It checked RLS/grants but did not blame them or change them." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as affected and described the 8 HTTP 503 gateway failures recurring across the morning of 2026-04-28 from 07:00Z to 12:00Z, distinguishing them from unrelated billing-webhook 503s." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/Edge platform layer, not function code, and grounds this in valid observations: gateway 503s had no matching edge-function invocations, nearby invocations succeeded, deployment/version stayed unchanged, and avatar-upload's function-level 500 is distinguished as a separate class of error." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening/support escalation to the platform with the specific time window and evidence, investigating the scheduled caller, adding jitter/retries, and separately triaging a code-path error." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts. Did not use permissive policies or disable RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy auth.uid() select insert performance index\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 67906 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change related to RLS, row level security, policies, the Data API / PostgREST, grants, or auth.uid(). Just list titles, dates and one-line summaries.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1403 - } - ] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in action #21, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #18, after which `supabase migration list` showed local/remote alignment and the push proceeded. I saw read-only psql inspections and a rolled-back verification schema, but no prohibited workaround or direct application of the avatar migration outside the CLI." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 5a5d5991-7eae-49b4-8f13-2487fba8366c, signUp returned {\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 3 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "3 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query:\"declarative database schemas migration workflow db diff\", limit:4){nodes{title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 62949 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 13) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute queues pgmq send\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - } - ], - "resultChars": 69432 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues create queue read delete messages edge function\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - } - ], - "resultChars": 55770 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" - }, - { - "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SERVICE_ROLE_KEY SB_SECRET_KEY publishable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 94601 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-alpha.pdf, 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, RLS kept enabled, and supabase-js createSignedUrl/createSignedUrls with expiry for temporary sharing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "3 file(s): supabase/tests/database/02_posts_tenant_isolation.test.sql, supabase/tests/database/03_memberships_and_writes.test.sql, supabase/tests/database/01_notes_tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 2 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from orgs they are not members of, and grounds this in pgTAP/direct test results. It also distinguishes `notes` as passing isolation." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, targets a supabase.co project ref, preserves the app job, and docker-compose mounts the secrets directory containing the password file path." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file path, Compose restart/up and reload guidance, plus concrete verification via Prometheus targets, PromQL API, and Grafana dashboard checks." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics observability\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 19943 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables secrets manage\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-secrets", - "title": "Manage Supabase secrets" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 61195 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer diagnoses the original flow as only soft-deleting public.profiles and not touching auth.users/sessions/refresh tokens. It implements real revocation by banning/soft-deleting the auth user and deleting refresh_tokens/sessions, plus RLS gating. It consistently explains JWT access tokens remain cryptographically valid until expiry for local validation, while its implemented RLS/session-existence check closes the Data API window. It also correctly distinguishes publishable frontend keys under anon/authenticated + RLS from secret/service_role backend-only keys that bypass RLS." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{\n banned: searchDocs(query: \"ban user banned_until revoke session delete user admin API\", limit: 4) {\n nodes { title href content }\n }\n keys: searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration RLS\", limit: 6) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - } - ], - "resultChars": 245869 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key API keys\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 61983 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"sb_secret sb_publishable new API keys migration\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 29035 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Understanding API keys publishable key anon role RLS\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 13676 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key bypasses Row Level Security service_role postgres role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 329 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"ban user ban_duration admin updateUserById\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-updateuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-updateuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - } - ], - "resultChars": 3573 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"deleteUser soft delete admin\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" - } - ], - "resultChars": 1237 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"disable legacy anon service_role key dashboard rotate JWT secret\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" - } - ], - "resultChars": 34590 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT expiry access token lifetime sessions\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 59617 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.admin.signOut revoke sessions scope global\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - } - ], - "resultChars": 5307 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"legacy API keys deprecation end of 2026 removal timeline\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 45967 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"banned_until auth.users table columns\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - } - ], - "resultChars": 31852 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete account user edge function service role\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - } - ], - "resultChars": 20425 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"do not modify auth schema managed by Supabase\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" - }, - { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - } - ], - "resultChars": 16231 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verifying a JWT from Supabase getClaims asymmetric local verification\", limit: 2) { nodes { title href } subsections: nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - } - ], - "resultChars": 490 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JSON Web Token JWT verifying revocation\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 14157 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"banned user cannot sign in ban\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" - } - ], - "resultChars": 392 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"cannot delete user foreign key constraint troubleshooting\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0021_fkey_to_auth_unique", - "title": "Database Advisor: Lint 0021_fkey_to_auth_unique" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/cascade-deletes", - "title": "Cascade Deletes" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 468 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user management managing users admin api\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 406 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Users guide auth users table permanent anonymous\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 11521 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"managing user data profiles trigger accessing user data\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - } - ], - "resultChars": 34746 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signOut jwt scope admin server side revoke refresh tokens\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-server-revokegrant" - } - ], - "resultChars": 38073 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT claims reference session_id is_anonymous fields\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 12679 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"shouldSoftDelete\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/soft-deletes-with-supabase-js", - "title": "Implementing soft deletes with supabase-js" - } - ], - "resultChars": 12330 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"service role key never expose server side edge function admin client\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 326 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Data API PostgREST how requests are authorized JWT role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 273 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"access token expiry setting jwt_exp configuration auth\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - } - ], - "resultChars": 358 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user_banned error code\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 11900 - }, - { - "source": "search_docs", - "query": "{ error(code: \"user_banned\", service: AUTH) { code httpStatusCode message } }", - "hasContent": false, - "pages": [], - "resultChars": 280 - }, - { - "source": "search_docs", - "query": "{ a: error(code: \"session_not_found\", service: AUTH) { code message } b: error(code: \"user_not_found\", service: AUTH) { code message } c: error(code: \"refresh_token_not_found\", service: AUTH) { code message } d: error(code: \"refresh_token_already_used\", service: AUTH) { code message } e: error(code: \"session_expired\", service: AUTH) { code message } }", - "hasContent": false, - "pages": [], - "resultChars": 971 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The answer correctly identifies that the channel can reach SUBSCRIBED while INSERT events are silent because public.orders was missing from the supabase_realtime publication. It fixes exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserves courier_locations, keeps RLS/policies intact, and explicitly does not blame RLS or client code." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and described the recurring pattern of eight HTTP 503 gateway failures spread through the morning of 2026-04-28, while distinguishing them from older billing-webhook 503s." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer, not function code, and grounds this in valid observations: 503s only in gateway logs with no runtime invocations, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: investigate half-hour scheduled jobs/concurrency, query function_edge_logs for status>=500, check Edge Function limits/metrics, open a support ticket with timestamps, and examine avatar-upload error output." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with zero policies as the cause of empty Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push` in action #19, showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `20240115000000_add_profile_bio.sql` in action #16, after which Supabase CLI migration list showed it matched remote (#17) and the subsequent `db push` succeeded. No disallowed direct-SQL mutation or prepared-statement workaround was used." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 0ac704f8-596e-4f98-84c9-00595301873d, signUp returned {\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"cron.schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 35564 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions read from queue pgmq delete message worker\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 93380 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL automatically available\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28165 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Queues schedule cron job to process messages Edge Function example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", - "title": "Slack Bot Mention Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - } - ], - "resultChars": 81933 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"7ef32f13-71a0-4757-84bc-00f9632c3443\",\"metric\":\"steps_b_msj1bz52\",\"value\":222}]}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY secret key\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 36932 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions verify_jwt\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 70715 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret publishable multiple auth withSupabase array\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 45970 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers edge functions apikey header verify_jwt disable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - } - ], - "resultChars": 95133 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"combining auth modes user secret verify_jwt false config.toml example\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 24076 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package version pin npm install deno.json import map\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - } - ], - "resultChars": 34326 - }, - { - "source": "web_search", - "query": "\"failed to determine entrypoint\" supabase edge-runtime worker boot error", - "pages": [], - "resultChars": 2031 - }, - { - "source": "web_search", - "query": "supabase cli issue 4190 podman edge functions serve entrypoint bind mount", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/troubleshooting", - "title": "Supabase Docs | Edge Functions Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Supabase Docs | Troubleshooting | Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips | Supabase Docs" - } - ], - "resultChars": 2355 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Issues serving Edge Functions locally troubleshooting entrypoint bind mount docker context\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 45231 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-alpha.pdf, 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, defines authenticated SELECT and INSERT policies on storage.objects scoped to bucket_id and the user's UID folder with WITH CHECK for uploads, keeps RLS enabled, avoids public/anon/service-role pitfalls, and provides supabase-js createSignedUrl with an expiry for temporary sharing." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy owner folder path user id\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 61599 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring share link\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" - } - ], - "resultChars": 7873 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "6 passed, 4 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, specifically that members of any org can read posts from other orgs, and grounds this in pgTAP test 5 failing. It treats test results as authoritative and contrasts `posts` with `notes`, which passed isolation tests. It also notes additional membership-table issues, but does not blame `notes` instead of `posts`." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search edge functions pgvector gte-small match_document_sections\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/database/full-text-search", - "title": "Full Text Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - } - ], - "resultChars": 61699 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"vector extension schema best practice extensions schema\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - } - ], - "resultChars": 46198 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape at the correct path with project target, basic_auth using password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes steps to create/use a Supabase Secret API key, place it in the mounted secret file matching password_file, reload/restart Prometheus via Compose or lifecycle reload, and verify via Prometheus targets showing the supabase job UP." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics prometheus endpoint observability\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 29057 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables secrets Deno.env deploy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 38791 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management API invoke edge function endpoint\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 66572 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only bug, implements real revocation by deleting auth.sessions and banning the auth user, adds RLS checks to close the data-access stale-JWT window, and consistently explains that stateless JWTs remain locally valid until expiry. It also correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys and notes secret bypasses RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED status, added only public.orders to the existing publication, preserved courier_locations/RLS/policies, and did not blame or alter client code, RLS, grants, or networking." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the main affected function and described the recurring HTTP 503 pattern across the 2026-04-28 morning window, including the eight gateway failures from about 07:00Z to 12:00Z. Also correctly distinguished older billing-webhook 503s as unrelated." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes image-transform 503s to upstream gateway/platform rejection rather than handler code, grounded in missing edge-function runtime logs for 503s while 200s appear, and distinguishes avatar-upload's runtime 500 as a separate function-level error." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant provided multiple concrete actionable next steps, including instrumenting memory/input sizes, checking for a recurring scheduler/batch job, adding retry/backoff, investigating the separate avatar-upload stack trace, and considering architectural changes for heavy image processing." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for insert, and verified behavior." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push` in #11, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local file `20240115000000_add_profile_bio.sql` in #9, after which `supabase migration list` matched local/remote and `db push` proceeded. No disallowed workaround or direct mutation observed; psql usage was read-only inspection." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 863566c0-fe1b-4264-9906-53f26c545be8, signUp returned {\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"36482d33-6a70-43e3-bd27-459a7428cf03\",\"metric\":\"steps_b_msj0y6nu\",\"value\":222}]}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5516 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL default secrets\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 41771 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions env var migration from anon service_role\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 95976 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers verify_jwt combining auth modes user secret apikey Edge Functions\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 24799 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-alpha.pdf, 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "10 passed, 5 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw, explains the missing `org_id` predicate, and grounds the conclusion in the pgTAP failures showing cross-org post visibility. It does not blame `notes` and treats the test results as authoritative." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: Supabase Metrics API scrape uses HTTPS, correct metrics path, Basic Auth with password_file, valid supabase.co project target, app scrape is preserved, and docker-compose mounts the secrets directory containing the password file." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose start/reload, and concrete verification via Prometheus targets." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" - } - ], - "resultChars": 23548 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets all rubric requirements: identifies soft-delete/no auth revocation root cause, implements session/refresh-token revocation plus sign-in blocking and RLS enforcement, explains remaining stateless JWT/local-validation window consistently with the RLS fix, and correctly distinguishes publishable vs secret keys and RLS behavior." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified that the channel can reach SUBSCRIBED while INSERT events do not fire because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified courier_locations remained included, and did not disable RLS or weaken policies." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, including all 8 failures from 07:00Z through 12:00Z. It also avoided misattributing the main issue to the older `billing-webhook` 503s." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the recurring 503s to gateway/platform/boot-level failure rather than application code, grounded in valid observations: no corresponding execution logs for failed requests, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: checking Edge Function dashboard logs for BOOT_ERROR/WORKER_RESOURCE_LIMIT in a specific time window, reviewing recent deployment img-deploy-42 and considering rollback, investigating resource limits/input size/concurrency, and adding alerting." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function 503 error worker boot\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", - "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" - } - ], - "resultChars": 43344 - } - ] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "PASS: avatar_url was applied through `supabase db push` in #14, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file in #12 (`20240115000000_add_profile_bio.sql`), after which `supabase migration list` in #13 showed local and remote histories aligned. psql usage was read-only inspection only; no disallowed workaround seen." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 3d73a23b-4cc8-4789-9fd3-9abe325e1baa, signUp returned {\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession auth admin createUser user_metadata display_name profiles\", limit: 5) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - } - ], - "resultChars": 79187 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"signInWithPassword supabase-js reference signUp options data getUser getSession auth.currentUser\", limit: 10) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - } - ], - "resultChars": 193019 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"local development migrations RLS select authenticated anon expose table data api\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 152845 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Securing your API grants authenticated anon RLS select policy\", limit: 3) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 63887 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query SearchDocs {\n searchDocs(query: \"declarative database schemas local development migration generate supabase schema_paths\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 65847 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute run function Supabase local queue pgmq pop delete\", limit: 5) {\n nodes {\n title\n href\n ... on Guide { content }\n ... on Subsection { content }\n ... on CLICommandReference { content }\n ... on ClientLibraryFunctionReference { content }\n ... on TroubleshootingGuide { content }\n ... on ManagementApiReference { content }\n }\n }\n}", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute Supabase queue pgmq pop delete\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 69751 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron unschedule jobname cron.schedule same name idempotent\", limit: 5) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", - "title": "`pg_cron launcher crashes with 'duplicate key value violates unique constraint'`" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - } - ], - "resultChars": 42853 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase local psql database query cli worker node supabase-js examples\", limit: 5) { edges { node { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" - } - ], - "resultChars": 36819 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,140p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7789 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase db start --from-backup logical backup pg_restore dump\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/download-logical-backups", - "title": "How to download logical backups in Supabase with physical backups enabled?" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - } - ], - "resultChars": 24731 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify_jwt auth.getUser Bearer token\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - } - ], - "resultChars": 32605 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"af4aed6d-bffa-426b-98f4-2f78b0fb1835\",\"metric\":\"steps_b_msj0xq5b\",\"value\":222}]}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth service_role key verify supabase access token request header user_id body\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", - "title": "Setting up auth context" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", - "title": "Fetching the user" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", - "title": "Security considerations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", - "title": "What secret keys allow access to" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", - "title": "Known limitations and compatibility differences" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", - "title": "Best practices for handling secret keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", - "title": "What to do if a secret key or service_role has been leaked or compromised?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", - "title": "Publishable keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", - "title": "Interaction with Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#public-functions", - "title": "Public functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", - "title": "External webhooks" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", - "title": "Combining modes" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", - "title": "Custom error responses" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", - "title": "Environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", - "title": "Authenticated user calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", - "title": "Service-to-service calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", - "title": "Understanding authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", - "title": "The verify_jwt platform check" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", - "title": "Why this pattern works" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", - "title": "Browser client" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", - "title": "Edge Function (WebSocket proxy)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", - "title": "Database schema" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", - "title": "Architecture" - } - ], - "resultChars": 91747 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase functions serve verify_jwt false config.toml edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 32331 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions service role key env var SUPABASE_SERVICE_ROLE_KEY secret key Deno.env\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - } - ], - "resultChars": 44193 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security organization members docs auth.uid policy security invoker\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 83587 - } - ] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-alpha.pdf, 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated role using first folder segment = auth.uid(), keeps RLS intact (no disabling), and provides createSignedUrl with a 15-minute expiry. Extra UPDATE/DELETE policies are acceptable." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage bucket policies objects path owner user id signed URL supabase\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - } - ], - "resultChars": 119861 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy owner path prefix foldername user id signed URL createSignedUrl\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsigneduploadurl", - "title": "from.createSignedUploadURL()" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurls", - "title": "from.createSignedUrls()" - } - ], - "resultChars": 17609 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage access control policy examples foldername auth uid owner\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" - } - ], - "resultChars": 60503 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage object ownership owner column path prefix foldername access control\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 32342 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl storage from bucket expiresIn share file temporary link\", limit: 3) { nodes { __typename title href ... on ClientLibraryFunctionReference { content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - } - ], - "resultChars": 1476 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identified `posts` as the broken tenant isolation policy, grounded it in the pgTAP result (`cross-org posts` check failed), and did not blame `notes` or dismiss the tests. It also noted `notes` passed and treated the test results as authoritative." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1140 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pgTAP tests RLS Supabase local database tests\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 68853 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"request.jwt.claims auth.uid set local pgtap Supabase\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 66662 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"semantic search pgvector row level security documents ownership\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - } - ], - "resultChars": 68480 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector extension vector schema extensions match_documents function RLS authenticated ownership\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/introduction", - "title": "Vector Buckets" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - } - ], - "resultChars": 49023 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": false, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml does not add a deployable Supabase scrape job. It relies on an entrypoint/env injection not shown, uses SUPABASE_SECRET_API_KEY instead of an HTTP Basic Auth password_file, and docker-compose.yml does not mount that password_file via a volume or Compose secret. The existing app job is preserved, but required Supabase scrape wiring is missing." - }, - { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README explains env-based setup and restart, but it does not require placing a matching secret file, and it lacks concrete verification steps via Prometheus targets, PromQL/Grafana, or equivalent." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase metrics Prometheus project metrics\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on TroubleshootingGuide { } } totalCount } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"metrics prometheus\", limit: 5) { nodes { title href content } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" - } - ], - "resultChars": 23603 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase secrets set function env var runtime\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - } - ], - "resultChars": 149695 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions JWT verification no-verify-jwt public browser invoke\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 47787 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase breaking-change self-hosting docker compose", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/self-hosting/docker", - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker" - } - ] - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets the rubric: identifies soft-delete-only bug, updates flow to delete auth user and lock down RLS, explains stale JWT caveat consistently with the RLS mitigation, and correctly distinguishes publishable/anon vs secret/service_role keys." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user revoke sessions auth admin sign out user delete account\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - } - ], - "resultChars": 20977 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key anon key service role frontend rls\", limit: 5) { nodes { title href ... on Guide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 68789 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog markdown", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/getting-started/api-keys Supabase publishable secret key anon service_role", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" - } - ] - }, - { - "source": "web_search", - "query": "'Deleting users' in https://supabase.com/docs/guides/auth/managing-user-data", - "pages": [] - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified the root cause as orders missing from supabase_realtime despite successful subscription, fixed by ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations plus RLS/policies." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Realtime postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", - "title": "Broadcast authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", - "title": "Streaming updates" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", - "title": "Streaming inserts" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", - "title": "Enable Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", - "title": "Using Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", - "title": "Listening on client side" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", - "title": "Create a trigger" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", - "title": "Create a trigger function" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", - "title": "Using Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", - "title": "Methodology" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#workloads", - "title": "Workloads" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#results", - "title": "Results" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-websockets", - "title": "Broadcast: Using WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-the-database", - "title": "Broadcast: Using the database" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-impact-of-payload-size", - "title": "Broadcast: Impact of payload size" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#1kb-payload", - "title": "1KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#10kb-payload", - "title": "10KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#50kb-payload", - "title": "50KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-scalability-scenarios", - "title": "Broadcast: Scalability scenarios" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#realtime-auth", - "title": "Realtime Auth" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#refreshed-tokens", - "title": "Refreshed tokens" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#custom-tokens", - "title": "Custom tokens" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#private-schemas", - "title": "Private schemas" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#receiving-old-records", - "title": "Receiving old records" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#quick-start", - "title": "Quick start" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#usage", - "title": "Usage" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-schemas", - "title": "Listening to specific schemas" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-insert-events", - "title": "Listening to INSERT events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-update-events", - "title": "Listening to UPDATE events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-delete-events", - "title": "Listening to DELETE events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-tables", - "title": "Listening to specific tables" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-multiple-changes", - "title": "Listening to multiple changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#filtering-for-specific-changes", - "title": "Filtering for specific changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#available-filters", - "title": "Available filters" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#equal-to-eq", - "title": "Equal to (eq)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#not-equal-to-neq", - "title": "Not equal to (neq)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-lt", - "title": "Less than (lt)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-or-equal-to-lte", - "title": "Less than or equal to (lte)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-gt", - "title": "Greater than (gt)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-or-equal-to-gte", - "title": "Greater than or equal to (gte)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#contained-in-list-in", - "title": "Contained in list (in)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#database-instance-and-realtime-performance", - "title": "Database instance and realtime performance" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#spaces-in-table-names", - "title": "Spaces in table names" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#delete-events-are-not-filterable", - "title": "Delete events are not filterable" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#join-errors", - "title": "Join errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#websocket-connection-setup", - "title": "WebSocket connection setup" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#protocol-messages", - "title": "Protocol messages" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#100", - "title": "1.0.0" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#200", - "title": "2.0.0" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#text-frames", - "title": "Text frames" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#binary-frames", - "title": "Binary frames" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast-push", - "title": "User Broadcast Push" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast", - "title": "User Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#event-types", - "title": "Event types" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#client-sent-events", - "title": "Client sent events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_join", - "title": "phx_join" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_leave", - "title": "phx_leave" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#heartbeat", - "title": "heartbeat" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#access_token", - "title": "access_token" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame", - "title": "broadcast (text frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame", - "title": "broadcast (binary frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence", - "title": "presence" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#server-sent-events", - "title": "Server sent events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_close", - "title": "phx_close" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_error", - "title": "phx_error" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_reply", - "title": "phx_reply" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#system", - "title": "system" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame-1", - "title": "broadcast (text frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame-1", - "title": "broadcast (binary frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres_changes", - "title": "postgres_changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_state", - "title": "presence_state" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_diff", - "title": "presence_diff" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#error-handling", - "title": "Error handling" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#channel-level-system-errors", - "title": "Channel-level system errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres-changes-subscription-errors", - "title": "Postgres Changes subscription errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-errors", - "title": "Broadcast errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence-errors", - "title": "Presence errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#access-token-refresh", - "title": "Access token refresh" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#reconnection", - "title": "Reconnection" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", - "title": "When to use Presence" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", - "title": "Quick start" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", - "title": "Framework examples" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", - "title": "Integration guides" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", - "title": "Security & configuration" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", - "title": "Core features" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", - "title": "When to use Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", - "title": "When to use Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", - "title": "Choose the right feature" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", - "title": "Clean up subscriptions" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", - "title": "Follow naming conventions" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", - "title": "Use private channels" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", - "title": "Essential best practices" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", - "title": "5.3 using database triggers" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", - "title": "5.2 using HTTP/REST API" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", - "title": "5.1 using client libraries" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", - "title": "5. Send and receive messages" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", - "title": "4. Set up authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", - "title": "3. Create your first Channel" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", - "title": "Get API details" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", - "title": "2. Initialize the client" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", - "title": "1. Install the client library" - } - ], - "resultChars": 389763 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described repeated HTTP 503 gateway responses throughout the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing unrelated issues." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the edge/API gateway/platform layer rather than function code, and grounds this in API/gateway 503s alongside successful edge-function executions, plus distinguishes the avatar-upload 500 as a separate function-level error." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including pulling fuller traces for failing request windows, checking Supabase edge/runtime incidents or rollouts, adding retry/backoff, and investigating a specific isolated 500 code path." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for Data API, kept RLS enabled, created authenticated SELECT policy scoped to user_id = auth.uid(), and created authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS bookmarks anon authenticated GRANT\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", - "title": "Database Advisor: Lint 0017_foreign_table_in_api" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", - "title": "Database Advisor: Lint 0013_rls_disabled_in_public" - } - ], - "resultChars": 58013 - } - ] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push --db-url \"$DB_URL\"` (#42), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio history by adding local migration `supabase/migrations/20240115000000_add_profile_bio.sql` (#36-37), after which `supabase migration list --db-url` showed local/remote aligned (#39, #45). Only read-only psql inspection observed; no prohibited workaround." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"composite index query optimization order by desc limit user_id created_at\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" - } - ], - "resultChars": 18373 - } - ] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policy auth.uid select using with check memberships supabase\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - } - ], - "resultChars": 75855 - } - ] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user fbab494f-6a4b-4376-b26a-10e30c44a54d, signUp returned {\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": false, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": false - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": false - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 1 -> 2" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "Supabase pg_cron schedule cron.schedule syntax official docs", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq create queue function create queue", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions" - } - ] - }, - { - "source": "web_search", - "query": "'pop(' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs cron.schedule pg_cron supabase example", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com supabase pg_cron cron.schedule supabase migration", - "pages": [] - }, - { - "source": "web_search", - "query": "'if not exists' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pg_cron unschedule cron.schedule idempotent", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq message_record msg_id message field", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs edge function config.toml verify_jwt false", - "pages": [] - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt auth getUser service role private notes\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 39139 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"edge function get user auth supabase service role apikey header SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 56285 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-alpha.pdf, 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket, RLS remains enabled, authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix, and supabase-js uses createSignedUrl with expiry." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage.foldername policy storage.objects auth.uid bucket private files\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 25446 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "4 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the tenant isolation flaw, not `notes`, and treats pgTAP verification as the signal after adding isolation tests. It describes fixing `posts` RLS so authenticated users cannot read other organizations' posts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com gte-small 384 Supabase AI Session dimension", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgvector Supabase vector search match function", - "pages": [] - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": false, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails because Prometheus uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret. App scrape and endpoint are otherwise preserved." - }, - { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes env setup, restart/recreate Compose, and Prometheus target verification, but it does not provide steps to create the Secret API key in Supabase or place a matching secret file. The setup uses .env rather than the required secret file, so the secret setup requirement is not met." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"metrics prometheus project metrics scrape endpoint\", limit: 10) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 29978 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Function environment variable deploy secrets WEATHER_API_KEY supabase functions secrets set\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#changing-compute-sizes", - "title": "Changing compute sizes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#recommended-api-keys", - "title": "Recommended API keys" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#nano-compute-instance", - "title": "Nano compute instance" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#launching-projects", - "title": "Launching projects" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#platform-kit", - "title": "Platform kit" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#claim-flow", - "title": "Claim flow" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#disaster-recovery-for-production", - "title": "Disaster recovery for production" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#security-checks-for-production", - "title": "Security checks for production" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#merge-all-changes", - "title": "Merge all changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#deploying-edge-functions", - "title": "Deploying Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#add-seed-data", - "title": "Add seed data" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#reverting-changes", - "title": "Reverting changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#create-a-restore-point", - "title": "Create a restore point" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#make-database-changes", - "title": "Make database changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#creating-a-dev-branch", - "title": "Creating a DEV branch" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#development-workflow", - "title": "Development workflow" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#configuration-changes", - "title": "Configuration changes" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#debugging-projects", - "title": "Debugging projects" - } - ], - "resultChars": 164975 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions invoke URL functions/v1 project ref invoke runtime endpoint\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 93047 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Management API edge functions get function by slug project ref functions/{function_slug}\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function", - "title": "Retrieve a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-delete-a-function", - "title": "Delete a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-update-a-function", - "title": "Update a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-list-all-functions", - "title": "List all functions" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function-body", - "title": "Retrieve a function body" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-create-a-function", - "title": "Create a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-deploy-a-function", - "title": "Deploy a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-bulk-update-functions", - "title": "Bulk update functions" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-get-project-function-combined-stats", - "title": "Gets a project's function combined statistics" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" - } - ], - "resultChars": 18063 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query:\"get_project_url project url supabase management api\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" - }, - { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - } - ], - "resultChars": 69182 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "Supabase self-hosting docker compose official secrets env file", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/supabase docker generate-keys.sh .env.example", - "pages": [] - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The answer correctly diagnoses the soft-delete bug, deletes the auth user so sessions/refresh tokens cascade, adds RLS that blocks stale JWTs on the data path, and correctly explains publishable vs secret keys. However, it does not clearly state the required caveat that access tokens are stateless JWTs not recalled by deletion/revocation and that purely local validation such as getClaims/custom JWT middleware will continue accepting them until expiry." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs publishable secret keys supabase frontend secret keys rls", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" - } - ] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/auth/jwts", - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts" - } - ] - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although the assistant correctly attributes the 503s to the gateway/Edge invocation path and grounds this in valid observations, it recommends redeploying the functions as a remediation, which the rubric explicitly lists as a fail condition." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps including redeploying Edge Functions, checking Supabase incidents/status, adding retry/backoff, and opening a support ticket with exact timestamps and deployment IDs." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id, including WITH CHECK for INSERT. Extra update/delete owner policies do not violate the rubric." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through Supabase CLI with `supabase db push --db-url 'postgresql://postgres:postgres@172.17.0.1:41925/postgres?sslmode=disable'` (#34), which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#31), after which `supabase migration list` showed local and remote matched for `20240115000000` (#32), allowing the push to proceed. Only read-only `psql` inspections were used; no prohibited direct SQL mutation or prepared-statement workaround was seen." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 54178c78-bfc4-4a47-8331-65384278fdef, signUp returned {\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 80822 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog auth mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|supabase-js' | sed -n '1,220p'; printf '%s\\\\n' '--- auth.mjs ---'; cat app/src/auth.mjs; printf '%s\\\\n' '--- app directory ---'; find app -maxdepth 3 -type f -o -type l | sort | xargs -r ls -l; printf '%s\\\\n' '--- service status ---'; supabase status -o env\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1410 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data display_name\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-verifyotp" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-passkey-list" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signup" - } - ], - "resultChars": 19311 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript auth getUser current user select maybeSingle profiles table\", limit: 8) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-getuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/javascript/select" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" - } - ], - "resultChars": 7456 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CLAUDE SUPABASE ---'; sed -n '1,240p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- CLAUDE POSTGRES ---'; sed -n '1,220p' .claude/skills/supabase-postgres-best-practices/SKILL.md; printf '%s\\\\n' '--- SECURITY REFS ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg 'security|schema'; printf '%s\\\\n' '--- RLS REFERENCES ---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null; cat .agents/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null; printf '%s\\\\n' '--- CLI VERSION ---'; supabase --version; printf '%s\\\\n' '--- GIT ---'; git status --short --branch 2>/dev/null || true; printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row level|rls|local development|cli' | head -n 120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8747 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Data API expose table grant anon authenticated RLS select policy authenticated local development CLI migrations db reset\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" - } - ], - "resultChars": 124182 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase declarative database schemas generate migration db diff local\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - } - ], - "resultChars": 82866 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|declarative|migration|db diff' | head -80; printf '\\\\n--- CONFIG ---\\\\n'; sed -n '1,220p' supabase/config.toml; printf '\\\\n--- SCHEMA ---\\\\n'; cat supabase/schemas/products.sql; printf '\\\\n--- MIGRATION ---\\\\n'; cat supabase/migrations/20260101000000_create_products.sql; printf '\\\\n--- STATUS ---\\\\n'; supabase status || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8129 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function cron schedule every minute\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 44996 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Cron schedule database function cron.schedule pgmq send queue\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 35015 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|cron|edge function' | head -n 160 || true; printf '%s\\\\n' '--- CONFIG RELEVANT ---'; rg -n '\"'^'\"\\\\[|enabled|verify_jwt|additional_search_path|port' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION HEAD ---'; sed -n '1,100p' supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION TAIL ---'; tail -n 80 supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION SYMBOLS ---'; rg -n 'create extension|pgmq.create|cron.schedule|pgmq_public|create schema' supabase/migrations/20260101000000_enable_cron_and_queues.sql\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 19533 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY environment variables local serve createClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 40557 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested foreign tables service role secret key createClient Node\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" - } - ], - "resultChars": 22127 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|supabase-js|secret key|postgrest|data api' || true; printf '%s\\\\n' '--- root listing ---'; ls -la; printf '%s\\\\n' '--- app listing ---'; find app -maxdepth 2 -type f -print -exec sed -n '1,80p' {} \\\\;; printf '%s\\\\n' '--- node/npm ---'; node --version; npm --version; npm root -g; printf '%s\\\\n' '--- installed candidates ---'; find . -maxdepth 3 -type d \\\\( -name '@supabase' -o -name 'node_modules' \\\\) -print; printf '%s\\\\n' '--- supabase status env ---'; supabase status -o env\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1554 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested foreign tables createClient secret key Node backend order results\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/api/quickstart", - "title": "Build an API route in less than 2 minutes." - } - ], - "resultChars": 27813 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/package.json ---'; cat app/package.json; printf '%s\\\\n' '--- git status ---'; git status --short; printf '%s\\\\n' '--- supabase status ---'; supabase status -o env; printf '%s\\\\n' '--- relevant changelog scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|secret key|supabase-js|postgrest|data api' | head -80\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 15363 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restore pg_restore local Supabase binary dump roles ownership extensions\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", - "title": "Custom Postgres Extensions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 50510 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient service role getUser RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - } - ], - "resultChars": 73053 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth getUser Authorization header SUPABASE_ANON_KEY createClient\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 96751 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking change Edge Functions auth API keys", - "pages": [] - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"29d6a26d-c31f-496e-a486-a4bd4a0adeee\",\"metric\":\"steps_b_msj16f6m\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification authorization header service role apikey header Deno serve createClient auth getUser\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - } - ], - "resultChars": 60939 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG BREAKING TAGS ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|secret key|service.role' | head -80; printf '%s\\\\n' '--- PROJECT FILES ---'; find .. -name AGENTS.md -print; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sort | sed -n '1,240p'; printf '%s\\\\n' '--- SUPABASE CONFIG ---'; test -f supabase/config.toml && sed -n '1,240p' supabase/config.toml || true; printf '%s\\\\n' '--- MIGRATION MATCHES ---'; rg -n \\\"user_stats|create policy|enable row level security\\\" supabase . --glob '*.sql' --glob '\"'!node_modules'\"' || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 24755 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions verify_jwt false custom JWT validation apikey service role key Authorization header\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 121781 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies auth.uid organization membership security definer performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 58492 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com changelog row level security breaking change Supabase", - "pages": [] - } - ] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-alpha.pdf, 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, keeps storage.objects RLS enabled, adds authenticated SELECT and INSERT policies scoped to bucket and user-owned path via auth.uid(), and provides supabase-js createSignedUrl with a 15-minute expiry. No fail conditions present." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl private bucket upload download\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 46298 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|storage|signed url|rls' | head -80\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14020 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn private bucket signed URL\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - } - ], - "resultChars": 25918 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "36 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw: the policy omits `org_id`, allowing tenant A to see posts from tenant B. It grounds this in reproduced query results and adds pgTAP coverage. It does not blame `notes` instead or dismiss test results." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security testing policies auth.uid tenant isolation pgTAP\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 56902 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- skill relevant sections ---'; rg -n \\\"RLS|policy|test|docs|changelog\\\" .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- security references ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|index'; printf '%s\\\\n' '--- read security rules ---'; cat .claude/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null || true; cat .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null || true; printf '%s\\\\n' '--- changelog relevant ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row.level|rls|pgtap' | head -n 120; printf '%s\\\\n' '--- workspace ---'; find . -name AGENTS.md -print; ls -la; printf '%s\\\\n' '--- repo files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,240p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 17714 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match documents RLS HNSW vector extension Edge Functions\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - } - ], - "resultChars": 52858 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions gte-small vector dimensions 384 match_document_sections\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" - } - ], - "resultChars": 110370 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog pgvector Edge Functions semantic search", - "pages": [] - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Prometheus preserves the app job and adds a deployable Supabase scrape over HTTPS to /customer/v1/privileged/metrics for evalshostedprojectxy.supabase.co using basic_auth with password_file. docker-compose wires the matching password file via a Compose secret." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, storing it in the Compose secret file path, recreating the stack, and verifying via Prometheus targets plus a direct metrics API curl check." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics endpoint customer v1 privileged metrics service_role basic auth hosted project\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - } - ], - "resultChars": 20081 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'metrics|prometheus|breaking-change' | head -80\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3574 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets environment variables WEATHER_API_KEY deploy CLI invoke CORS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 64693 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions deploy secrets set env-file Deno.env.get invoke no verify jwt browser CORS\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - } - ], - "resultChars": 82037 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT keys Vault logflare pooler official\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 131466 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- changelog tags relevant to self-hosting ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -n -m 30 'breaking-change|self-host|docker|auth key|api key' || true\nrm -rf /tmp/supabase-upstream\ngit clone --depth 1 --filter=blob:none --sparse https://github.com/supabase/supabase.git /tmp/supabase-upstream >/tmp/supabase-clone.log 2>&1\ngit -C /tmp/supabase-upstream sparse-checkout set docker\nprintf '%s\\\\n' '--- upstream revision ---'\ngit -C /tmp/supabase-upstream rev-parse HEAD\ngit -C /tmp/supabase-upstream log -1 --format='%cI %s'\nprintf '%s\\\\n' '--- docker files ---'\nfind /tmp/supabase-upstream/docker -maxdepth 3 -type f | sed 's#/tmp/supabase-upstream/docker/##' | sort | sed -n '1,240p'\nprintf '%s\\\\n' '--- docker changelog head ---'\nsed -n '1,220p' /tmp/supabase-upstream/docker/CHANGELOG.md\nprintf '%s\\\\n' '--- env variables ---'\nsed -n '1,260p' /tmp/supabase-upstream/docker/.env.example\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 29108 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.users and RLS checks to close the data path for stale JWTs, consistently explains that JWTs remain cryptographically valid until exp while data access is blocked by the implemented RLS, and correctly distinguishes publishable/frontend/RLS keys from secret/backend/bypass-RLS keys." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token session revoke refresh token admin deleteUser signOut scope global\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 68746 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---RLS---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|auth|jwt|api key|publishable|secret' | head -80\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14841 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key RLS bypass service_role anon new API keys sb_publishable sb_secret\", limit: 10) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 137700 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime despite SUBSCRIBED channel, fixed with ALTER PUBLICATION ADD TABLE public.orders, preserved courier_locations and RLS/policies." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!dist'\"' | sed -n '1,240p'; printf '%s\\\\n' '--- realtime/order refs ---'; rg -n \\\"postgres_changes|orders|courier|supabase_realtime|publication\\\" -g '\"'!node_modules'\"' -g '\"'!dist'\"' . || true; printf '%s\\\\n' '--- changelog realtime refs ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres_changes|publication\\\" | head -40 || true\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 28635 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication add table postgres_changes RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 92624 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as affected and described eight recurring 503 responses across 07:00–12:00 UTC on 2026-04-28, matching the required pattern." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "The assistant explicitly attributes the recurring image-transform 503s to the API Gateway/Edge Runtime/platform layer before invocation, not function code. This is grounded in valid observations: the 503s reached the gateway but had no Edge Function execution records, nearby invocations succeeded, and it distinguishes the unrelated avatar-upload 500 as an application-level error. It also advises not to redeploy or modify function/storage/database config." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with the project, UTC window, and request IDs, plus capturing specific failure metadata and adding retries." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog storage April 2026", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 gateway before invocation logs troubleshooting retry transient errors\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 43053 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"function_edge_logs execution_id function_id internal_failure 503\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", - "title": "Edge Function 500 error response" - } - ], - "resultChars": 32009 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response" - } - ] - } - ] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts. No permissive/public/anon policies or RLS disabling." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---PERF---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '\\\\n---PRIVILEGES---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|data api|row level|rls' | head -120\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8535 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Data API table grants authenticated RLS select insert policy auth.uid bookmarks\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 51520 - } - ] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "PASS. The agent added the missing local bio migration file `supabase/migrations/20240115000000_add_profile_bio.sql`, which reconciled the orphan remote history entry for CLI pushes. The pending avatar migration was then applied through a real `supabase db push` in command #10, whose output includes `Finished supabase db push` (and #11 confirms remote history contains `20240220000000 | add_avatar_url`). No prohibited direct-SQL mutation or prepared-statement workaround was seen." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair linked project deploy migrations\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 61657 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- CHANGELOG BREAKING CHANGES ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|migration|db push' | head -120 || true\nprintf '%s\\\\n' '--- GIT STATUS ---'\ngit status --short --branch 2>&1 || true\nprintf '%s\\\\n' '--- SUPABASE FILES ---'\nfind supabase -maxdepth 4 -type f -print | sort\nprintf '%s\\\\n' '--- CONFIG ---'\nsed -n '1,260p' supabase/config.toml 2>/dev/null || true\nprintf '%s\\\\n' '--- MIGRATIONS ---'\nfor f in supabase/migrations/*.sql; do echo \\\"===== \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done\nprintf '\"'%s\\\\n' '--- LOCAL LINK METADATA ---'\nfind supabase/.temp -maxdepth 2 -type f -print -exec sh -c 'echo \\\"--- \"'$1\"; cat \"$1\"'\"' _ {} \\\\; 2>/dev/null || true\nprintf '%s\\\\n' '--- CLI / ENV NAMES ---'\nsupabase --version\nprintf 'SUPABASE vars: '; env | cut -d= -f1 | rg '\"'^(SUPABASE|POSTGRES|PG)'\"' | sort | tr '\\\\n' ' '; echo\nprintf '%s\\\\n' '--- PROFILE ---'\ncat .supabase-eval-profile.yaml\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 478 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase migration history remote migration versions not found local migration repair db pull migration fetch\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/new-branch-doesnt-copy-database", - "title": "Why are my Supabase branches empty?" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/branch-in-migrations-failed-status", - "title": "Troubleshooting MIGRATIONS_FAILED: missing tables or an incomplete schema on your branch" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-fetch", - "title": "Fetch migration files from history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - } - ], - "resultChars": 58381 - } - ] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|postgres|index|database\" | head -40'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7495 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres query optimization composite index ORDER BY WHERE EXPLAIN\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - } - ], - "resultChars": 63315 - } - ] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g 'package.json' -g '*.sql' | sed -n '1,240p'; printf '%s\\\\n' '--- changelog security/rls mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row level|rls|policy|security'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4349 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth uid index policy performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors", - "title": "Performance and Security Advisors" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 48168 - } - ] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 14e42a99-dd08-453c-a361-a8b7871b40cc, signUp returned {\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getSession select single profiles\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - } - ], - "resultChars": 166581 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated users SELECT policy anon no rows migrations seed local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 95821 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 54403 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send messages Edge Function read delete pgmq_public cron schedule every minute\", limit: 8) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - } - ], - "resultChars": 39939 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Queues Quickstart pgmq.create cron.schedule SQL schedule database jobs\", limit: 6) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - } - ], - "resultChars": 42534 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript supabase-js select foreign tables nested relationships service role Node backend\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - } - ], - "resultChars": 45250 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI restore pg_dump custom format local database migrate existing Postgres\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - } - ], - "resultChars": 42818 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser createClient SUPABASE_ANON_KEY RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 32789 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7bc5ace7-c5be-42dc-ba44-d5537f012dc1\",\"metric\":\"steps_b_msj0yta6\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user JWT Authorization header service role secret key apikey header verify_jwt false getClaims\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 46978 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key apikey header Edge Functions secret key service_role authorization\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 177206 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server Edge Functions API key environment SUPABASE_SECRET_KEYS authenticate request\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - } - ], - "resultChars": 90248 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-alpha.pdf, 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket (public=false), authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix with WITH CHECK for upload, no RLS disabling/permissive public policies, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage access control RLS policies storage.objects foldername auth.uid signed URL createSignedUrl upload download private bucket\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - } - ], - "resultChars": 22016 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "5 passed, 2 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant rows for authenticated users, distinguishes `notes` as correctly isolated, and grounds the conclusion in failing pgTAP results." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"database testing pgTAP row level security auth.uid tests\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - } - ], - "resultChars": 70562 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase AI gte-small embedding dimensions pgvector semantic search match_documents RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 63278 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"gte-small 384 dimensions Supabase.ai.Session\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" - } - ], - "resultChars": 76269 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and Docker Compose wires the secret to /run/secrets." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase secret API key, writing it to the matching Docker secret file, recreating the Compose Prometheus service, and verifying via Prometheus targets or an up{job=\"supabase\"} query." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus project metrics endpoint customer v1 privileged metrics service_role basic auth observability\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" - } - ], - "resultChars": 82042 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions secrets environment variables Deno.env deploy functions invoke no verify jwt\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 37288 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose install docker .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD SECRET_KEY_BASE VAULT_ENC_KEY\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", - "title": "Variables added to the environment but provider still not working" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-a-templates-directory", - "title": "Step 1: Create a templates directory" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#overview", - "title": "Overview" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#authentication-email-templates", - "title": "Authentication email templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml", - "title": "Step 2: Update docker-compose.yml" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#what-this-configuration-does", - "title": "What this configuration does" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers", - "title": "Step 3: Restart containers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#notification-email-templates", - "title": "Notification email templates" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example-1", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-the-templates-directory", - "title": "Step 1: Create the templates directory" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml-1", - "title": "Step 2: Update docker-compose.yml" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers-1", - "title": "Step 3: Restart containers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", - "title": "Signature mismatch errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", - "title": "Test with the AWS CLI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", - "title": "Test with rclone" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", - "title": "How to configure an S3 backend" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", - "title": "Using RustFS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", - "title": "Using MinIO" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", - "title": "Using AWS S3" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", - "title": "S3-compatible providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", - "title": "Session token" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", - "title": "TUS upload errors on Cloudflare R2" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", - "title": "Permission denied on uploads" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", - "title": "Upload URLs point to localhost" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", - "title": "Enable the S3 protocol endpoint" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", - "title": "New deployment with Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", - "title": "Run the upgrade" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", - "title": "Restoring from a manual backup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", - "title": "Postgres 17 fails to start with a leftover db-config volume" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", - "title": "Disk space issues during upgrade" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", - "title": "Services fail to connect after upgrade" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", - "title": "pgsodium / Supabase Vault errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", - "title": "pg_upgrade fails with replication slot errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", - "title": "Upgrade process details" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", - "title": "Custom Postgres configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", - "title": "Rollback" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", - "title": "After the upgrade" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", - "title": "Extensions removed in Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", - "title": "Requirements" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", - "title": "Create a backup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", - "title": "What the upgrade does" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", - "title": "Upgrade an existing Postgres 15 deployment" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" - } - ], - "resultChars": 450330 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Docker self-hosting quick start Linux git clone supabase docker copy .env.example generate-keys.sh\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#how-to-opt-out", - "title": "How to opt out" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#telemetry", - "title": "Telemetry" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#stopping-local-services", - "title": "Stopping local services" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#access-your-projects-services", - "title": "Access your project's services" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#running-supabase-locally", - "title": "Running Supabase locally" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#updating-the-supabase-cli", - "title": "Updating the Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#beta-channel", - "title": "Beta channel" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#installing-the-supabase-cli", - "title": "Installing the Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#learn-more", - "title": "Learn more" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", - "title": "Community-driven projects" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", - "title": "About self-hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", - "title": "How self-hosted Supabase differs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", - "title": "Your responsibilities when self-hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#telemetry", - "title": "Telemetry" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", - "title": "Support and community" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", - "title": "Enterprise self-hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#get-started", - "title": "Get started" - } - ], - "resultChars": 353761 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete-only root cause, implements auth user deletion plus RLS live-account gate, correctly explains JWTs remain cryptographically valid while protected data path has no post-commit window due to RLS, and correctly distinguishes publishable frontend keys from secret backend-only RLS-bypassing keys." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { keys: searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } } } sessions: searchDocs(query: \"delete user access token remains valid until expiry session sign out JWT revoke\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } delete: searchDocs(query: \"delete user admin API JWT remains valid delete account\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" - } - ], - "resultChars": 230051 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key sb_publishable sb_secret RLS\", limit: 10) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", - "title": "Build a User Management App with Vue 3" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - } - ], - "resultChars": 225824 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"sign out JWT remains valid until expires revoke refresh tokens access token\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", - "title": "How do I make the cookies HttpOnly?" - } - ], - "resultChars": 103031 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"admin deleteUser delete user existing JWT access token valid until expiry\", limit: 6) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 33037 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified that orders was absent from the supabase_realtime publication despite the channel being subscribed, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations and RLS/policies, and did not blame or alter RLS/client/networking." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines", - "title": "Set up Pipelines" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - } - ], - "resultChars": 121815 - } - ] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as affected and described eight intermittent HTTP 503s across 07:00–12:00 UTC on 2026-04-28, noting the recurring gateway pattern." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer before function execution, grounded in gateway logs without corresponding Edge Function logs and nearby successful invocations on the same deployment. It also distinguishes the separate avatar-upload 500 as a runtime-level issue." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "Recommended concrete next steps, including escalating to Supabase support with request IDs and timestamps, plus retry and alerting actions." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\" --yes` in action #11, which succeeded and left remote migration history containing `20240220000000 | add_avatar_url`. The orphan bio migration was reconciled by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #10, then running the same Supabase CLI push. No forbidden direct SQL mutation or prepared-statement workaround was seen; psql usage was read-only inspection." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history mismatch repair deploy hosted project\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - } - ], - "resultChars": 96189 - } - ] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user cbbc700e-775b-4489-94a7-6a2ce1ab7257, signUp returned {\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata display_name options data\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - } - ], - "resultChars": 36311 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-auth-001-email-password-flow.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-002-declarative-schema.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queue cron schedule edge function pop read delete messages\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 68777 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 \"breaking\" | grep -i -B1 -A1 -E \"cron|pgmq|queue|edge function\" | head -40; echo \"---done---\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1492 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/build-dataapi-001-relational-report.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify JWT auth getUser getClaims authorization header\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 52010 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 7275 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"9a247c74-daaa-4e1f-9bd6-4c24ce42abcd\",\"metric\":\"steps_b_msj163uc\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 94601 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions createServerClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", - "title": "Migrating to the SSR package from Auth Helpers" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 59816 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 7275 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-alpha.pdf, 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or permissive/public policies, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage createSignedUrl supabase-js expiresIn private bucket\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - } - ], - "resultChars": 44528 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "3 passed, 1 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with broken tenant isolation, explains that authenticated members can read posts from orgs they are not members of, and grounds this in the pgTAP result where the `posts` negative test fails. It also correctly states `notes` is isolated and treats test results as authoritative." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections embedding function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - } - ], - "resultChars": 68270 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/ai/semantic-search.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search.md" - } - ], - "resultChars": 12827 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase Metrics API scrape for the project target, correct metrics path, Basic Auth with password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, stack restart/reload, and concrete verification via Prometheus targets plus direct curl check. Endpoint/auth and mounted secret path are consistent, with no hardcoded secret." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"metrics API prometheus endpoint scrape\") { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 29060 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" - } - ], - "resultChars": 30709 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - } - ] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer diagnoses the soft-delete-only bug, implements real auth user/session/refresh-token removal, explains the remaining stale JWT access-token window consistently with its fix (including mitigation), and correctly distinguishes publishable frontend keys from secret/server keys that bypass RLS." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - } - ], - "resultChars": 101422 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime publication as root cause, added public.orders to the existing publication, and preserved courier_locations, RLS, and policies. Did not blame client/RLS/networking as root cause or weaken security." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and explicitly described the recurring 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it cites a valid observation (503s only in gateway logs with no function execution logs), it ultimately attributes the primary cause to the function's unpinned npm dependency/worker boot and recommends pinning/redeploying the functions, rather than attributing the 503s to the gateway/platform layer in front of the function." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: pinning Edge Function dependencies, vendoring dependencies with lockfile checks, checking npm publish/registry history for the affected time window, and adding gateway 5xx alerting." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions 503 error npm dependencies pin version deno.json import map cold start\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-dependency-analysis", - "title": "Edge Function dependency analysis" - }, - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/importing-stripe-or-other-modules-from-esmsh-on-deno-edge-functions-throws-an-error-TmbB5p", - "title": "Importing Stripe or other modules from esm.sh on Deno Edge Functions throws an error" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 43609 - } - ] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as the cause of empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid(), including WITH CHECK for inserts." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "PASS: The avatar_url migration was applied through the Supabase CLI with `supabase db push` in action #26, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_bio.sql` in actions #22-#24, after which `supabase migration list` (#25/#28) showed local and remote history aligned. Read-only psql inspections were used, but no prohibited direct SQL mutation or prepared-statement workaround was seen." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 21c579b2-ae12-44ca-83f7-24fe17e54af5, signUp returned {\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-auth-001-email-password-flow.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 3 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "3 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues consume messages with edge function read delete\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 68440 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href subsections { nodes { title content } } } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href ... on Guide { subsections { nodes { title content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - } - ], - "resultChars": 22663 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"b33c6f83-e1ca-49a7-8520-64ff05e415e9\",\"metric\":\"steps_b_msj15kas\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query: \"edge function verify JWT get user from access token service role bypass RLS\") {nodes {title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getuser", - "title": "user()" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0012_auth_allow_anonymous_sign_ins", - "title": "Database Advisor: Lint 0012_auth_allow_anonymous_sign_ins" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 271984 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-rls-003-org-roles-permissions", - "stage": "build", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on documents", - "passed": true - }, - { - "name": "viewer sees only documents in their org", - "passed": true - }, - { - "name": "viewer cannot insert", - "passed": true - }, - { - "name": "editor can insert own org document", - "passed": true - }, - { - "name": "editor can update own document", - "passed": true - }, - { - "name": "editor cannot update another user's document", - "passed": true - }, - { - "name": "editor cannot delete another user's document", - "passed": true - }, - { - "name": "admin can update any document in their org", - "passed": true - }, - { - "name": "admin can delete any document in their org", - "passed": true - }, - { - "name": "admin cannot affect another org", - "passed": true - }, - { - "name": "WITH CHECK blocks editor from moving document to another org", - "passed": true - }, - { - "name": "editor can still see their own org's roster", - "passed": true - }, - { - "name": "cannot see another org's membership roster", - "passed": true - }, - { - "name": "a viewer role in one org doesn't grant admin power in another org", - "passed": true - }, - { - "name": "multi-org user can act as admin in the org where they hold that role", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. People should only be able to see who's in their own org.", - "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-rls-003-org-roles-permissions.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-alpha.pdf, 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets the rubric: creates a private user-files bucket, adds authenticated owner-scoped SELECT and INSERT policies on storage.objects using the user-id path segment, does not disable RLS or use public access, and provides supabase-js createSignedUrl code with an expiry." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "10 passed, 2 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from organizations they do not belong to, and grounds the conclusion in pgTAP failures. It also correctly notes `notes` is isolated and treats the test results as authoritative." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and docker-compose mounts the secrets directory containing the password file." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes correct live setup steps: create/copy a Supabase Secret API key, write it to the mounted secret file matching prometheus.yml, restart/recreate or reload the Compose stack, and verify via Prometheus targets and PromQL/Grafana. Endpoint/auth and secret setup are consistent, with no hardcoded real secret." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape hosted project observability\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 19334 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management api create project api key secret sb_secret\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", - "title": "Creates a new API key for the project" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 63685 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": true - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth/session/refresh-token revocation by deleting sessions and auth user, hardens RLS to close the data API stale-JWT window, and explains the remaining stateless JWT validity caveat consistently. It also correctly distinguishes publishable frontend keys with RLS from secret server-only keys that bypass RLS." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identifies orders missing from supabase_realtime publication as root cause despite SUBSCRIBED status, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verifies existing courier_locations remains, and does not weaken RLS/policies or blame client/networking." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant clearly identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 gateway responses on 2026-04-28 between 07:00Z and 12:00Z, while correctly distinguishing unrelated old billing-webhook 503s." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/edge platform layer, not function code, and grounds this in valid evidence: gateway-only 503s with no failed function invocations, successful nearby executions on same deployment/version, and distinction from a function-level avatar-upload 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project ref, request IDs, and time window, checking platform incident/status, adding retries, and setting up alerting." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to auth.uid() = user_id using USING and WITH CHECK." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar_url was applied through `supabase db push` in #17, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in #15, after which Supabase CLI migration list showed local and remote aligned. No prohibited workaround or direct mutation was used." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "opencode-kimi-k3/build-cli-001-bootstrap-app.json" } ]