feat: o11y eval suites

evals/o11y-0001-resolve-performance-missing-index/EVAL.ts

@@ -0,0 +1,81 @@
+import {
+  type CheckResult,
+  type ToolEvalContext,
+  type ToolScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+
+// Fault: public.orders.customer_id is unindexed. The fix creates an index on
+// customer_id so the per-customer lookup uses it instead of a sequential scan.
+
+const scorer: ToolScorer = async (ctx) => {
+  try {
+    const checks: CheckResult[] = [
+      await checkIndexExists(ctx),
+      await checkQueryPlanUsesIndex(ctx),
+      await checkInsertsStillWork(ctx),
+    ];
+
+    return { passed: checks.every((c) => c.passed), checks };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        { name: 'scorer evaluated index fix', passed: false, notes: msg },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+/** An index leading with customer_id must exist on public.orders. */
+async function checkIndexExists(ctx: ToolEvalContext): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    SELECT indexname, indexdef
+    FROM pg_indexes
+    WHERE schemaname = 'public' AND tablename = 'orders';
+  `);
+  const hasIndex = rows.some((r) =>
+    /ON\s+(?:public\.)?orders\s+.*\(\s*customer_id/i.test(String(r.indexdef))
+  );
+  return {
+    name: 'index on orders(customer_id) exists',
+    passed: hasIndex,
+    notes: rows.map((r) => r.indexname).join(', '),
+  };
+}
+
+/** The per-customer lookup should plan with an index and no sequential scan. */
+async function checkQueryPlanUsesIndex(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    EXPLAIN SELECT id, total_cents, created_at
+    FROM public.orders
+    WHERE customer_id = 42;
+  `);
+  const plan = rows.map((r) => Object.values(r).join(' ')).join('\n');
+  return {
+    name: 'query plan uses an index and avoids a sequential scan',
+    passed:
+      /(Index Scan|Index Only Scan|Bitmap Index Scan)/i.test(plan) &&
+      !/Seq Scan on orders/i.test(plan),
+    notes: plan,
+  };
+}
+
+async function checkInsertsStillWork(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    INSERT INTO public.orders (customer_id, total_cents)
+    VALUES (1, 999)
+    RETURNING id;
+  `);
+  return {
+    name: 'inserts still work after the fix',
+    passed: rows.length === 1,
+  };
+}

evals/o11y-0001-resolve-performance-missing-index/PROMPT.md

@@ -0,0 +1,18 @@
+---
+stage: resolve
+suite: regression
+interface: mcp
+product:
+  - database
+topic:
+  - sql
+  - observability
+motivation: apps/docs/content/troubleshooting/how-to-interpret-and-explore-the-postgres-advisor.mdx
+---
+
+Loading a customer's orders has gotten slow as the `orders` table has grown, and
+CPU spikes when the app does it. Can you work out why that lookup is slow and
+make the database change needed to speed it up? Make sure normal inserts into
+`orders` still work afterward.
+
+End your turn with a short summary of what you changed and why.

evals/o11y-0001-resolve-performance-missing-index/remote/project.sql

@@ -0,0 +1,26 @@
+-- Broken starting state (probe: performance-missing-index).
+-- orders.customer_id is unindexed, so the hot per-customer lookup does a
+-- sequential scan over a large table.
+CREATE TABLE public.customers (
+  id serial PRIMARY KEY,
+  email text UNIQUE NOT NULL
+);
+
+CREATE TABLE public.orders (
+  id bigserial PRIMARY KEY,
+  customer_id int NOT NULL REFERENCES public.customers(id),
+  total_cents int NOT NULL DEFAULT 0,
+  created_at timestamptz DEFAULT now()
+);
+
+INSERT INTO public.customers (email)
+SELECT 'user' || g || '@example.com'
+FROM generate_series(1, 500) AS g;
+
+-- 50k orders, no index on customer_id.
+INSERT INTO public.orders (customer_id, total_cents)
+SELECT (floor(random() * 500) + 1)::int,
+       (floor(random() * 10000) + 1)::int
+FROM generate_series(1, 50000);
+
+ANALYZE public.orders;

evals/o11y-0002-resolve-security-auth-users-exposed/EVAL.ts

@@ -0,0 +1,72 @@
+import {
+  type CheckResult,
+  type ToolEvalContext,
+  type ToolScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+
+// Fault: public.user_list is a view over auth.users that exposes PII (email,
+// created_at) to any API caller. The fix drops the view or rewrites it to not
+// directly select from auth.users without restriction.
+
+const scorer: ToolScorer = async (ctx) => {
+  try {
+    const checks: CheckResult[] = [
+      await checkViewDroppedOrSafe(ctx),
+      await checkNoDirectAuthUsersExposure(ctx),
+    ];
+
+    return { passed: checks.every((c) => c.passed), checks };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        { name: 'scorer evaluated auth users exposure fix', passed: false, notes: msg },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+async function checkViewDroppedOrSafe(ctx: ToolEvalContext): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    SELECT table_name, view_definition
+    FROM information_schema.views
+    WHERE table_schema = 'public' AND table_name = 'user_list';
+  `);
+
+  if (rows.length === 0) {
+    return {
+      name: 'public.user_list view removed or does not expose auth.users',
+      passed: true,
+      notes: 'view dropped',
+    };
+  }
+
+  const def = String(rows[0]?.view_definition ?? '');
+  const stillExposesAuthUsers = /auth\.users/i.test(def) && /email/i.test(def);
+  return {
+    name: 'public.user_list view removed or does not expose auth.users',
+    passed: !stillExposesAuthUsers,
+    notes: `view still exists; definition references auth.users email: ${stillExposesAuthUsers}`,
+  };
+}
+
+async function checkNoDirectAuthUsersExposure(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    SELECT grantee, privilege_type
+    FROM information_schema.role_table_grants
+    WHERE table_schema = 'public'
+      AND table_name = 'user_list'
+      AND grantee IN ('anon', 'authenticated');
+  `);
+  return {
+    name: 'anon/authenticated grants on user_list revoked or view is gone',
+    passed: rows.length === 0,
+    notes: rows.length > 0 ? `still granted: ${rows.map(r => `${r.grantee}:${r.privilege_type}`).join(', ')}` : 'no grants',
+  };
+}

evals/o11y-0002-resolve-security-auth-users-exposed/PROMPT.md

@@ -0,0 +1,19 @@
+---
+stage: resolve
+suite: regression
+interface: mcp
+product:
+  - database
+  - auth
+topic:
+  - security
+motivation: apps/docs/content/troubleshooting/database-roles.mdx
+---
+
+A security review flagged that we have a view in our public schema that looks
+like it's exposing user PII from the auth system — emails and created-at
+timestamps — to anyone who can read the API. Can you confirm whether that view
+is leaking data, fix it so user details are no longer exposed via the public
+schema, and confirm it's resolved?
+
+End your turn with a short summary of what you changed and why.

evals/o11y-0002-resolve-security-auth-users-exposed/remote/project.sql

@@ -0,0 +1,7 @@
+-- Broken starting state (probe: security-auth-users-exposed / Splinter lint 0002).
+-- public.user_list is a view over auth.users that exposes user PII (email,
+-- created_at) to any role with access to the public schema via the Data API.
+CREATE OR REPLACE VIEW public.user_list AS
+  SELECT id, email, created_at FROM auth.users;
+
+GRANT SELECT ON public.user_list TO anon, authenticated;

evals/o11y-0003-resolve-security-rls-initplan/EVAL.ts

@@ -0,0 +1,71 @@
+import {
+  type CheckResult,
+  type ToolEvalContext,
+  type ToolScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+
+// Fault: profiles RLS policy calls auth.uid() via a VOLATILE wrapper function,
+// forcing per-row re-evaluation. The fix rewrites the policy to call auth.uid()
+// directly so Postgres uses the efficient initplan path.
+
+const scorer: ToolScorer = async (ctx) => {
+  try {
+    const checks: CheckResult[] = [
+      await checkPolicyCallsAuthUidDirectly(ctx),
+      await checkVolatileWrapperGone(ctx),
+    ];
+
+    return { passed: checks.every((c) => c.passed), checks };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        { name: 'scorer evaluated RLS initplan fix', passed: false, notes: msg },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+async function checkPolicyCallsAuthUidDirectly(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    SELECT policyname, qual
+    FROM pg_policies
+    WHERE schemaname = 'public'
+      AND tablename = 'profiles'
+      AND (cmd = 'SELECT' OR cmd = 'ALL');
+  `);
+
+  const hasDirectAuthUid = rows.some((r) => {
+    const qual = String(r.qual ?? '');
+    return /auth\.uid\(\)/i.test(qual) && !/current_user_id|get_user_id/i.test(qual);
+  });
+
+  return {
+    name: 'SELECT policy calls auth.uid() directly (no volatile wrapper)',
+    passed: hasDirectAuthUid,
+    notes: rows.map(r => `${r.policyname}: ${r.qual}`).join('; '),
+  };
+}
+
+async function checkVolatileWrapperGone(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const { rows } = await ctx.query(stripIndent`
+    SELECT proname
+    FROM pg_proc
+    WHERE proname IN ('current_user_id', 'get_user_id')
+      AND pronamespace = 'public'::regnamespace
+      AND provolatile = 'v';
+  `);
+  return {
+    name: 'VOLATILE wrapper function removed or no longer used in policy',
+    passed: rows.length === 0,
+    notes: rows.length > 0 ? `still exists: ${rows.map(r => r.proname).join(', ')}` : 'none found',
+  };
+}

evals/o11y-0003-resolve-security-rls-initplan/PROMPT.md

@@ -0,0 +1,23 @@
+---
+stage: resolve
+suite: regression
+interface: mcp
+product:
+  - database
+topic:
+  - rls
+  - security
+  - sql
+motivation: apps/docs/content/troubleshooting/slow-queries.mdx
+---
+
+Queries against our `profiles` table have become catastrophically slow at scale
+— we're seeing full table scan times even with RLS policies in place. The
+SELECT policy uses a wrapper function (`current_user_id()`) that is VOLATILE,
+causing Postgres to re-evaluate it for every row instead of once per query.
+
+Please fix the SELECT policy so it calls `auth.uid()` directly (no wrapper
+function), then drop the `current_user_id()` wrapper function since it's no
+longer needed.
+
+End your turn with a short summary of what you changed and why.

evals/o11y-0003-resolve-security-rls-initplan/remote/project.sql

@@ -0,0 +1,21 @@
+-- Broken starting state (probe: security-rls-initplan / Splinter lint 0003).
+-- profiles RLS policy calls auth.uid() via a VOLATILE wrapper function,
+-- forcing per-row re-evaluation (subplan) instead of the efficient single
+-- initplan Postgres uses for the built-in directly.
+CREATE TABLE public.profiles (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  handle text UNIQUE NOT NULL,
+  created_at timestamptz DEFAULT now()
+);
+ALTER TABLE public.profiles ENABLE ROW LEVEL SECURITY;
+
+CREATE OR REPLACE FUNCTION public.current_user_id()
+  RETURNS uuid LANGUAGE sql VOLATILE
+  AS $$ SELECT auth.uid() $$;
+
+CREATE POLICY "profiles_select_own" ON public.profiles
+  FOR SELECT TO authenticated
+  USING (id = public.current_user_id());
+
+INSERT INTO public.profiles (handle)
+SELECT 'user_' || g FROM generate_series(1, 100) g;