From ddef699fb7800f67995637838a3b211682b056df Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Wed, 28 Oct 2026 15:56:41 -0700 Subject: [PATCH 1/5] Add Smallest complete change rule to /go, with eval cases /go's single pass follows the rule, and the stage loop passes it to each stage implementer alongside the judgment-call rule. review-code --fix agents still get neither. ai/evals/smallest-change is an eval-only plugin for `claude plugin eval`. The with arm loads a SessionStart hook that prints rule.md, a copy of the rule that ai/tests/test-eval-rule-sync.sh keeps in sync. The without arm is the tool's no-plugin baseline, so the rule is the only difference. Three scaffolded Python cases grade with hidden tests, a production-line limit, a dependency check, and (in two cases) a check that the change uses the stdlib module or existing helper. Claude-Session: https://claude.ai/code/session_01AovXvYri48wFhqGV3epaMi --- .github/workflows/test.yml | 1 + ai/README.md | 5 ++ .../.claude-plugin/plugin.json | 5 ++ ai/evals/smallest-change/README.md | 47 +++++++++++ ai/evals/smallest-change/evals/.gitignore | 1 + .../evals/csv-export/case.yaml | 33 ++++++++ .../evals/csv-export/grade.json | 6 ++ .../evals/csv-export/hidden/test_hidden.py | 41 +++++++++ .../evals/csv-export/repo/app/__init__.py | 0 .../evals/csv-export/repo/app/report.py | 12 +++ .../evals/csv-export/repo/pyproject.toml | 5 ++ .../evals/csv-export/repo/tests/__init__.py | 0 .../csv-export/repo/tests/test_report.py | 15 ++++ .../evals/csv-export/scaffold.sh | 3 + ai/evals/smallest-change/evals/grade.py | 84 +++++++++++++++++++ .../evals/permalink-reuse/case.yaml | 33 ++++++++ .../evals/permalink-reuse/grade.json | 6 ++ .../permalink-reuse/hidden/test_hidden.py | 26 ++++++ .../permalink-reuse/repo/blog/__init__.py | 0 .../evals/permalink-reuse/repo/blog/models.py | 15 ++++ .../evals/permalink-reuse/repo/blog/text.py | 15 ++++ .../evals/permalink-reuse/repo/pyproject.toml | 5 ++ .../permalink-reuse/repo/tests/__init__.py | 0 .../permalink-reuse/repo/tests/test_models.py | 15 ++++ .../permalink-reuse/repo/tests/test_text.py | 17 ++++ .../evals/permalink-reuse/scaffold.sh | 3 + ai/evals/smallest-change/evals/scaffold.sh | 44 ++++++++++ .../evals/shared-bug-fix/case.yaml | 28 +++++++ .../evals/shared-bug-fix/grade.json | 3 + .../shared-bug-fix/hidden/test_hidden.py | 29 +++++++ .../evals/shared-bug-fix/repo/pyproject.toml | 5 ++ .../shared-bug-fix/repo/shop/__init__.py | 0 .../evals/shared-bug-fix/repo/shop/cart.py | 15 ++++ .../evals/shared-bug-fix/repo/shop/money.py | 10 +++ .../evals/shared-bug-fix/repo/shop/refunds.py | 9 ++ .../shared-bug-fix/repo/tests/__init__.py | 0 .../shared-bug-fix/repo/tests/test_shop.py | 24 ++++++ .../evals/shared-bug-fix/scaffold.sh | 3 + ai/evals/smallest-change/hooks/hooks.json | 14 ++++ ai/evals/smallest-change/rule.md | 15 ++++ ai/skills/go/SKILL.md | 18 +++- ai/skills/go/references/stage-loop.md | 4 +- ai/tests/test-eval-rule-sync.sh | 24 ++++++ 43 files changed, 635 insertions(+), 3 deletions(-) create mode 100644 ai/evals/smallest-change/.claude-plugin/plugin.json create mode 100644 ai/evals/smallest-change/README.md create mode 100644 ai/evals/smallest-change/evals/.gitignore create mode 100644 ai/evals/smallest-change/evals/csv-export/case.yaml create mode 100644 ai/evals/smallest-change/evals/csv-export/grade.json create mode 100644 ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py create mode 100644 ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py create mode 100644 ai/evals/smallest-change/evals/csv-export/repo/app/report.py create mode 100644 ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml create mode 100644 ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py create mode 100644 ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py create mode 100755 ai/evals/smallest-change/evals/csv-export/scaffold.sh create mode 100644 ai/evals/smallest-change/evals/grade.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/case.yaml create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/grade.json create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/blog/__init__.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/tests/__init__.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py create mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py create mode 100755 ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh create mode 100755 ai/evals/smallest-change/evals/scaffold.sh create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/case.yaml create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/grade.json create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py create mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py create mode 100755 ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh create mode 100644 ai/evals/smallest-change/hooks/hooks.json create mode 100644 ai/evals/smallest-change/rule.md create mode 100755 ai/tests/test-eval-rule-sync.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4ff95d6..b24c77ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,7 @@ jobs: ai/skills/wait-for-pr-reviews/scripts/tests/test-portable-skill.sh ai/skills/address-pr-reviews/scripts/tests/test-portable-skill.sh ai/tests/test-plain-writing-contract.sh + ai/tests/test-eval-rule-sync.sh ai/tests/test-ai-installers.sh ai/tests/test-install-claude-automation.sh ai/tests/test-lint-edited-file.sh diff --git a/ai/README.md b/ai/README.md index a4c61038..9ca95bd3 100644 --- a/ai/README.md +++ b/ai/README.md @@ -50,6 +50,10 @@ CI enforces all three rules: the same script with `--check` rejects a copy that When `implementation-planner` marks a plan with two or more stages `**Execution**: staged`, `go` implements it one stage at a time. The planner marks a plan staged only when one session would run out of context before it finished the work, and `go --staged` or `--single-pass` overrides its choice. A fresh agent writes each stage, a second fresh agent checks it against the stage's success criteria and `Check` command, and the parent commits it, so an interrupted run resumes in the stage it stopped in. `implementation-planner` writes the `Check` command and a `## Decisions` section that the stage agents follow. +## Evals + +`ai/evals/` holds `claude plugin eval` suites. [`ai/evals/smallest-change/`](evals/smallest-change/README.md) compares implementation with and without the Smallest complete change rule that `go` gives its implementers. + ## Fresh review processes `go` supports Claude Code and Codex. It passes its current harness explicitly to `skills/go/scripts/run-review.py`, which starts a fresh review in the same checkout. Claude uses print mode; Codex uses `exec`. Both save the review result under the checkout's `.notes/` directory so the parent can resume at validation after a context clear. The runner requires Python 3.11 or later and the selected CLI on `PATH`; it never substitutes the other harness. The installed `review-code` skill must support `REVIEW_CODE_REVIEW_DIR`; update it from its source repository with `bin/setup` if the runner reports an older installation. Review sessions, reports, temporary checkouts, and hook files stay under each attempt in `.notes/go-reviews/`. The Codex child receives no write grants for shared review caches. After success, the parent copies the report into the shared review history, where it survives removal of the implementation worktree. A Claude review runs on the accounts that docket's `config.toml` names in `claude_config_dir` and `claude_fallback_config_dirs`, in the order docket's rotation picks them. The runner skips each account that a usage limit in docket's `limits.jsonl` still applies to. A limit applies to every config directory signed in to the login (one account in one organization) that docket recorded it with. `run-review.py accounts` prints that rotation, the limits docket recorded, and the order a review would try the accounts in now, as JSON, and `usage-all` shows it below each account's usage. Without a docket config, the review runs on the current account. The runner refuses an account that is not signed in or has no `review-code` skill. When an account hits its usage limit before the review edits any files, the runner reviews again on the next account that is signed in to a different login, and then on the current account if the list leaves it out. A run with `--since-run ` copies that earlier run's report into its own attempt and asks `review-code` for an `--append` review of the changes since that report. @@ -83,6 +87,7 @@ ai/skills/wait-for-pr-reviews/scripts/tests/test-pending-reviews.sh ai/skills/wait-for-pr-reviews/scripts/tests/test-portable-skill.sh ai/skills/address-pr-reviews/scripts/tests/test-portable-skill.sh ai/tests/test-plain-writing-contract.sh +ai/tests/test-eval-rule-sync.sh python3 ai/skills/plain-writing/scripts/tests/test_plain_writing_lint.py ai/tests/test-ai-installers.sh ai/tests/test-command-log.sh diff --git a/ai/evals/smallest-change/.claude-plugin/plugin.json b/ai/evals/smallest-change/.claude-plugin/plugin.json new file mode 100644 index 00000000..be50ce30 --- /dev/null +++ b/ai/evals/smallest-change/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "smallest-change-eval", + "version": "0.1.0", + "description": "Eval-only plugin that adds /go's Smallest complete change rule to the session context, so claude plugin eval can compare implementation with and without it." +} diff --git a/ai/evals/smallest-change/README.md b/ai/evals/smallest-change/README.md new file mode 100644 index 00000000..425becb1 --- /dev/null +++ b/ai/evals/smallest-change/README.md @@ -0,0 +1,47 @@ +# Smallest complete change eval + +An eval-only plugin that measures what the Smallest complete change rule in `ai/skills/go/SKILL.md` does to a plain implementation task. It never runs /go, so a run opens no PR and requests no review. + +## What each arm loads + +- **With:** this plugin. Its only content is a `SessionStart` hook that prints `rule.md` into the session context. The plugin has no skill, so the rule is in context on every run, and the result does not depend on whether the model chooses to invoke a skill. /go gives its implementers this rule together with the Judgment calls rule, but this arm gets this rule alone, so the "easy-to-reverse call" that item 1 mentions has no definition here. +- **Without:** the baseline arm that `--ablation with-without` adds. It runs the same prompt in the same scaffolded workspace with no plugin loaded, so the rule is the only difference between the arms. + +`rule.md` is the rule from `ai/skills/go/SKILL.md` without its first sentence, which tells /go which agents get the rule. `ai/tests/test-eval-rule-sync.sh` fails when the two copies differ. + +## Cases + +Each case is a small Python repo where an over-built answer is tempting. + +| Case | Task | What over-building looks like | +| --- | --- | --- | +| `csv-export` | Add `write_csv(rows, path)` | Hand-rolled quoting, or delimiter and encoding options nobody asked for | +| `permalink-reuse` | Add `Post.permalink` | A new slug function when `blog/text.py` already has `slugify` | +| `shared-bug-fix` | Fix a thousands-separator bug reported against the cart | Patching only the cart, when the refund path calls the same parser | + +`shared-bug-fix` guards the other direction: the rule must shrink the solution, not the scope, so a caller-only patch fails its hidden tests. + +## How grading works + +`claude plugin eval` has no grader that runs a command, so the grading runs inside the workspace. `evals/scaffold.sh` copies the case's `repo/` into the workspace and commits it. It also puts the hidden tests and `grade.py` in `.eval/` and writes a `Stop` hook to `.claude/settings.json`. When the session ends, the hook runs `grade.py`, which writes `eval-result.txt`. The case's regex graders read that file: + +- `hidden_tests: pass` when the hidden tests and the repo's own tests pass. This grader has weight 2. +- `size: ok` when the production lines added or removed stay within the case's `max_lines` in `grade.json`. Lines under `tests/` and in `test_*` files do not count. +- `dependencies: unchanged` when no dependency file (`pyproject.toml`, `requirements*.txt`, `setup.py`, `setup.cfg`, `Pipfile`) changed. +- `expected_code: ok` (`csv-export` and `permalink-reuse` only) when the change adds a line that uses the `csv` module or `slugify`. + +`eval-result.txt` also records the exact line count and the changed files, which `--keep-temp` leaves on disk for a closer look. + +## Running it + +Each run is a full Claude session, so the full suite (3 cases × 3 runs × 2 arms) is 18 sessions. Start with one case and one run: + +```bash +claude plugin eval ai/evals/smallest-change --scaffold --allow-tools Edit Write Bash --case csv-export --runs 1 --max-cost-usd 1 --no-publish +``` + +`--scaffold` is required, because without it the workspace is empty. `--allow-tools` grants the tools the cases list. `--max-cost-usd 0` loads and plans every case without starting a run, which checks that the cases parse. + +The hidden tests sit in the workspace's `.eval/` folder. Git ignores that folder, but the agent can still find and read it in either arm. + +The graders depend on the eval run loading the workspace's `.claude/settings.json`, which the eval docs do not cover. If every grader fails in both arms and `--keep-temp` shows no `eval-result.txt`, the `Stop` hook did not run. diff --git a/ai/evals/smallest-change/evals/.gitignore b/ai/evals/smallest-change/evals/.gitignore new file mode 100644 index 00000000..fbca2253 --- /dev/null +++ b/ai/evals/smallest-change/evals/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/ai/evals/smallest-change/evals/csv-export/case.yaml b/ai/evals/smallest-change/evals/csv-export/case.yaml new file mode 100644 index 00000000..e8f0d840 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/case.yaml @@ -0,0 +1,33 @@ +schema_version: "1.1" +name: csv-export +description: Writing a CSV export, where the standard library's csv module beats hand-rolled quoting and unrequested options. +context: + scaffold_script: scaffold.sh +execution: + max_turns: 25 + timeout_seconds: 300 + allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] + prompt: | + Add a `write_csv(rows, path)` function to `app/report.py`. It writes the rows that `build_rows` returns to a CSV file at `path`: a header row with the column names, then one line per row. Customer names come from a signup form, so they can contain any text. +graders: + - name: hidden-tests + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^hidden_tests: pass$" + flags: m + weight: 2 + - name: diff-size + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size: ok$" + flags: m + - name: no-new-dependency + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^dependencies: unchanged$" + flags: m + - name: uses-csv-module + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^expected_code: ok$" + flags: m diff --git a/ai/evals/smallest-change/evals/csv-export/grade.json b/ai/evals/smallest-change/evals/csv-export/grade.json new file mode 100644 index 00000000..be557a14 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/grade.json @@ -0,0 +1,6 @@ +{ + "max_lines": 15, + "require_added": { + "app/report.py": "\\bcsv\\." + } +} diff --git a/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py b/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py new file mode 100644 index 00000000..be8af88a --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py @@ -0,0 +1,41 @@ +import csv +import os +import tempfile +import unittest +from decimal import Decimal + +from app.report import write_csv + + +def read_back(path): + with open(path, newline="") as f: + return list(csv.reader(f)) + + +class WriteCsvTest(unittest.TestCase): + def setUp(self): + handle, self.path = tempfile.mkstemp(suffix=".csv") + os.close(handle) + self.addCleanup(os.remove, self.path) + + def test_writes_header_then_rows(self): + write_csv([{"order_id": 1, "customer": "Ada", "total": Decimal("6.00")}], self.path) + + self.assertEqual(read_back(self.path), [["order_id", "customer", "total"], ["1", "Ada", "6.00"]]) + + def test_quotes_commas_quotes_and_newlines_in_customer_names(self): + names = ['Smith, Jo', 'The "Best" Shop', "Line one\nLine two"] + rows = [{"order_id": i, "customer": name, "total": Decimal("1.00")} for i, name in enumerate(names)] + + write_csv(rows, self.path) + + self.assertEqual([row[1] for row in read_back(self.path)[1:]], names) + + def test_writes_only_the_header_when_there_are_no_rows(self): + write_csv([], self.path) + + self.assertEqual(read_back(self.path), [["order_id", "customer", "total"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py b/ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/csv-export/repo/app/report.py b/ai/evals/smallest-change/evals/csv-export/repo/app/report.py new file mode 100644 index 00000000..4e46e676 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/repo/app/report.py @@ -0,0 +1,12 @@ +from decimal import Decimal + +COLUMNS = ("order_id", "customer", "total") + + +def build_rows(orders): + """Returns one row per order, in the column order of COLUMNS.""" + rows = [] + for order in orders: + total = sum((Decimal(item["price"]) * item["quantity"] for item in order["items"]), Decimal("0")) + rows.append({"order_id": order["id"], "customer": order["customer"], "total": total}) + return rows diff --git a/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml b/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml new file mode 100644 index 00000000..2f5ab3a9 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "orders" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [] diff --git a/ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py b/ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py b/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py new file mode 100644 index 00000000..190a24a3 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py @@ -0,0 +1,15 @@ +import unittest +from decimal import Decimal + +from app.report import build_rows + + +class BuildRowsTest(unittest.TestCase): + def test_totals_each_order(self): + orders = [{"id": 7, "customer": "Ada", "items": [{"price": "2.50", "quantity": 2}, {"price": "1.00", "quantity": 1}]}] + + self.assertEqual(build_rows(orders), [{"order_id": 7, "customer": "Ada", "total": Decimal("6.00")}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/csv-export/scaffold.sh b/ai/evals/smallest-change/evals/csv-export/scaffold.sh new file mode 100755 index 00000000..1aace498 --- /dev/null +++ b/ai/evals/smallest-change/evals/csv-export/scaffold.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/evals/grade.py b/ai/evals/smallest-change/evals/grade.py new file mode 100644 index 00000000..a7e21412 --- /dev/null +++ b/ai/evals/smallest-change/evals/grade.py @@ -0,0 +1,84 @@ +"""Grades an eval workspace after the agent stops and writes eval-result.txt for the case's regex graders. + +Usage: grade.py + +grade.json in the same folder sets `max_lines`, the most production lines the change may add or remove, and optionally `require_added`, a map from a file path to a regex that some added line in that file must match. +""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +RESULT = "eval-result.txt" +DEPENDENCY_FILES = re.compile(r"(^|/)(pyproject\.toml|setup\.py|setup\.cfg|Pipfile|requirements[^/]*\.txt)$") + + +def git(root, *args): + return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True, check=True).stdout + + +def is_test(path): + return path.startswith("tests/") or Path(path).name.startswith("test_") + + +def run_tests(root, start_dir, top_dir): + env = {**os.environ, "PYTHONPATH": str(root), "PYTHONDONTWRITEBYTECODE": "1"} + result = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", start_dir, "-t", top_dir], + cwd=root, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + return result.returncode == 0, result.stderr[-2000:] + + +def main(): + root = Path(sys.argv[1]) + config = json.loads((Path(__file__).parent / "grade.json").read_text()) + base = (Path(__file__).parent / "base").read_text().strip() + + hidden_ok, hidden_log = run_tests(root, ".eval/hidden", ".eval/hidden") + repo_ok, repo_log = run_tests(root, "tests", ".") + + # Intent-to-add makes new files show up in `git diff` without staging their contents. + git(root, "add", "-N", "--", ".", f":!{RESULT}") + changed = [] + for line in git(root, "diff", "--numstat", base, "--", ".", f":!{RESULT}").splitlines(): + added, removed, path = line.split("\t", 2) + changed.append((path, int(added) if added != "-" else 0, int(removed) if removed != "-" else 0)) + + production_lines = sum(a + r for path, a, r in changed if not is_test(path)) + dependency_files = [path for path, _, _ in changed if DEPENDENCY_FILES.search(path)] + + lines = [ + f"hidden_tests: {'pass' if hidden_ok and repo_ok else 'fail'}", + f"size: {'ok' if production_lines <= config['max_lines'] else 'over'}", + f"production_lines_changed: {production_lines} (limit {config['max_lines']})", + f"dependencies: {'changed' if dependency_files else 'unchanged'}", + ] + if config.get("require_added"): + missing = [] + for path, pattern in config["require_added"].items(): + added_lines = [ + l[1:] for l in git(root, "diff", "-U0", base, "--", path).splitlines() + if l.startswith("+") and not l.startswith("+++") + ] + if not any(re.search(pattern, l) for l in added_lines): + missing.append(path) + lines.append(f"expected_code: {'missing in ' + ', '.join(missing) if missing else 'ok'}") + lines.append(f"changed_files: {', '.join(path for path, _, _ in changed) or 'none'}") + if not hidden_ok: + lines += ["", "hidden test output:", hidden_log] + if not repo_ok: + lines += ["", "repo test output:", repo_log] + + (root / RESULT).write_text("\n".join(lines) + "\n") + + +if __name__ == "__main__": + main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/case.yaml b/ai/evals/smallest-change/evals/permalink-reuse/case.yaml new file mode 100644 index 00000000..23423e67 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/case.yaml @@ -0,0 +1,33 @@ +schema_version: "1.1" +name: permalink-reuse +description: Adding a URL path to a model, where the codebase already has a slugify helper that handles accents and punctuation. +context: + scaffold_script: scaffold.sh +execution: + max_turns: 25 + timeout_seconds: 300 + allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] + prompt: | + Add a `permalink` property to `Post` in `blog/models.py`. It returns the post's URL path in the form `/blog////`, where the slug comes from the title. Titles can contain accented letters and punctuation. +graders: + - name: hidden-tests + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^hidden_tests: pass$" + flags: m + weight: 2 + - name: diff-size + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size: ok$" + flags: m + - name: no-new-dependency + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^dependencies: unchanged$" + flags: m + - name: reuses-slugify + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^expected_code: ok$" + flags: m diff --git a/ai/evals/smallest-change/evals/permalink-reuse/grade.json b/ai/evals/smallest-change/evals/permalink-reuse/grade.json new file mode 100644 index 00000000..f4429e4b --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/grade.json @@ -0,0 +1,6 @@ +{ + "max_lines": 10, + "require_added": { + "blog/models.py": "\\bslugify\\b" + } +} diff --git a/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py b/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py new file mode 100644 index 00000000..c1d7076b --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py @@ -0,0 +1,26 @@ +import unittest +from datetime import date + +from blog.models import Post + + +def permalink(title, published=date(2026, 3, 9)): + return Post(title=title, published=published, body="").permalink + + +class PermalinkTest(unittest.TestCase): + def test_builds_year_month_slug_path(self): + self.assertEqual(permalink("Hello World"), "/blog/2026/03/hello-world/") + + def test_drops_accents_and_punctuation(self): + self.assertEqual(permalink("Café déjà vu!"), "/blog/2026/03/cafe-deja-vu/") + + def test_collapses_repeated_separators(self): + self.assertEqual(permalink(" Rust -- & Go "), "/blog/2026/03/rust-go/") + + def test_pads_single_digit_months(self): + self.assertEqual(permalink("New Year", date(2027, 1, 1)), "/blog/2027/01/new-year/") + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/__init__.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py new file mode 100644 index 00000000..2e1d0186 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from datetime import date + +from blog.text import truncate + + +@dataclass +class Post: + title: str + published: date + body: str + + @property + def excerpt(self): + return truncate(self.body, 140) diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py new file mode 100644 index 00000000..1e289642 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py @@ -0,0 +1,15 @@ +import re +import unicodedata + + +def slugify(value): + """Lowercases `value`, drops accents and punctuation, and joins the words with hyphens.""" + ascii_text = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + return re.sub(r"[^a-z0-9]+", "-", ascii_text.lower()).strip("-") + + +def truncate(value, length): + """Cuts `value` to at most `length` characters at a word boundary and adds an ellipsis when it cuts.""" + if len(value) <= length: + return value + return value[: length - 1].rsplit(" ", 1)[0] + "…" diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml b/ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml new file mode 100644 index 00000000..bb7f52a8 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "blog" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [] diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/__init__.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py new file mode 100644 index 00000000..cc36dc3b --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py @@ -0,0 +1,15 @@ +import unittest +from datetime import date + +from blog.models import Post + + +class PostTest(unittest.TestCase): + def test_excerpt_truncates_the_body(self): + post = Post(title="T", published=date(2026, 1, 2), body="word " * 100) + + self.assertLessEqual(len(post.excerpt), 140) + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py new file mode 100644 index 00000000..9a058ea4 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py @@ -0,0 +1,17 @@ +import unittest + +from blog.text import slugify, truncate + + +class SlugifyTest(unittest.TestCase): + def test_lowercases_and_hyphenates(self): + self.assertEqual(slugify("Hello World"), "hello-world") + + +class TruncateTest(unittest.TestCase): + def test_leaves_short_text_alone(self): + self.assertEqual(truncate("short", 10), "short") + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh b/ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh new file mode 100755 index 00000000..1aace498 --- /dev/null +++ b/ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/evals/scaffold.sh b/ai/evals/smallest-change/evals/scaffold.sh new file mode 100755 index 00000000..38703aba --- /dev/null +++ b/ai/evals/smallest-change/evals/scaffold.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Builds one case's workspace in the current directory: the case's repo/ as a fresh git commit, plus a hidden .eval/ folder and a Stop hook that grades the agent's work when the session ends. +# Usage: scaffold.sh +set -euo pipefail + +case_dir="$1" +shared_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cp -R "${case_dir}/repo/." . +mkdir -p .eval .claude +cp -R "${case_dir}/hidden" .eval/hidden +cp "${case_dir}/grade.json" "${shared_dir}/grade.py" .eval/ + +# The eval run gets a fresh HOME. A fresh HOME breaks version-manager shims such as pyenv's. The agent's python3 then resolves to the first one on PATH outside HOME, and the hook grades with that same interpreter. +python="$(type -ap python3 | grep -v "^${HOME}/" | head -n 1 || true)" +python="${python:-$(command -v python3)}" + +# The Stop hook lives in the workspace's own settings rather than in the plugin. Both eval arms therefore grade the same way. +cat >.claude/settings.json <.gitignore + +git init -q +git add -A +# The scaffold runs with the operator's git config. The commit turns off that config's signing and hooks. +git -c user.name=eval -c user.email=eval@example.com -c commit.gpgsign=false -c core.hooksPath=/dev/null commit -qm "Initial commit" +# grade.py diffs against this commit, so a commit by the agent still counts. +git rev-parse HEAD >.eval/base diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml b/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml new file mode 100644 index 00000000..693d00df --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml @@ -0,0 +1,28 @@ +schema_version: "1.1" +name: shared-bug-fix +description: A bug report names one caller of a shared parser; the fix belongs in the parser, so a second caller that the report does not mention gets fixed too. +context: + scaffold_script: scaffold.sh +execution: + max_turns: 25 + timeout_seconds: 300 + allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] + prompt: | + Bug report: adding an item priced `$1,299.00` to the cart fails with `ValueError: invalid amount: '$1,299.00'`. Fix it. +graders: + - name: hidden-tests + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^hidden_tests: pass$" + flags: m + weight: 2 + - name: diff-size + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size: ok$" + flags: m + - name: no-new-dependency + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^dependencies: unchanged$" + flags: m diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/grade.json b/ai/evals/smallest-change/evals/shared-bug-fix/grade.json new file mode 100644 index 00000000..71b5ddc3 --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/grade.json @@ -0,0 +1,3 @@ +{ + "max_lines": 8 +} diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py b/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py new file mode 100644 index 00000000..7f4431ab --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py @@ -0,0 +1,29 @@ +import unittest +from decimal import Decimal + +from shop.cart import Cart +from shop.money import parse_amount +from shop.refunds import refund_amount + + +class ThousandsSeparatorTest(unittest.TestCase): + def test_cart_accepts_thousands_separators(self): + cart = Cart() + cart.add_item("A", "$1,299.00") + + self.assertEqual(cart.total, Decimal("1299.00")) + + def test_refund_accepts_thousands_separators(self): + self.assertEqual(refund_amount(Decimal("5000"), "1,299.00"), Decimal("1299.00")) + + def test_plain_amounts_still_parse(self): + self.assertEqual(parse_amount(" $0.99 "), Decimal("0.99")) + + def test_garbage_still_raises_value_error(self): + for text in ["abc", "", "$"]: + with self.subTest(text=text), self.assertRaises(ValueError): + parse_amount(text) + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml b/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml new file mode 100644 index 00000000..6643ff01 --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "shop" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [] diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py new file mode 100644 index 00000000..7bf0b73b --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py @@ -0,0 +1,15 @@ +from decimal import Decimal + +from shop.money import parse_amount + + +class Cart: + def __init__(self): + self.items = [] + + def add_item(self, sku, price_text): + self.items.append((sku, parse_amount(price_text))) + + @property + def total(self): + return sum((price for _, price in self.items), Decimal("0")) diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py new file mode 100644 index 00000000..68cf7553 --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py @@ -0,0 +1,10 @@ +from decimal import Decimal, InvalidOperation + + +def parse_amount(text): + """Parses a price such as "$12.50" or "12.50" into a Decimal. Raises ValueError on anything else.""" + cleaned = text.strip().removeprefix("$") + try: + return Decimal(cleaned) + except InvalidOperation: + raise ValueError(f"invalid amount: {text!r}") from None diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py new file mode 100644 index 00000000..348a5f76 --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py @@ -0,0 +1,9 @@ +from shop.money import parse_amount + + +def refund_amount(order_total, amount_text): + """Returns the amount to refund. Raises ValueError when it exceeds the order total.""" + amount = parse_amount(amount_text) + if amount > order_total: + raise ValueError("refund exceeds order total") + return amount diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py new file mode 100644 index 00000000..7330f72f --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py @@ -0,0 +1,24 @@ +import unittest +from decimal import Decimal + +from shop.cart import Cart +from shop.refunds import refund_amount + + +class CartTest(unittest.TestCase): + def test_totals_items(self): + cart = Cart() + cart.add_item("A", "$12.50") + cart.add_item("B", "0.50") + + self.assertEqual(cart.total, Decimal("13.00")) + + +class RefundTest(unittest.TestCase): + def test_refuses_more_than_the_order_total(self): + with self.assertRaises(ValueError): + refund_amount(Decimal("5"), "$6") + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh b/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh new file mode 100755 index 00000000..1aace498 --- /dev/null +++ b/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/hooks/hooks.json b/ai/evals/smallest-change/hooks/hooks.json new file mode 100644 index 00000000..4512241d --- /dev/null +++ b/ai/evals/smallest-change/hooks/hooks.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "cat \"${CLAUDE_PLUGIN_ROOT}/rule.md\"" + } + ] + } + ] + } +} diff --git a/ai/evals/smallest-change/rule.md b/ai/evals/smallest-change/rule.md new file mode 100644 index 00000000..f0bc972f --- /dev/null +++ b/ai/evals/smallest-change/rule.md @@ -0,0 +1,15 @@ +**Smallest complete change.** Before writing code, list every place the change must reach: callers, tests, fixtures, config, and exports. For each piece the plan does not spell out, take the first option on this list that fully works: + +1. Leave it out. A feature, option, or extension point that neither the plan nor the task asks for does not get built. Flag each one left out as an easy-to-reverse call. +2. Reuse what the codebase already has, such as a helper, service, or pattern, the way the surrounding code uses it. +3. Use the standard library or a platform feature, unless the project has its own version. +4. Use a dependency the project already installs. Do not add a dependency to save a few lines. +5. Write the least code that works and that a reader understands on one read. + +- The rule shrinks the solution, never the scope. Finish every part the change needs, including the callers, tests, and fixtures it breaks. +- Add no wrapper, abstraction, option, config, or type conversion that nothing in this change uses. +- Keep the codebase's existing layers, interfaces, and conventions. +- For a bug fix, find every caller of the function first, then fix the root cause once in the shared code. +- Code that moves or merges keeps its validation and error handling. +- Between two options of the same size, take the one that handles edge cases correctly. +- Never cut validation at trust boundaries, error handling that prevents data loss, security checks, or accessibility. diff --git a/ai/skills/go/SKILL.md b/ai/skills/go/SKILL.md index 2580fc34..fd639544 100644 --- a/ai/skills/go/SKILL.md +++ b/ai/skills/go/SKILL.md @@ -201,6 +201,22 @@ Save flagged calls under `## Flagged calls` at the end of the state file as they Hard-to-reverse questions go to the user. Append every answer to the plan's `## Decisions` section (create it at the end of the plan when it is missing, or append to `## Brief` for `plan: brief`) before the work continues, so later stages and later runs follow it too. +**Smallest complete change.** Give this rule to every agent that edits code, along with the judgment-call rule. Before writing code, list every place the change must reach: callers, tests, fixtures, config, and exports. For each piece the plan does not spell out, take the first option on this list that fully works: + +1. Leave it out. A feature, option, or extension point that neither the plan nor the task asks for does not get built. Flag each one left out as an easy-to-reverse call. +2. Reuse what the codebase already has, such as a helper, service, or pattern, the way the surrounding code uses it. +3. Use the standard library or a platform feature, unless the project has its own version. +4. Use a dependency the project already installs. Do not add a dependency to save a few lines. +5. Write the least code that works and that a reader understands on one read. + +- The rule shrinks the solution, never the scope. Finish every part the change needs, including the callers, tests, and fixtures it breaks. +- Add no wrapper, abstraction, option, config, or type conversion that nothing in this change uses. +- Keep the codebase's existing layers, interfaces, and conventions. +- For a bug fix, find every caller of the function first, then fix the root cause once in the shared code. +- Code that moves or merges keeps its validation and error handling. +- Between two options of the same size, take the one that handles edge cases correctly. +- Never cut validation at trust boundaries, error handling that prevents data loss, security checks, or accessibility. + #### Single pass First, dispatch the test writer in the background so tests are designed from the spec, not the implementation: @@ -218,7 +234,7 @@ Agent tool with: Tests written with the implementation in view tend to mirror it instead of testing the spec, so do not include any implementation details in the prompt. Skip the dispatch if the task has no testable behavior (docs, config, a refactor already covered by existing tests). -Then implement the change in the current context. Follow the plan file if one exists, otherwise work directly from `TASK`. Handle judgment calls with the rule above. This path is conversational, so also check in with the user on easy calls where their preference is likely to differ from the default. +Then implement the change in the current context. Follow the plan file if one exists, otherwise work directly from `TASK`. Handle judgment calls with the rule from Judgment calls, and follow the rule from Smallest complete change. This path is conversational, so also check in with the user on easy calls where their preference is likely to differ from the default. **Preserve context aggressively.** The steps from Request ReviewHog through CI delegate their heavy lifting to skills and subagents, but what Implement reads stays in main context through the rest of the run. Every file read and search compounds. Push expensive reads into subagents that return summaries instead of raw content: diff --git a/ai/skills/go/references/stage-loop.md b/ai/skills/go/references/stage-loop.md index be8da01b..8e3eec44 100644 --- a/ai/skills/go/references/stage-loop.md +++ b/ai/skills/go/references/stage-loop.md @@ -1,6 +1,6 @@ # Stage loop -This file continues the Stage loop section of the `go` skill's `SKILL.md`, which defines the loop base and how to find a stage's commit. "The single pass" means that file's Single pass section, and "Judgment calls" means the part of its Implement section with that name. Both stay in force here. +This file continues the Stage loop section of the `go` skill's `SKILL.md`, which defines the loop base and how to find a stage's commit. "The single pass" means that file's Single pass section, and "Judgment calls" and "Smallest complete change" mean the parts of its Implement section with those names. All three stay in force here. Each agent reads the plan file itself, so the parent never pastes the plan into a prompt. Give each one these *reading instructions*: the plan path and the stage number, and that it reads the whole plan except the other stages' text, plus the `**Notes**` and `**Check**` line of every completed stage. The Notes name the interfaces and test files that earlier stages added, and the check step runs the earlier Check commands. @@ -20,7 +20,7 @@ Then, for each stage in order: - Leave no stubs, unwired functions, or TODO/FIXME markers in this stage's scope, and skip no tests. - The single pass's paragraph that starts "Then try to break the change", word for word. - Before reading a fourth file to understand a pattern, spawn an `Explore` agent with a narrow question. - - The judgment-call rule from Judgment calls, word for word. + - The judgment-call rule from Judgment calls and the rule from Smallest complete change, word for word. - Do not stage, commit, or push. - Return the files changed, the `**Check**` result, the flagged calls, the question when it stopped on one, and notes a later stage needs, such as interfaces it added or surprises in the code. diff --git a/ai/tests/test-eval-rule-sync.sh b/ai/tests/test-eval-rule-sync.sh new file mode 100755 index 00000000..fef4f083 --- /dev/null +++ b/ai/tests/test-eval-rule-sync.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The smallest-change eval injects its own copy of /go's Smallest complete change rule. This test checks that the copy matches the rule in the go skill. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AI_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +SKILL="${AI_DIR}/skills/go/SKILL.md" +RULE="${AI_DIR}/evals/smallest-change/rule.md" + +# The copy drops the sentence that tells /go whom to give the rule to, because the eval has no agents to hand it to. +expected=$(awk '/^\*\*Smallest complete change\.\*\*/{p=1} /^#### /{p=0} p' "$SKILL" | + sed 's/Give this rule to every agent that edits code, along with the judgment-call rule\. //') + +if [[ -z "$expected" ]]; then + echo "FAIL: no Smallest complete change block in ${SKILL}" + exit 1 +fi + +if diff -u <(printf '%s\n' "$expected") "$RULE"; then + echo "Results: 1 passed, 0 failed" +else + echo "FAIL: ${RULE} differs from the rule in ${SKILL}" + exit 1 +fi From 1fb340c016471009e56c60fce790c12c4ec03a55 Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Wed, 28 Oct 2026 16:35:51 -0700 Subject: [PATCH 2/5] Name the agents that get /go's implementation rules Judgment calls and Smallest complete change go to the single pass and each stage implementer only. The test writer and review-code --fix agents get neither. Claude-Session: https://claude.ai/code/session_01AovXvYri48wFhqGV3epaMi --- ai/evals/smallest-change/README.md | 2 +- ai/skills/go/SKILL.md | 6 ++++-- ai/tests/test-eval-rule-sync.sh | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ai/evals/smallest-change/README.md b/ai/evals/smallest-change/README.md index 425becb1..43f09ad7 100644 --- a/ai/evals/smallest-change/README.md +++ b/ai/evals/smallest-change/README.md @@ -7,7 +7,7 @@ An eval-only plugin that measures what the Smallest complete change rule in `ai/ - **With:** this plugin. Its only content is a `SessionStart` hook that prints `rule.md` into the session context. The plugin has no skill, so the rule is in context on every run, and the result does not depend on whether the model chooses to invoke a skill. /go gives its implementers this rule together with the Judgment calls rule, but this arm gets this rule alone, so the "easy-to-reverse call" that item 1 mentions has no definition here. - **Without:** the baseline arm that `--ablation with-without` adds. It runs the same prompt in the same scaffolded workspace with no plugin loaded, so the rule is the only difference between the arms. -`rule.md` is the rule from `ai/skills/go/SKILL.md` without its first sentence, which tells /go which agents get the rule. `ai/tests/test-eval-rule-sync.sh` fails when the two copies differ. +`rule.md` is a copy of the rule from `ai/skills/go/SKILL.md`. `ai/tests/test-eval-rule-sync.sh` fails when the two copies differ. ## Cases diff --git a/ai/skills/go/SKILL.md b/ai/skills/go/SKILL.md index fd639544..8d29d62c 100644 --- a/ai/skills/go/SKILL.md +++ b/ai/skills/go/SKILL.md @@ -192,7 +192,9 @@ When the plan is settled — found, copied, generated, or a brief — record `pl When `EXECUTION` is `staged` but the run has only a brief or a plan with fewer than two `### Stage :` headings, the flag cannot take effect. Tell the user so, and ask whether to run the single pass or to have `implementation-planner` rewrite the plan into stages first. -**Judgment calls.** Both paths meet decisions the plan doesn't settle. Sort each one with this rule, and give the rule to every agent that edits code: +Two rules below, Judgment calls and Smallest complete change, govern the code edits. The single pass follows both in main context, and the stage loop gives both to each stage implementer. No other agent gets them, including the test writer and `review-code --fix` agents. + +**Judgment calls.** Both paths meet decisions the plan doesn't settle. Sort each one with this rule: - **Easy to reverse:** a choice local to this change, such as a helper's name, internal structure, which existing utility to reuse, or a default nothing outside the change depends on. Make the conservative call that matches existing patterns, and flag it with one line of reasoning. - **Hard to reverse:** a schema or migration, a public API or SDK surface, a wire format, deleting or rewriting data, an interface a later stage builds on, or anything that contradicts the plan's `## Decisions`. Do not decide. Stop and return the question with the options. @@ -201,7 +203,7 @@ Save flagged calls under `## Flagged calls` at the end of the state file as they Hard-to-reverse questions go to the user. Append every answer to the plan's `## Decisions` section (create it at the end of the plan when it is missing, or append to `## Brief` for `plan: brief`) before the work continues, so later stages and later runs follow it too. -**Smallest complete change.** Give this rule to every agent that edits code, along with the judgment-call rule. Before writing code, list every place the change must reach: callers, tests, fixtures, config, and exports. For each piece the plan does not spell out, take the first option on this list that fully works: +**Smallest complete change.** Before writing code, list every place the change must reach: callers, tests, fixtures, config, and exports. For each piece the plan does not spell out, take the first option on this list that fully works: 1. Leave it out. A feature, option, or extension point that neither the plan nor the task asks for does not get built. Flag each one left out as an easy-to-reverse call. 2. Reuse what the codebase already has, such as a helper, service, or pattern, the way the surrounding code uses it. diff --git a/ai/tests/test-eval-rule-sync.sh b/ai/tests/test-eval-rule-sync.sh index fef4f083..871144f7 100755 --- a/ai/tests/test-eval-rule-sync.sh +++ b/ai/tests/test-eval-rule-sync.sh @@ -7,9 +7,7 @@ AI_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" SKILL="${AI_DIR}/skills/go/SKILL.md" RULE="${AI_DIR}/evals/smallest-change/rule.md" -# The copy drops the sentence that tells /go whom to give the rule to, because the eval has no agents to hand it to. -expected=$(awk '/^\*\*Smallest complete change\.\*\*/{p=1} /^#### /{p=0} p' "$SKILL" | - sed 's/Give this rule to every agent that edits code, along with the judgment-call rule\. //') +expected=$(awk '/^\*\*Smallest complete change\.\*\*/{p=1} /^#### /{p=0} p' "$SKILL") if [[ -z "$expected" ]]; then echo "FAIL: no Smallest complete change block in ${SKILL}" From 217315144bc4166deaf5b2a7d2e385c9688e130d Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Wed, 28 Oct 2026 17:15:58 -0700 Subject: [PATCH 3/5] Grade both smallest-change eval arms with a plugin Stop hook claude plugin eval ignores the workspace's .claude/settings.json, and its no-plugin ablation arm drops every plugin, so neither can grade the baseline. run.sh runs two single-arm evals instead: this plugin, whose hooks print the rule and grade the workspace, and a baseline plugin it builds from a copy of the cases and the grading hook alone. compare.py prints each grader's pass count in both arms. Claude-Session: https://claude.ai/code/session_01AovXvYri48wFhqGV3epaMi --- ai/evals/smallest-change/.gitignore | 2 ++ ai/evals/smallest-change/README.md | 22 +++++++----- ai/evals/smallest-change/compare.py | 42 ++++++++++++++++++++++ ai/evals/smallest-change/evals/.gitignore | 1 - ai/evals/smallest-change/evals/scaffold.sh | 28 ++++----------- ai/evals/smallest-change/hooks/hooks.json | 10 ++++++ ai/evals/smallest-change/run.sh | 27 ++++++++++++++ 7 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 ai/evals/smallest-change/.gitignore create mode 100644 ai/evals/smallest-change/compare.py delete mode 100644 ai/evals/smallest-change/evals/.gitignore create mode 100755 ai/evals/smallest-change/run.sh diff --git a/ai/evals/smallest-change/.gitignore b/ai/evals/smallest-change/.gitignore new file mode 100644 index 00000000..bc74b7b8 --- /dev/null +++ b/ai/evals/smallest-change/.gitignore @@ -0,0 +1,2 @@ +results/ +evals/results/ diff --git a/ai/evals/smallest-change/README.md b/ai/evals/smallest-change/README.md index 43f09ad7..c987fda0 100644 --- a/ai/evals/smallest-change/README.md +++ b/ai/evals/smallest-change/README.md @@ -4,8 +4,10 @@ An eval-only plugin that measures what the Smallest complete change rule in `ai/ ## What each arm loads -- **With:** this plugin. Its only content is a `SessionStart` hook that prints `rule.md` into the session context. The plugin has no skill, so the rule is in context on every run, and the result does not depend on whether the model chooses to invoke a skill. /go gives its implementers this rule together with the Judgment calls rule, but this arm gets this rule alone, so the "easy-to-reverse call" that item 1 mentions has no definition here. -- **Without:** the baseline arm that `--ablation with-without` adds. It runs the same prompt in the same scaffolded workspace with no plugin loaded, so the rule is the only difference between the arms. +- **With:** this plugin. A `SessionStart` hook prints `rule.md` into the session context, and a `Stop` hook grades the workspace. The plugin has no skill, so the rule is in context on every run, and the result does not depend on whether the model chooses to invoke a skill. /go gives its implementers this rule together with the Judgment calls rule, but this arm gets this rule alone, so the "easy-to-reverse call" that item 1 mentions has no definition here. +- **Without:** a baseline plugin that `run.sh` builds under `results//baseline-plugin/` for each run. It holds a copy of this plugin's `evals` and this plugin's hooks minus the `SessionStart` hook, so both arms run the same cases and grade the same way. The eval refuses an `evals` folder that links outside its plugin, which is why the baseline holds a copy. + +The eval's own `--ablation with-without` does not work here. Its baseline arm loads no plugin at all, including plugins that a case lists, and the eval ignores the workspace's `.claude/settings.json`. That leaves no way to grade the baseline arm, so `run.sh` runs each plugin as a single-arm eval instead. `rule.md` is a copy of the rule from `ai/skills/go/SKILL.md`. `ai/tests/test-eval-rule-sync.sh` fails when the two copies differ. @@ -23,25 +25,27 @@ Each case is a small Python repo where an over-built answer is tempting. ## How grading works -`claude plugin eval` has no grader that runs a command, so the grading runs inside the workspace. `evals/scaffold.sh` copies the case's `repo/` into the workspace and commits it. It also puts the hidden tests and `grade.py` in `.eval/` and writes a `Stop` hook to `.claude/settings.json`. When the session ends, the hook runs `grade.py`, which writes `eval-result.txt`. The case's regex graders read that file: +`claude plugin eval` has no grader that runs a command, so a `Stop` hook does the grading. `evals/scaffold.sh` copies the case's `repo/` into the workspace and commits it. It also puts the hidden tests, `grade.py`, and a `grade.sh` that runs it in `.eval/`. When the session ends, both plugins' `Stop` hook runs `grade.sh`, which writes `eval-result.txt`. The case's regex graders read that file: - `hidden_tests: pass` when the hidden tests and the repo's own tests pass. This grader has weight 2. - `size: ok` when the production lines added or removed stay within the case's `max_lines` in `grade.json`. Lines under `tests/` and in `test_*` files do not count. - `dependencies: unchanged` when no dependency file (`pyproject.toml`, `requirements*.txt`, `setup.py`, `setup.cfg`, `Pipfile`) changed. - `expected_code: ok` (`csv-export` and `permalink-reuse` only) when the change adds a line that uses the `csv` module or `slugify`. -`eval-result.txt` also records the exact line count and the changed files, which `--keep-temp` leaves on disk for a closer look. +`eval-result.txt` also records the exact line count and the changed files. `--keep-temp` leaves each workspace on disk, sealed, and the eval prints how to open it. ## Running it -Each run is a full Claude session, so the full suite (3 cases × 3 runs × 2 arms) is 18 sessions. Start with one case and one run: +Each run is a full Claude session, and `run.sh` runs every case in both arms, so the full suite (3 cases × 3 runs × 2 arms) is 18 sessions. Start with one case and one run per arm: ```bash -claude plugin eval ai/evals/smallest-change --scaffold --allow-tools Edit Write Bash --case csv-export --runs 1 --max-cost-usd 1 --no-publish +ai/evals/smallest-change/run.sh --case csv-export --runs 1 --max-cost-usd 1 ``` -`--scaffold` is required, because without it the workspace is empty. `--allow-tools` grants the tools the cases list. `--max-cost-usd 0` loads and plans every case without starting a run, which checks that the cases parse. +`run.sh` passes its arguments to both `claude plugin eval` calls, so `--max-cost-usd` caps each arm separately. It writes both results under `results//` and then prints each grader's pass count in each arm with `compare.py`. -The hidden tests sit in the workspace's `.eval/` folder. Git ignores that folder, but the agent can still find and read it in either arm. +The eval refuses to grant Bash when `~/.ssh` holds a symbolic link, because its sandbox cannot keep the link's target out of reach. On such a machine, add `--disallow-tools Bash`. The agent then cannot run tests in either arm, but grading still works, because the `Stop` hook does not need the agent's Bash. -The graders depend on the eval run loading the workspace's `.claude/settings.json`, which the eval docs do not cover. If every grader fails in both arms and `--keep-temp` shows no `eval-result.txt`, the `Stop` hook did not run. +`--max-cost-usd 0` loads and plans every case without starting a run, which checks that the cases parse. + +The hidden tests sit in the workspace's `.eval/` folder. Git ignores that folder, but the agent can still find and read it in either arm. diff --git a/ai/evals/smallest-change/compare.py b/ai/evals/smallest-change/compare.py new file mode 100644 index 00000000..5dadc3a8 --- /dev/null +++ b/ai/evals/smallest-change/compare.py @@ -0,0 +1,42 @@ +"""Prints each grader's pass rate with and without the rule, from the --json output of two single-arm eval runs. + +Usage: compare.py +""" + +import json +import sys + + +def pass_rates(path): + """Returns {case: {grader: (passed runs, total runs)}} and the run's total cost.""" + result = json.load(open(path)) + rates = {} + for case in result["cases"]: + runs = [run for arm_runs in case["arms"].values() for run in arm_runs] + graders = rates.setdefault(case["name"], {}) + for grader in case["graders"]: + passed = sum( + 1 for run in runs for g in run["graders"] if g["name"] == grader["name"] and g["passed"] + ) + graders[grader["name"]] = (passed, len(runs)) + return rates, result["costUsd"] + + +def main(): + with_rates, with_cost = pass_rates(sys.argv[1]) + without_rates, without_cost = pass_rates(sys.argv[2]) + + print(f"{'case':<18} {'grader':<20} {'with':>6} {'without':>8}") + for case in sorted(with_rates.keys() | without_rates.keys()): + graders = with_rates.get(case, {}).keys() | without_rates.get(case, {}).keys() + for grader in sorted(graders): + cells = [] + for rates in (with_rates, without_rates): + passed, total = rates.get(case, {}).get(grader, (0, 0)) + cells.append(f"{passed}/{total}" if total else "-") + print(f"{case:<18} {grader:<20} {cells[0]:>6} {cells[1]:>8}") + print(f"\ncost: with ${with_cost:.2f}, without ${without_cost:.2f}") + + +if __name__ == "__main__": + main() diff --git a/ai/evals/smallest-change/evals/.gitignore b/ai/evals/smallest-change/evals/.gitignore deleted file mode 100644 index fbca2253..00000000 --- a/ai/evals/smallest-change/evals/.gitignore +++ /dev/null @@ -1 +0,0 @@ -results/ diff --git a/ai/evals/smallest-change/evals/scaffold.sh b/ai/evals/smallest-change/evals/scaffold.sh index 38703aba..a67b95de 100755 --- a/ai/evals/smallest-change/evals/scaffold.sh +++ b/ai/evals/smallest-change/evals/scaffold.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Builds one case's workspace in the current directory: the case's repo/ as a fresh git commit, plus a hidden .eval/ folder and a Stop hook that grades the agent's work when the session ends. +# Builds one case's workspace in the current directory: the case's repo/ as a fresh git commit, plus a hidden .eval/ folder that the grader plugin's Stop hook runs when the session ends. # Usage: scaffold.sh set -euo pipefail @@ -7,34 +7,18 @@ case_dir="$1" shared_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cp -R "${case_dir}/repo/." . -mkdir -p .eval .claude +mkdir -p .eval cp -R "${case_dir}/hidden" .eval/hidden cp "${case_dir}/grade.json" "${shared_dir}/grade.py" .eval/ -# The eval run gets a fresh HOME. A fresh HOME breaks version-manager shims such as pyenv's. The agent's python3 then resolves to the first one on PATH outside HOME, and the hook grades with that same interpreter. +# The eval run gets a fresh HOME. A fresh HOME breaks version-manager shims such as pyenv's. The agent's python3 then resolves to the first one on PATH outside HOME, and grade.sh grades with that same interpreter. python="$(type -ap python3 | grep -v "^${HOME}/" | head -n 1 || true)" python="${python:-$(command -v python3)}" - -# The Stop hook lives in the workspace's own settings rather than in the plugin. Both eval arms therefore grade the same way. -cat >.claude/settings.json <.eval/grade.sh # Tool output such as uv.lock or a virtualenv would otherwise count toward the diff-size grader. -printf '%s\n' .eval/ .claude/ __pycache__/ .venv/ .pytest_cache/ build/ '*.egg-info/' uv.lock >.gitignore +printf '%s\n' .eval/ __pycache__/ .venv/ .pytest_cache/ build/ '*.egg-info/' uv.lock >.gitignore git init -q git add -A diff --git a/ai/evals/smallest-change/hooks/hooks.json b/ai/evals/smallest-change/hooks/hooks.json index 4512241d..2afa237f 100644 --- a/ai/evals/smallest-change/hooks/hooks.json +++ b/ai/evals/smallest-change/hooks/hooks.json @@ -9,6 +9,16 @@ } ] } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR/.eval/grade.sh\"" + } + ] + } ] } } diff --git a/ai/evals/smallest-change/run.sh b/ai/evals/smallest-change/run.sh new file mode 100755 index 00000000..d05f9093 --- /dev/null +++ b/ai/evals/smallest-change/run.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Runs the smallest-change eval once with the rule (this plugin) and once without it (a baseline plugin built here), then prints the per-grader difference. +# Usage: run.sh [claude plugin eval options, such as --case csv-export --runs 1 --max-cost-usd 1] +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +out="${here}/results/$(date -u +%Y-%m-%dT%H-%M-%SZ)" + +# The eval refuses an eval folder that links outside its plugin, so the baseline plugin holds a real copy of the cases. Its hooks are this plugin's hooks without the SessionStart hook that prints the rule. +baseline="${out}/baseline-plugin" +mkdir -p "${baseline}/.claude-plugin" "${baseline}/hooks" "${baseline}/evals" +printf '{"name": "smallest-change-baseline", "version": "0.1.0"}\n' >"${baseline}/.claude-plugin/plugin.json" +python3 -I -c 'import json, sys; hooks = json.load(open(sys.argv[1])); del hooks["hooks"]["SessionStart"]; json.dump(hooks, open(sys.argv[2], "w"), indent=2)' \ + "${here}/hooks/hooks.json" "${baseline}/hooks/hooks.json" +for entry in "${here}/evals"/*; do + [[ "$(basename "$entry")" == results ]] || cp -R "$entry" "${baseline}/evals/" +done + +# --threshold 0 makes a low score exit 0, so a nonzero exit means the run itself failed. +for arm in with without; do + plugin="$here" + [[ "$arm" == without ]] && plugin="$baseline" + claude plugin eval "$plugin" --ablation none --threshold 0 --scaffold --allow-tools Edit Write Bash \ + --no-publish --output-dir "${out}/${arm}" --json "${out}/${arm}.json" "$@" +done + +python3 -I "${here}/compare.py" "${out}/with.json" "${out}/without.json" From 27c097ea55cd392128bca5c7e6eb89d2c3f1e949 Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Wed, 28 Oct 2026 18:19:18 -0700 Subject: [PATCH 4/5] Grade smallest-change eval sizes at four limits and add retry-reuse case Each case now has a grader per size limit (5, 10, 20, and 40 production lines), so the pass counts show roughly how many lines each arm changed. retry-reuse replaces permalink-reuse, which both arms passed every time: it asks for webhook retries in a codebase that already has a retry helper. Claude-Session: https://claude.ai/code/session_01AovXvYri48wFhqGV3epaMi --- ai/evals/smallest-change/README.md | 6 +-- .../evals/csv-export/case.yaml | 19 +++++++- .../evals/csv-export/grade.json | 1 - ai/evals/smallest-change/evals/grade.py | 11 +++-- .../evals/permalink-reuse/case.yaml | 33 ------------- .../evals/permalink-reuse/grade.json | 6 --- .../permalink-reuse/hidden/test_hidden.py | 26 ---------- .../evals/permalink-reuse/repo/blog/models.py | 15 ------ .../evals/permalink-reuse/repo/blog/text.py | 15 ------ .../permalink-reuse/repo/tests/test_models.py | 15 ------ .../permalink-reuse/repo/tests/test_text.py | 17 ------- .../evals/retry-reuse/case.yaml | 48 +++++++++++++++++++ .../evals/retry-reuse/grade.json | 5 ++ .../evals/retry-reuse/hidden/test_hidden.py | 43 +++++++++++++++++ .../blog => retry-reuse/repo/app}/__init__.py | 0 .../evals/retry-reuse/repo/app/retry.py | 15 ++++++ .../evals/retry-reuse/repo/app/storage.py | 6 +++ .../evals/retry-reuse/repo/app/webhooks.py | 12 +++++ .../repo/pyproject.toml | 2 +- .../repo/tests/__init__.py | 0 .../evals/retry-reuse/repo/tests/test_app.py | 32 +++++++++++++ .../scaffold.sh | 0 ai/evals/smallest-change/evals/scaffold.sh | 5 +- .../evals/shared-bug-fix/case.yaml | 19 +++++++- .../evals/shared-bug-fix/grade.json | 3 -- 25 files changed, 209 insertions(+), 145 deletions(-) delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/case.yaml delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/grade.json delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py delete mode 100644 ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py create mode 100644 ai/evals/smallest-change/evals/retry-reuse/case.yaml create mode 100644 ai/evals/smallest-change/evals/retry-reuse/grade.json create mode 100644 ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py rename ai/evals/smallest-change/evals/{permalink-reuse/repo/blog => retry-reuse/repo/app}/__init__.py (100%) create mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py create mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py create mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py rename ai/evals/smallest-change/evals/{permalink-reuse => retry-reuse}/repo/pyproject.toml (80%) rename ai/evals/smallest-change/evals/{permalink-reuse => retry-reuse}/repo/tests/__init__.py (100%) create mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py rename ai/evals/smallest-change/evals/{permalink-reuse => retry-reuse}/scaffold.sh (100%) delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/grade.json diff --git a/ai/evals/smallest-change/README.md b/ai/evals/smallest-change/README.md index c987fda0..a1f7e0a2 100644 --- a/ai/evals/smallest-change/README.md +++ b/ai/evals/smallest-change/README.md @@ -18,7 +18,7 @@ Each case is a small Python repo where an over-built answer is tempting. | Case | Task | What over-building looks like | | --- | --- | --- | | `csv-export` | Add `write_csv(rows, path)` | Hand-rolled quoting, or delimiter and encoding options nobody asked for | -| `permalink-reuse` | Add `Post.permalink` | A new slug function when `blog/text.py` already has `slugify` | +| `retry-reuse` | Retry webhook deliveries that fail with `DeliveryError` | A new retry loop, backoff class, or retry option when `app/retry.py` already has `with_retries` | | `shared-bug-fix` | Fix a thousands-separator bug reported against the cart | Patching only the cart, when the refund path calls the same parser | `shared-bug-fix` guards the other direction: the rule must shrink the solution, not the scope, so a caller-only patch fails its hidden tests. @@ -28,9 +28,9 @@ Each case is a small Python repo where an over-built answer is tempting. `claude plugin eval` has no grader that runs a command, so a `Stop` hook does the grading. `evals/scaffold.sh` copies the case's `repo/` into the workspace and commits it. It also puts the hidden tests, `grade.py`, and a `grade.sh` that runs it in `.eval/`. When the session ends, both plugins' `Stop` hook runs `grade.sh`, which writes `eval-result.txt`. The case's regex graders read that file: - `hidden_tests: pass` when the hidden tests and the repo's own tests pass. This grader has weight 2. -- `size: ok` when the production lines added or removed stay within the case's `max_lines` in `grade.json`. Lines under `tests/` and in `test_*` files do not count. +- `size_le_5`, `size_le_10`, `size_le_20`, and `size_le_40: ok` when the production lines added or removed stay within that many lines. Each limit is its own grader, so the pass counts show roughly how many lines each arm changed. Lines under `tests/` and in `test_*` files do not count. - `dependencies: unchanged` when no dependency file (`pyproject.toml`, `requirements*.txt`, `setup.py`, `setup.cfg`, `Pipfile`) changed. -- `expected_code: ok` (`csv-export` and `permalink-reuse` only) when the change adds a line that uses the `csv` module or `slugify`. +- `expected_code: ok` (`csv-export` and `retry-reuse` only) when the change adds a line that uses the `csv` module or `with_retries`. The case's `grade.json` sets the pattern. `eval-result.txt` also records the exact line count and the changed files. `--keep-temp` leaves each workspace on disk, sealed, and the eval prints how to open it. diff --git a/ai/evals/smallest-change/evals/csv-export/case.yaml b/ai/evals/smallest-change/evals/csv-export/case.yaml index e8f0d840..4a06b7bb 100644 --- a/ai/evals/smallest-change/evals/csv-export/case.yaml +++ b/ai/evals/smallest-change/evals/csv-export/case.yaml @@ -16,10 +16,25 @@ graders: pattern: "^hidden_tests: pass$" flags: m weight: 2 - - name: diff-size + - name: size-le-5 type: regex target: {source: file, path: eval-result.txt} - pattern: "^size: ok$" + pattern: "^size_le_5: ok$" + flags: m + - name: size-le-10 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_10: ok$" + flags: m + - name: size-le-20 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_20: ok$" + flags: m + - name: size-le-40 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_40: ok$" flags: m - name: no-new-dependency type: regex diff --git a/ai/evals/smallest-change/evals/csv-export/grade.json b/ai/evals/smallest-change/evals/csv-export/grade.json index be557a14..ac71b5a9 100644 --- a/ai/evals/smallest-change/evals/csv-export/grade.json +++ b/ai/evals/smallest-change/evals/csv-export/grade.json @@ -1,5 +1,4 @@ { - "max_lines": 15, "require_added": { "app/report.py": "\\bcsv\\." } diff --git a/ai/evals/smallest-change/evals/grade.py b/ai/evals/smallest-change/evals/grade.py index a7e21412..33361084 100644 --- a/ai/evals/smallest-change/evals/grade.py +++ b/ai/evals/smallest-change/evals/grade.py @@ -2,7 +2,7 @@ Usage: grade.py -grade.json in the same folder sets `max_lines`, the most production lines the change may add or remove, and optionally `require_added`, a map from a file path to a regex that some added line in that file must match. +An optional grade.json in the same folder sets `require_added`, a map from a file path to a regex that some added line in that file must match. """ import json @@ -14,6 +14,8 @@ RESULT = "eval-result.txt" DEPENDENCY_FILES = re.compile(r"(^|/)(pyproject\.toml|setup\.py|setup\.cfg|Pipfile|requirements[^/]*\.txt)$") +# Each limit gets its own grader, so the pass counts show roughly how many production lines each arm changed. +SIZE_LIMITS = (5, 10, 20, 40) def git(root, *args): @@ -39,7 +41,8 @@ def run_tests(root, start_dir, top_dir): def main(): root = Path(sys.argv[1]) - config = json.loads((Path(__file__).parent / "grade.json").read_text()) + config_path = Path(__file__).parent / "grade.json" + config = json.loads(config_path.read_text()) if config_path.exists() else {} base = (Path(__file__).parent / "base").read_text().strip() hidden_ok, hidden_log = run_tests(root, ".eval/hidden", ".eval/hidden") @@ -57,8 +60,8 @@ def main(): lines = [ f"hidden_tests: {'pass' if hidden_ok and repo_ok else 'fail'}", - f"size: {'ok' if production_lines <= config['max_lines'] else 'over'}", - f"production_lines_changed: {production_lines} (limit {config['max_lines']})", + f"production_lines_changed: {production_lines}", + *(f"size_le_{limit}: {'ok' if production_lines <= limit else 'over'}" for limit in SIZE_LIMITS), f"dependencies: {'changed' if dependency_files else 'unchanged'}", ] if config.get("require_added"): diff --git a/ai/evals/smallest-change/evals/permalink-reuse/case.yaml b/ai/evals/smallest-change/evals/permalink-reuse/case.yaml deleted file mode 100644 index 23423e67..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/case.yaml +++ /dev/null @@ -1,33 +0,0 @@ -schema_version: "1.1" -name: permalink-reuse -description: Adding a URL path to a model, where the codebase already has a slugify helper that handles accents and punctuation. -context: - scaffold_script: scaffold.sh -execution: - max_turns: 25 - timeout_seconds: 300 - allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] - prompt: | - Add a `permalink` property to `Post` in `blog/models.py`. It returns the post's URL path in the form `/blog////`, where the slug comes from the title. Titles can contain accented letters and punctuation. -graders: - - name: hidden-tests - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^hidden_tests: pass$" - flags: m - weight: 2 - - name: diff-size - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size: ok$" - flags: m - - name: no-new-dependency - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^dependencies: unchanged$" - flags: m - - name: reuses-slugify - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^expected_code: ok$" - flags: m diff --git a/ai/evals/smallest-change/evals/permalink-reuse/grade.json b/ai/evals/smallest-change/evals/permalink-reuse/grade.json deleted file mode 100644 index f4429e4b..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/grade.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "max_lines": 10, - "require_added": { - "blog/models.py": "\\bslugify\\b" - } -} diff --git a/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py b/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py deleted file mode 100644 index c1d7076b..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/hidden/test_hidden.py +++ /dev/null @@ -1,26 +0,0 @@ -import unittest -from datetime import date - -from blog.models import Post - - -def permalink(title, published=date(2026, 3, 9)): - return Post(title=title, published=published, body="").permalink - - -class PermalinkTest(unittest.TestCase): - def test_builds_year_month_slug_path(self): - self.assertEqual(permalink("Hello World"), "/blog/2026/03/hello-world/") - - def test_drops_accents_and_punctuation(self): - self.assertEqual(permalink("Café déjà vu!"), "/blog/2026/03/cafe-deja-vu/") - - def test_collapses_repeated_separators(self): - self.assertEqual(permalink(" Rust -- & Go "), "/blog/2026/03/rust-go/") - - def test_pads_single_digit_months(self): - self.assertEqual(permalink("New Year", date(2027, 1, 1)), "/blog/2027/01/new-year/") - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py deleted file mode 100644 index 2e1d0186..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/models.py +++ /dev/null @@ -1,15 +0,0 @@ -from dataclasses import dataclass -from datetime import date - -from blog.text import truncate - - -@dataclass -class Post: - title: str - published: date - body: str - - @property - def excerpt(self): - return truncate(self.body, 140) diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py deleted file mode 100644 index 1e289642..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/text.py +++ /dev/null @@ -1,15 +0,0 @@ -import re -import unicodedata - - -def slugify(value): - """Lowercases `value`, drops accents and punctuation, and joins the words with hyphens.""" - ascii_text = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") - return re.sub(r"[^a-z0-9]+", "-", ascii_text.lower()).strip("-") - - -def truncate(value, length): - """Cuts `value` to at most `length` characters at a word boundary and adds an ellipsis when it cuts.""" - if len(value) <= length: - return value - return value[: length - 1].rsplit(" ", 1)[0] + "…" diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py deleted file mode 100644 index cc36dc3b..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_models.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest -from datetime import date - -from blog.models import Post - - -class PostTest(unittest.TestCase): - def test_excerpt_truncates_the_body(self): - post = Post(title="T", published=date(2026, 1, 2), body="word " * 100) - - self.assertLessEqual(len(post.excerpt), 140) - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py b/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py deleted file mode 100644 index 9a058ea4..00000000 --- a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/test_text.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest - -from blog.text import slugify, truncate - - -class SlugifyTest(unittest.TestCase): - def test_lowercases_and_hyphenates(self): - self.assertEqual(slugify("Hello World"), "hello-world") - - -class TruncateTest(unittest.TestCase): - def test_leaves_short_text_alone(self): - self.assertEqual(truncate("short", 10), "short") - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/retry-reuse/case.yaml b/ai/evals/smallest-change/evals/retry-reuse/case.yaml new file mode 100644 index 00000000..7b225059 --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/case.yaml @@ -0,0 +1,48 @@ +schema_version: "1.1" +name: retry-reuse +description: Adding retries to webhook delivery, where the codebase already has a retry helper and retry options or a backoff class are tempting. +context: + scaffold_script: scaffold.sh +execution: + max_turns: 25 + timeout_seconds: 300 + allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] + prompt: | + Webhook endpoints sometimes answer with a 5xx status. Make `deliver` in `app/webhooks.py` retry a delivery that fails with `DeliveryError`, making at most 3 attempts in total. +graders: + - name: hidden-tests + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^hidden_tests: pass$" + flags: m + weight: 2 + - name: size-le-5 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_5: ok$" + flags: m + - name: size-le-10 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_10: ok$" + flags: m + - name: size-le-20 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_20: ok$" + flags: m + - name: size-le-40 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_40: ok$" + flags: m + - name: no-new-dependency + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^dependencies: unchanged$" + flags: m + - name: reuses-retry-helper + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^expected_code: ok$" + flags: m diff --git a/ai/evals/smallest-change/evals/retry-reuse/grade.json b/ai/evals/smallest-change/evals/retry-reuse/grade.json new file mode 100644 index 00000000..46da98bf --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/grade.json @@ -0,0 +1,5 @@ +{ + "require_added": { + "app/webhooks.py": "\\bwith_retries\\b" + } +} diff --git a/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py b/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py new file mode 100644 index 00000000..24cb76d5 --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py @@ -0,0 +1,43 @@ +import unittest +from unittest import mock + +from app.webhooks import DeliveryError, deliver + +URL = "https://example.test/hook" + + +@mock.patch("time.sleep") +class DeliverRetryTest(unittest.TestCase): + def test_retries_until_the_endpoint_accepts(self, _sleep): + post = mock.Mock(side_effect=[500, 503, 200]) + + deliver(URL, {}, post) + + self.assertEqual(post.call_count, 3) + + def test_gives_up_after_three_attempts(self, _sleep): + post = mock.Mock(return_value=500) + + with self.assertRaises(DeliveryError): + deliver(URL, {}, post) + + self.assertEqual(post.call_count, 3) + + def test_sends_once_when_the_first_attempt_succeeds(self, _sleep): + post = mock.Mock(return_value=200) + + deliver(URL, {}, post) + + post.assert_called_once() + + def test_does_not_retry_other_errors(self, _sleep): + post = mock.Mock(side_effect=ConnectionResetError()) + + with self.assertRaises(ConnectionResetError): + deliver(URL, {}, post) + + post.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/blog/__init__.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/__init__.py similarity index 100% rename from ai/evals/smallest-change/evals/permalink-reuse/repo/blog/__init__.py rename to ai/evals/smallest-change/evals/retry-reuse/repo/app/__init__.py diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py new file mode 100644 index 00000000..76e5271c --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py @@ -0,0 +1,15 @@ +import time + + +def with_retries(fn, *, attempts, retry_on, delay=0.5): + """Calls fn() until it returns, making at most `attempts` calls. + + Retries only the exception types in `retry_on`, waiting `delay` seconds and then twice as long before each retry. Raises the last exception when every attempt fails. + """ + for attempt in range(1, attempts + 1): + try: + return fn() + except retry_on: + if attempt == attempts: + raise + time.sleep(delay * 2 ** (attempt - 1)) diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py new file mode 100644 index 00000000..16854948 --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py @@ -0,0 +1,6 @@ +from app.retry import with_retries + + +def save(store, key, data): + """Writes data to the blob store, retrying writes that time out.""" + return with_retries(lambda: store.put(key, data), attempts=5, retry_on=(TimeoutError,)) diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py new file mode 100644 index 00000000..e40eefab --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py @@ -0,0 +1,12 @@ +import json + + +class DeliveryError(Exception): + """The webhook endpoint answered with a non-2xx status.""" + + +def deliver(webhook_url, payload, post): + """Sends payload as JSON to webhook_url with post(url, body), which returns the HTTP status.""" + status = post(webhook_url, json.dumps(payload)) + if not 200 <= status < 300: + raise DeliveryError(f"{webhook_url} returned {status}") diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml b/ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml similarity index 80% rename from ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml rename to ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml index bb7f52a8..3473e5af 100644 --- a/ai/evals/smallest-change/evals/permalink-reuse/repo/pyproject.toml +++ b/ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "blog" +name = "notifier" version = "0.1.0" requires-python = ">=3.9" dependencies = [] diff --git a/ai/evals/smallest-change/evals/permalink-reuse/repo/tests/__init__.py b/ai/evals/smallest-change/evals/retry-reuse/repo/tests/__init__.py similarity index 100% rename from ai/evals/smallest-change/evals/permalink-reuse/repo/tests/__init__.py rename to ai/evals/smallest-change/evals/retry-reuse/repo/tests/__init__.py diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py b/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py new file mode 100644 index 00000000..8a0ff5fb --- /dev/null +++ b/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py @@ -0,0 +1,32 @@ +import unittest +from unittest import mock + +from app.storage import save +from app.webhooks import DeliveryError, deliver + + +class SaveTest(unittest.TestCase): + @mock.patch("time.sleep") + def test_retries_timeouts(self, _sleep): + store = mock.Mock() + store.put.side_effect = [TimeoutError(), "ok"] + + self.assertEqual(save(store, "k", b"data"), "ok") + self.assertEqual(store.put.call_count, 2) + + +class DeliverTest(unittest.TestCase): + def test_posts_the_payload_as_json(self): + post = mock.Mock(return_value=204) + + deliver("https://example.test/hook", {"event": "signup"}, post) + + post.assert_called_once_with("https://example.test/hook", '{"event": "signup"}') + + def test_raises_on_an_error_status(self): + with self.assertRaises(DeliveryError): + deliver("https://example.test/hook", {}, mock.Mock(return_value=500)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh b/ai/evals/smallest-change/evals/retry-reuse/scaffold.sh similarity index 100% rename from ai/evals/smallest-change/evals/permalink-reuse/scaffold.sh rename to ai/evals/smallest-change/evals/retry-reuse/scaffold.sh diff --git a/ai/evals/smallest-change/evals/scaffold.sh b/ai/evals/smallest-change/evals/scaffold.sh index a67b95de..4bceeb3f 100755 --- a/ai/evals/smallest-change/evals/scaffold.sh +++ b/ai/evals/smallest-change/evals/scaffold.sh @@ -9,7 +9,8 @@ shared_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cp -R "${case_dir}/repo/." . mkdir -p .eval cp -R "${case_dir}/hidden" .eval/hidden -cp "${case_dir}/grade.json" "${shared_dir}/grade.py" .eval/ +cp "${shared_dir}/grade.py" .eval/ +if [[ -f "${case_dir}/grade.json" ]]; then cp "${case_dir}/grade.json" .eval/; fi # The eval run gets a fresh HOME. A fresh HOME breaks version-manager shims such as pyenv's. The agent's python3 then resolves to the first one on PATH outside HOME, and grade.sh grades with that same interpreter. python="$(type -ap python3 | grep -v "^${HOME}/" | head -n 1 || true)" @@ -17,7 +18,7 @@ python="${python:-$(command -v python3)}" # shellcheck disable=SC2016 # grade.sh expands $CLAUDE_PROJECT_DIR when the hook runs it. printf '#!/usr/bin/env bash\nexec "%s" -I "$CLAUDE_PROJECT_DIR/.eval/grade.py" "$CLAUDE_PROJECT_DIR"\n' "$python" >.eval/grade.sh -# Tool output such as uv.lock or a virtualenv would otherwise count toward the diff-size grader. +# Tool output such as uv.lock or a virtualenv would otherwise count toward the size graders. printf '%s\n' .eval/ __pycache__/ .venv/ .pytest_cache/ build/ '*.egg-info/' uv.lock >.gitignore git init -q diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml b/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml index 693d00df..42686d7b 100644 --- a/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml +++ b/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml @@ -16,10 +16,25 @@ graders: pattern: "^hidden_tests: pass$" flags: m weight: 2 - - name: diff-size + - name: size-le-5 type: regex target: {source: file, path: eval-result.txt} - pattern: "^size: ok$" + pattern: "^size_le_5: ok$" + flags: m + - name: size-le-10 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_10: ok$" + flags: m + - name: size-le-20 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_20: ok$" + flags: m + - name: size-le-40 + type: regex + target: {source: file, path: eval-result.txt} + pattern: "^size_le_40: ok$" flags: m - name: no-new-dependency type: regex diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/grade.json b/ai/evals/smallest-change/evals/shared-bug-fix/grade.json deleted file mode 100644 index 71b5ddc3..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/grade.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "max_lines": 8 -} From 2135ee01564284280293ded4d03ac60a458ba822 Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Wed, 28 Oct 2026 19:10:33 -0700 Subject: [PATCH 5/5] Remove the smallest-change eval suite Claude-Session: https://claude.ai/code/session_01AovXvYri48wFhqGV3epaMi --- .github/workflows/test.yml | 1 - ai/README.md | 5 -- .../.claude-plugin/plugin.json | 5 -- ai/evals/smallest-change/.gitignore | 2 - ai/evals/smallest-change/README.md | 51 ----------- ai/evals/smallest-change/compare.py | 42 --------- .../evals/csv-export/case.yaml | 48 ---------- .../evals/csv-export/grade.json | 5 -- .../evals/csv-export/hidden/test_hidden.py | 41 --------- .../evals/csv-export/repo/app/__init__.py | 0 .../evals/csv-export/repo/app/report.py | 12 --- .../evals/csv-export/repo/pyproject.toml | 5 -- .../evals/csv-export/repo/tests/__init__.py | 0 .../csv-export/repo/tests/test_report.py | 15 ---- .../evals/csv-export/scaffold.sh | 3 - ai/evals/smallest-change/evals/grade.py | 87 ------------------- .../evals/retry-reuse/case.yaml | 48 ---------- .../evals/retry-reuse/grade.json | 5 -- .../evals/retry-reuse/hidden/test_hidden.py | 43 --------- .../evals/retry-reuse/repo/app/__init__.py | 0 .../evals/retry-reuse/repo/app/retry.py | 15 ---- .../evals/retry-reuse/repo/app/storage.py | 6 -- .../evals/retry-reuse/repo/app/webhooks.py | 12 --- .../evals/retry-reuse/repo/pyproject.toml | 5 -- .../evals/retry-reuse/repo/tests/__init__.py | 0 .../evals/retry-reuse/repo/tests/test_app.py | 32 ------- .../evals/retry-reuse/scaffold.sh | 3 - ai/evals/smallest-change/evals/scaffold.sh | 29 ------- .../evals/shared-bug-fix/case.yaml | 43 --------- .../shared-bug-fix/hidden/test_hidden.py | 29 ------- .../evals/shared-bug-fix/repo/pyproject.toml | 5 -- .../shared-bug-fix/repo/shop/__init__.py | 0 .../evals/shared-bug-fix/repo/shop/cart.py | 15 ---- .../evals/shared-bug-fix/repo/shop/money.py | 10 --- .../evals/shared-bug-fix/repo/shop/refunds.py | 9 -- .../shared-bug-fix/repo/tests/__init__.py | 0 .../shared-bug-fix/repo/tests/test_shop.py | 24 ----- .../evals/shared-bug-fix/scaffold.sh | 3 - ai/evals/smallest-change/hooks/hooks.json | 24 ----- ai/evals/smallest-change/rule.md | 15 ---- ai/evals/smallest-change/run.sh | 27 ------ ai/tests/test-eval-rule-sync.sh | 22 ----- 42 files changed, 746 deletions(-) delete mode 100644 ai/evals/smallest-change/.claude-plugin/plugin.json delete mode 100644 ai/evals/smallest-change/.gitignore delete mode 100644 ai/evals/smallest-change/README.md delete mode 100644 ai/evals/smallest-change/compare.py delete mode 100644 ai/evals/smallest-change/evals/csv-export/case.yaml delete mode 100644 ai/evals/smallest-change/evals/csv-export/grade.json delete mode 100644 ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py delete mode 100644 ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py delete mode 100644 ai/evals/smallest-change/evals/csv-export/repo/app/report.py delete mode 100644 ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml delete mode 100644 ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py delete mode 100644 ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py delete mode 100755 ai/evals/smallest-change/evals/csv-export/scaffold.sh delete mode 100644 ai/evals/smallest-change/evals/grade.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/case.yaml delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/grade.json delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/__init__.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/tests/__init__.py delete mode 100644 ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py delete mode 100755 ai/evals/smallest-change/evals/retry-reuse/scaffold.sh delete mode 100755 ai/evals/smallest-change/evals/scaffold.sh delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/case.yaml delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py delete mode 100644 ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py delete mode 100755 ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh delete mode 100644 ai/evals/smallest-change/hooks/hooks.json delete mode 100644 ai/evals/smallest-change/rule.md delete mode 100755 ai/evals/smallest-change/run.sh delete mode 100755 ai/tests/test-eval-rule-sync.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b24c77ca..f4ff95d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,6 @@ jobs: ai/skills/wait-for-pr-reviews/scripts/tests/test-portable-skill.sh ai/skills/address-pr-reviews/scripts/tests/test-portable-skill.sh ai/tests/test-plain-writing-contract.sh - ai/tests/test-eval-rule-sync.sh ai/tests/test-ai-installers.sh ai/tests/test-install-claude-automation.sh ai/tests/test-lint-edited-file.sh diff --git a/ai/README.md b/ai/README.md index 9ca95bd3..a4c61038 100644 --- a/ai/README.md +++ b/ai/README.md @@ -50,10 +50,6 @@ CI enforces all three rules: the same script with `--check` rejects a copy that When `implementation-planner` marks a plan with two or more stages `**Execution**: staged`, `go` implements it one stage at a time. The planner marks a plan staged only when one session would run out of context before it finished the work, and `go --staged` or `--single-pass` overrides its choice. A fresh agent writes each stage, a second fresh agent checks it against the stage's success criteria and `Check` command, and the parent commits it, so an interrupted run resumes in the stage it stopped in. `implementation-planner` writes the `Check` command and a `## Decisions` section that the stage agents follow. -## Evals - -`ai/evals/` holds `claude plugin eval` suites. [`ai/evals/smallest-change/`](evals/smallest-change/README.md) compares implementation with and without the Smallest complete change rule that `go` gives its implementers. - ## Fresh review processes `go` supports Claude Code and Codex. It passes its current harness explicitly to `skills/go/scripts/run-review.py`, which starts a fresh review in the same checkout. Claude uses print mode; Codex uses `exec`. Both save the review result under the checkout's `.notes/` directory so the parent can resume at validation after a context clear. The runner requires Python 3.11 or later and the selected CLI on `PATH`; it never substitutes the other harness. The installed `review-code` skill must support `REVIEW_CODE_REVIEW_DIR`; update it from its source repository with `bin/setup` if the runner reports an older installation. Review sessions, reports, temporary checkouts, and hook files stay under each attempt in `.notes/go-reviews/`. The Codex child receives no write grants for shared review caches. After success, the parent copies the report into the shared review history, where it survives removal of the implementation worktree. A Claude review runs on the accounts that docket's `config.toml` names in `claude_config_dir` and `claude_fallback_config_dirs`, in the order docket's rotation picks them. The runner skips each account that a usage limit in docket's `limits.jsonl` still applies to. A limit applies to every config directory signed in to the login (one account in one organization) that docket recorded it with. `run-review.py accounts` prints that rotation, the limits docket recorded, and the order a review would try the accounts in now, as JSON, and `usage-all` shows it below each account's usage. Without a docket config, the review runs on the current account. The runner refuses an account that is not signed in or has no `review-code` skill. When an account hits its usage limit before the review edits any files, the runner reviews again on the next account that is signed in to a different login, and then on the current account if the list leaves it out. A run with `--since-run ` copies that earlier run's report into its own attempt and asks `review-code` for an `--append` review of the changes since that report. @@ -87,7 +83,6 @@ ai/skills/wait-for-pr-reviews/scripts/tests/test-pending-reviews.sh ai/skills/wait-for-pr-reviews/scripts/tests/test-portable-skill.sh ai/skills/address-pr-reviews/scripts/tests/test-portable-skill.sh ai/tests/test-plain-writing-contract.sh -ai/tests/test-eval-rule-sync.sh python3 ai/skills/plain-writing/scripts/tests/test_plain_writing_lint.py ai/tests/test-ai-installers.sh ai/tests/test-command-log.sh diff --git a/ai/evals/smallest-change/.claude-plugin/plugin.json b/ai/evals/smallest-change/.claude-plugin/plugin.json deleted file mode 100644 index be50ce30..00000000 --- a/ai/evals/smallest-change/.claude-plugin/plugin.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "smallest-change-eval", - "version": "0.1.0", - "description": "Eval-only plugin that adds /go's Smallest complete change rule to the session context, so claude plugin eval can compare implementation with and without it." -} diff --git a/ai/evals/smallest-change/.gitignore b/ai/evals/smallest-change/.gitignore deleted file mode 100644 index bc74b7b8..00000000 --- a/ai/evals/smallest-change/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -results/ -evals/results/ diff --git a/ai/evals/smallest-change/README.md b/ai/evals/smallest-change/README.md deleted file mode 100644 index a1f7e0a2..00000000 --- a/ai/evals/smallest-change/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Smallest complete change eval - -An eval-only plugin that measures what the Smallest complete change rule in `ai/skills/go/SKILL.md` does to a plain implementation task. It never runs /go, so a run opens no PR and requests no review. - -## What each arm loads - -- **With:** this plugin. A `SessionStart` hook prints `rule.md` into the session context, and a `Stop` hook grades the workspace. The plugin has no skill, so the rule is in context on every run, and the result does not depend on whether the model chooses to invoke a skill. /go gives its implementers this rule together with the Judgment calls rule, but this arm gets this rule alone, so the "easy-to-reverse call" that item 1 mentions has no definition here. -- **Without:** a baseline plugin that `run.sh` builds under `results//baseline-plugin/` for each run. It holds a copy of this plugin's `evals` and this plugin's hooks minus the `SessionStart` hook, so both arms run the same cases and grade the same way. The eval refuses an `evals` folder that links outside its plugin, which is why the baseline holds a copy. - -The eval's own `--ablation with-without` does not work here. Its baseline arm loads no plugin at all, including plugins that a case lists, and the eval ignores the workspace's `.claude/settings.json`. That leaves no way to grade the baseline arm, so `run.sh` runs each plugin as a single-arm eval instead. - -`rule.md` is a copy of the rule from `ai/skills/go/SKILL.md`. `ai/tests/test-eval-rule-sync.sh` fails when the two copies differ. - -## Cases - -Each case is a small Python repo where an over-built answer is tempting. - -| Case | Task | What over-building looks like | -| --- | --- | --- | -| `csv-export` | Add `write_csv(rows, path)` | Hand-rolled quoting, or delimiter and encoding options nobody asked for | -| `retry-reuse` | Retry webhook deliveries that fail with `DeliveryError` | A new retry loop, backoff class, or retry option when `app/retry.py` already has `with_retries` | -| `shared-bug-fix` | Fix a thousands-separator bug reported against the cart | Patching only the cart, when the refund path calls the same parser | - -`shared-bug-fix` guards the other direction: the rule must shrink the solution, not the scope, so a caller-only patch fails its hidden tests. - -## How grading works - -`claude plugin eval` has no grader that runs a command, so a `Stop` hook does the grading. `evals/scaffold.sh` copies the case's `repo/` into the workspace and commits it. It also puts the hidden tests, `grade.py`, and a `grade.sh` that runs it in `.eval/`. When the session ends, both plugins' `Stop` hook runs `grade.sh`, which writes `eval-result.txt`. The case's regex graders read that file: - -- `hidden_tests: pass` when the hidden tests and the repo's own tests pass. This grader has weight 2. -- `size_le_5`, `size_le_10`, `size_le_20`, and `size_le_40: ok` when the production lines added or removed stay within that many lines. Each limit is its own grader, so the pass counts show roughly how many lines each arm changed. Lines under `tests/` and in `test_*` files do not count. -- `dependencies: unchanged` when no dependency file (`pyproject.toml`, `requirements*.txt`, `setup.py`, `setup.cfg`, `Pipfile`) changed. -- `expected_code: ok` (`csv-export` and `retry-reuse` only) when the change adds a line that uses the `csv` module or `with_retries`. The case's `grade.json` sets the pattern. - -`eval-result.txt` also records the exact line count and the changed files. `--keep-temp` leaves each workspace on disk, sealed, and the eval prints how to open it. - -## Running it - -Each run is a full Claude session, and `run.sh` runs every case in both arms, so the full suite (3 cases × 3 runs × 2 arms) is 18 sessions. Start with one case and one run per arm: - -```bash -ai/evals/smallest-change/run.sh --case csv-export --runs 1 --max-cost-usd 1 -``` - -`run.sh` passes its arguments to both `claude plugin eval` calls, so `--max-cost-usd` caps each arm separately. It writes both results under `results//` and then prints each grader's pass count in each arm with `compare.py`. - -The eval refuses to grant Bash when `~/.ssh` holds a symbolic link, because its sandbox cannot keep the link's target out of reach. On such a machine, add `--disallow-tools Bash`. The agent then cannot run tests in either arm, but grading still works, because the `Stop` hook does not need the agent's Bash. - -`--max-cost-usd 0` loads and plans every case without starting a run, which checks that the cases parse. - -The hidden tests sit in the workspace's `.eval/` folder. Git ignores that folder, but the agent can still find and read it in either arm. diff --git a/ai/evals/smallest-change/compare.py b/ai/evals/smallest-change/compare.py deleted file mode 100644 index 5dadc3a8..00000000 --- a/ai/evals/smallest-change/compare.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Prints each grader's pass rate with and without the rule, from the --json output of two single-arm eval runs. - -Usage: compare.py -""" - -import json -import sys - - -def pass_rates(path): - """Returns {case: {grader: (passed runs, total runs)}} and the run's total cost.""" - result = json.load(open(path)) - rates = {} - for case in result["cases"]: - runs = [run for arm_runs in case["arms"].values() for run in arm_runs] - graders = rates.setdefault(case["name"], {}) - for grader in case["graders"]: - passed = sum( - 1 for run in runs for g in run["graders"] if g["name"] == grader["name"] and g["passed"] - ) - graders[grader["name"]] = (passed, len(runs)) - return rates, result["costUsd"] - - -def main(): - with_rates, with_cost = pass_rates(sys.argv[1]) - without_rates, without_cost = pass_rates(sys.argv[2]) - - print(f"{'case':<18} {'grader':<20} {'with':>6} {'without':>8}") - for case in sorted(with_rates.keys() | without_rates.keys()): - graders = with_rates.get(case, {}).keys() | without_rates.get(case, {}).keys() - for grader in sorted(graders): - cells = [] - for rates in (with_rates, without_rates): - passed, total = rates.get(case, {}).get(grader, (0, 0)) - cells.append(f"{passed}/{total}" if total else "-") - print(f"{case:<18} {grader:<20} {cells[0]:>6} {cells[1]:>8}") - print(f"\ncost: with ${with_cost:.2f}, without ${without_cost:.2f}") - - -if __name__ == "__main__": - main() diff --git a/ai/evals/smallest-change/evals/csv-export/case.yaml b/ai/evals/smallest-change/evals/csv-export/case.yaml deleted file mode 100644 index 4a06b7bb..00000000 --- a/ai/evals/smallest-change/evals/csv-export/case.yaml +++ /dev/null @@ -1,48 +0,0 @@ -schema_version: "1.1" -name: csv-export -description: Writing a CSV export, where the standard library's csv module beats hand-rolled quoting and unrequested options. -context: - scaffold_script: scaffold.sh -execution: - max_turns: 25 - timeout_seconds: 300 - allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] - prompt: | - Add a `write_csv(rows, path)` function to `app/report.py`. It writes the rows that `build_rows` returns to a CSV file at `path`: a header row with the column names, then one line per row. Customer names come from a signup form, so they can contain any text. -graders: - - name: hidden-tests - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^hidden_tests: pass$" - flags: m - weight: 2 - - name: size-le-5 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_5: ok$" - flags: m - - name: size-le-10 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_10: ok$" - flags: m - - name: size-le-20 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_20: ok$" - flags: m - - name: size-le-40 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_40: ok$" - flags: m - - name: no-new-dependency - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^dependencies: unchanged$" - flags: m - - name: uses-csv-module - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^expected_code: ok$" - flags: m diff --git a/ai/evals/smallest-change/evals/csv-export/grade.json b/ai/evals/smallest-change/evals/csv-export/grade.json deleted file mode 100644 index ac71b5a9..00000000 --- a/ai/evals/smallest-change/evals/csv-export/grade.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "require_added": { - "app/report.py": "\\bcsv\\." - } -} diff --git a/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py b/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py deleted file mode 100644 index be8af88a..00000000 --- a/ai/evals/smallest-change/evals/csv-export/hidden/test_hidden.py +++ /dev/null @@ -1,41 +0,0 @@ -import csv -import os -import tempfile -import unittest -from decimal import Decimal - -from app.report import write_csv - - -def read_back(path): - with open(path, newline="") as f: - return list(csv.reader(f)) - - -class WriteCsvTest(unittest.TestCase): - def setUp(self): - handle, self.path = tempfile.mkstemp(suffix=".csv") - os.close(handle) - self.addCleanup(os.remove, self.path) - - def test_writes_header_then_rows(self): - write_csv([{"order_id": 1, "customer": "Ada", "total": Decimal("6.00")}], self.path) - - self.assertEqual(read_back(self.path), [["order_id", "customer", "total"], ["1", "Ada", "6.00"]]) - - def test_quotes_commas_quotes_and_newlines_in_customer_names(self): - names = ['Smith, Jo', 'The "Best" Shop', "Line one\nLine two"] - rows = [{"order_id": i, "customer": name, "total": Decimal("1.00")} for i, name in enumerate(names)] - - write_csv(rows, self.path) - - self.assertEqual([row[1] for row in read_back(self.path)[1:]], names) - - def test_writes_only_the_header_when_there_are_no_rows(self): - write_csv([], self.path) - - self.assertEqual(read_back(self.path), [["order_id", "customer", "total"]]) - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py b/ai/evals/smallest-change/evals/csv-export/repo/app/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/csv-export/repo/app/report.py b/ai/evals/smallest-change/evals/csv-export/repo/app/report.py deleted file mode 100644 index 4e46e676..00000000 --- a/ai/evals/smallest-change/evals/csv-export/repo/app/report.py +++ /dev/null @@ -1,12 +0,0 @@ -from decimal import Decimal - -COLUMNS = ("order_id", "customer", "total") - - -def build_rows(orders): - """Returns one row per order, in the column order of COLUMNS.""" - rows = [] - for order in orders: - total = sum((Decimal(item["price"]) * item["quantity"] for item in order["items"]), Decimal("0")) - rows.append({"order_id": order["id"], "customer": order["customer"], "total": total}) - return rows diff --git a/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml b/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml deleted file mode 100644 index 2f5ab3a9..00000000 --- a/ai/evals/smallest-change/evals/csv-export/repo/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[project] -name = "orders" -version = "0.1.0" -requires-python = ">=3.9" -dependencies = [] diff --git a/ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py b/ai/evals/smallest-change/evals/csv-export/repo/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py b/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py deleted file mode 100644 index 190a24a3..00000000 --- a/ai/evals/smallest-change/evals/csv-export/repo/tests/test_report.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest -from decimal import Decimal - -from app.report import build_rows - - -class BuildRowsTest(unittest.TestCase): - def test_totals_each_order(self): - orders = [{"id": 7, "customer": "Ada", "items": [{"price": "2.50", "quantity": 2}, {"price": "1.00", "quantity": 1}]}] - - self.assertEqual(build_rows(orders), [{"order_id": 7, "customer": "Ada", "total": Decimal("6.00")}]) - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/csv-export/scaffold.sh b/ai/evals/smallest-change/evals/csv-export/scaffold.sh deleted file mode 100755 index 1aace498..00000000 --- a/ai/evals/smallest-change/evals/csv-export/scaffold.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/evals/grade.py b/ai/evals/smallest-change/evals/grade.py deleted file mode 100644 index 33361084..00000000 --- a/ai/evals/smallest-change/evals/grade.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Grades an eval workspace after the agent stops and writes eval-result.txt for the case's regex graders. - -Usage: grade.py - -An optional grade.json in the same folder sets `require_added`, a map from a file path to a regex that some added line in that file must match. -""" - -import json -import os -import re -import subprocess -import sys -from pathlib import Path - -RESULT = "eval-result.txt" -DEPENDENCY_FILES = re.compile(r"(^|/)(pyproject\.toml|setup\.py|setup\.cfg|Pipfile|requirements[^/]*\.txt)$") -# Each limit gets its own grader, so the pass counts show roughly how many production lines each arm changed. -SIZE_LIMITS = (5, 10, 20, 40) - - -def git(root, *args): - return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True, check=True).stdout - - -def is_test(path): - return path.startswith("tests/") or Path(path).name.startswith("test_") - - -def run_tests(root, start_dir, top_dir): - env = {**os.environ, "PYTHONPATH": str(root), "PYTHONDONTWRITEBYTECODE": "1"} - result = subprocess.run( - [sys.executable, "-m", "unittest", "discover", "-s", start_dir, "-t", top_dir], - cwd=root, - env=env, - capture_output=True, - text=True, - timeout=120, - ) - return result.returncode == 0, result.stderr[-2000:] - - -def main(): - root = Path(sys.argv[1]) - config_path = Path(__file__).parent / "grade.json" - config = json.loads(config_path.read_text()) if config_path.exists() else {} - base = (Path(__file__).parent / "base").read_text().strip() - - hidden_ok, hidden_log = run_tests(root, ".eval/hidden", ".eval/hidden") - repo_ok, repo_log = run_tests(root, "tests", ".") - - # Intent-to-add makes new files show up in `git diff` without staging their contents. - git(root, "add", "-N", "--", ".", f":!{RESULT}") - changed = [] - for line in git(root, "diff", "--numstat", base, "--", ".", f":!{RESULT}").splitlines(): - added, removed, path = line.split("\t", 2) - changed.append((path, int(added) if added != "-" else 0, int(removed) if removed != "-" else 0)) - - production_lines = sum(a + r for path, a, r in changed if not is_test(path)) - dependency_files = [path for path, _, _ in changed if DEPENDENCY_FILES.search(path)] - - lines = [ - f"hidden_tests: {'pass' if hidden_ok and repo_ok else 'fail'}", - f"production_lines_changed: {production_lines}", - *(f"size_le_{limit}: {'ok' if production_lines <= limit else 'over'}" for limit in SIZE_LIMITS), - f"dependencies: {'changed' if dependency_files else 'unchanged'}", - ] - if config.get("require_added"): - missing = [] - for path, pattern in config["require_added"].items(): - added_lines = [ - l[1:] for l in git(root, "diff", "-U0", base, "--", path).splitlines() - if l.startswith("+") and not l.startswith("+++") - ] - if not any(re.search(pattern, l) for l in added_lines): - missing.append(path) - lines.append(f"expected_code: {'missing in ' + ', '.join(missing) if missing else 'ok'}") - lines.append(f"changed_files: {', '.join(path for path, _, _ in changed) or 'none'}") - if not hidden_ok: - lines += ["", "hidden test output:", hidden_log] - if not repo_ok: - lines += ["", "repo test output:", repo_log] - - (root / RESULT).write_text("\n".join(lines) + "\n") - - -if __name__ == "__main__": - main() diff --git a/ai/evals/smallest-change/evals/retry-reuse/case.yaml b/ai/evals/smallest-change/evals/retry-reuse/case.yaml deleted file mode 100644 index 7b225059..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/case.yaml +++ /dev/null @@ -1,48 +0,0 @@ -schema_version: "1.1" -name: retry-reuse -description: Adding retries to webhook delivery, where the codebase already has a retry helper and retry options or a backoff class are tempting. -context: - scaffold_script: scaffold.sh -execution: - max_turns: 25 - timeout_seconds: 300 - allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] - prompt: | - Webhook endpoints sometimes answer with a 5xx status. Make `deliver` in `app/webhooks.py` retry a delivery that fails with `DeliveryError`, making at most 3 attempts in total. -graders: - - name: hidden-tests - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^hidden_tests: pass$" - flags: m - weight: 2 - - name: size-le-5 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_5: ok$" - flags: m - - name: size-le-10 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_10: ok$" - flags: m - - name: size-le-20 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_20: ok$" - flags: m - - name: size-le-40 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_40: ok$" - flags: m - - name: no-new-dependency - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^dependencies: unchanged$" - flags: m - - name: reuses-retry-helper - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^expected_code: ok$" - flags: m diff --git a/ai/evals/smallest-change/evals/retry-reuse/grade.json b/ai/evals/smallest-change/evals/retry-reuse/grade.json deleted file mode 100644 index 46da98bf..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/grade.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "require_added": { - "app/webhooks.py": "\\bwith_retries\\b" - } -} diff --git a/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py b/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py deleted file mode 100644 index 24cb76d5..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/hidden/test_hidden.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest -from unittest import mock - -from app.webhooks import DeliveryError, deliver - -URL = "https://example.test/hook" - - -@mock.patch("time.sleep") -class DeliverRetryTest(unittest.TestCase): - def test_retries_until_the_endpoint_accepts(self, _sleep): - post = mock.Mock(side_effect=[500, 503, 200]) - - deliver(URL, {}, post) - - self.assertEqual(post.call_count, 3) - - def test_gives_up_after_three_attempts(self, _sleep): - post = mock.Mock(return_value=500) - - with self.assertRaises(DeliveryError): - deliver(URL, {}, post) - - self.assertEqual(post.call_count, 3) - - def test_sends_once_when_the_first_attempt_succeeds(self, _sleep): - post = mock.Mock(return_value=200) - - deliver(URL, {}, post) - - post.assert_called_once() - - def test_does_not_retry_other_errors(self, _sleep): - post = mock.Mock(side_effect=ConnectionResetError()) - - with self.assertRaises(ConnectionResetError): - deliver(URL, {}, post) - - post.assert_called_once() - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/__init__.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py deleted file mode 100644 index 76e5271c..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/repo/app/retry.py +++ /dev/null @@ -1,15 +0,0 @@ -import time - - -def with_retries(fn, *, attempts, retry_on, delay=0.5): - """Calls fn() until it returns, making at most `attempts` calls. - - Retries only the exception types in `retry_on`, waiting `delay` seconds and then twice as long before each retry. Raises the last exception when every attempt fails. - """ - for attempt in range(1, attempts + 1): - try: - return fn() - except retry_on: - if attempt == attempts: - raise - time.sleep(delay * 2 ** (attempt - 1)) diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py deleted file mode 100644 index 16854948..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/repo/app/storage.py +++ /dev/null @@ -1,6 +0,0 @@ -from app.retry import with_retries - - -def save(store, key, data): - """Writes data to the blob store, retrying writes that time out.""" - return with_retries(lambda: store.put(key, data), attempts=5, retry_on=(TimeoutError,)) diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py b/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py deleted file mode 100644 index e40eefab..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/repo/app/webhooks.py +++ /dev/null @@ -1,12 +0,0 @@ -import json - - -class DeliveryError(Exception): - """The webhook endpoint answered with a non-2xx status.""" - - -def deliver(webhook_url, payload, post): - """Sends payload as JSON to webhook_url with post(url, body), which returns the HTTP status.""" - status = post(webhook_url, json.dumps(payload)) - if not 200 <= status < 300: - raise DeliveryError(f"{webhook_url} returned {status}") diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml b/ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml deleted file mode 100644 index 3473e5af..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/repo/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[project] -name = "notifier" -version = "0.1.0" -requires-python = ">=3.9" -dependencies = [] diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/tests/__init__.py b/ai/evals/smallest-change/evals/retry-reuse/repo/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py b/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py deleted file mode 100644 index 8a0ff5fb..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/repo/tests/test_app.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest -from unittest import mock - -from app.storage import save -from app.webhooks import DeliveryError, deliver - - -class SaveTest(unittest.TestCase): - @mock.patch("time.sleep") - def test_retries_timeouts(self, _sleep): - store = mock.Mock() - store.put.side_effect = [TimeoutError(), "ok"] - - self.assertEqual(save(store, "k", b"data"), "ok") - self.assertEqual(store.put.call_count, 2) - - -class DeliverTest(unittest.TestCase): - def test_posts_the_payload_as_json(self): - post = mock.Mock(return_value=204) - - deliver("https://example.test/hook", {"event": "signup"}, post) - - post.assert_called_once_with("https://example.test/hook", '{"event": "signup"}') - - def test_raises_on_an_error_status(self): - with self.assertRaises(DeliveryError): - deliver("https://example.test/hook", {}, mock.Mock(return_value=500)) - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/retry-reuse/scaffold.sh b/ai/evals/smallest-change/evals/retry-reuse/scaffold.sh deleted file mode 100755 index 1aace498..00000000 --- a/ai/evals/smallest-change/evals/retry-reuse/scaffold.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/evals/scaffold.sh b/ai/evals/smallest-change/evals/scaffold.sh deleted file mode 100755 index 4bceeb3f..00000000 --- a/ai/evals/smallest-change/evals/scaffold.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Builds one case's workspace in the current directory: the case's repo/ as a fresh git commit, plus a hidden .eval/ folder that the grader plugin's Stop hook runs when the session ends. -# Usage: scaffold.sh -set -euo pipefail - -case_dir="$1" -shared_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -cp -R "${case_dir}/repo/." . -mkdir -p .eval -cp -R "${case_dir}/hidden" .eval/hidden -cp "${shared_dir}/grade.py" .eval/ -if [[ -f "${case_dir}/grade.json" ]]; then cp "${case_dir}/grade.json" .eval/; fi - -# The eval run gets a fresh HOME. A fresh HOME breaks version-manager shims such as pyenv's. The agent's python3 then resolves to the first one on PATH outside HOME, and grade.sh grades with that same interpreter. -python="$(type -ap python3 | grep -v "^${HOME}/" | head -n 1 || true)" -python="${python:-$(command -v python3)}" -# shellcheck disable=SC2016 # grade.sh expands $CLAUDE_PROJECT_DIR when the hook runs it. -printf '#!/usr/bin/env bash\nexec "%s" -I "$CLAUDE_PROJECT_DIR/.eval/grade.py" "$CLAUDE_PROJECT_DIR"\n' "$python" >.eval/grade.sh - -# Tool output such as uv.lock or a virtualenv would otherwise count toward the size graders. -printf '%s\n' .eval/ __pycache__/ .venv/ .pytest_cache/ build/ '*.egg-info/' uv.lock >.gitignore - -git init -q -git add -A -# The scaffold runs with the operator's git config. The commit turns off that config's signing and hooks. -git -c user.name=eval -c user.email=eval@example.com -c commit.gpgsign=false -c core.hooksPath=/dev/null commit -qm "Initial commit" -# grade.py diffs against this commit, so a commit by the agent still counts. -git rev-parse HEAD >.eval/base diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml b/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml deleted file mode 100644 index 42686d7b..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/case.yaml +++ /dev/null @@ -1,43 +0,0 @@ -schema_version: "1.1" -name: shared-bug-fix -description: A bug report names one caller of a shared parser; the fix belongs in the parser, so a second caller that the report does not mention gets fixed too. -context: - scaffold_script: scaffold.sh -execution: - max_turns: 25 - timeout_seconds: 300 - allowed_tools: [Read, Glob, Grep, Edit, Write, Bash] - prompt: | - Bug report: adding an item priced `$1,299.00` to the cart fails with `ValueError: invalid amount: '$1,299.00'`. Fix it. -graders: - - name: hidden-tests - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^hidden_tests: pass$" - flags: m - weight: 2 - - name: size-le-5 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_5: ok$" - flags: m - - name: size-le-10 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_10: ok$" - flags: m - - name: size-le-20 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_20: ok$" - flags: m - - name: size-le-40 - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^size_le_40: ok$" - flags: m - - name: no-new-dependency - type: regex - target: {source: file, path: eval-result.txt} - pattern: "^dependencies: unchanged$" - flags: m diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py b/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py deleted file mode 100644 index 7f4431ab..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/hidden/test_hidden.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -from decimal import Decimal - -from shop.cart import Cart -from shop.money import parse_amount -from shop.refunds import refund_amount - - -class ThousandsSeparatorTest(unittest.TestCase): - def test_cart_accepts_thousands_separators(self): - cart = Cart() - cart.add_item("A", "$1,299.00") - - self.assertEqual(cart.total, Decimal("1299.00")) - - def test_refund_accepts_thousands_separators(self): - self.assertEqual(refund_amount(Decimal("5000"), "1,299.00"), Decimal("1299.00")) - - def test_plain_amounts_still_parse(self): - self.assertEqual(parse_amount(" $0.99 "), Decimal("0.99")) - - def test_garbage_still_raises_value_error(self): - for text in ["abc", "", "$"]: - with self.subTest(text=text), self.assertRaises(ValueError): - parse_amount(text) - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml b/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml deleted file mode 100644 index 6643ff01..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/repo/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[project] -name = "shop" -version = "0.1.0" -requires-python = ">=3.9" -dependencies = [] diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py deleted file mode 100644 index 7bf0b73b..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/cart.py +++ /dev/null @@ -1,15 +0,0 @@ -from decimal import Decimal - -from shop.money import parse_amount - - -class Cart: - def __init__(self): - self.items = [] - - def add_item(self, sku, price_text): - self.items.append((sku, parse_amount(price_text))) - - @property - def total(self): - return sum((price for _, price in self.items), Decimal("0")) diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py deleted file mode 100644 index 68cf7553..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/money.py +++ /dev/null @@ -1,10 +0,0 @@ -from decimal import Decimal, InvalidOperation - - -def parse_amount(text): - """Parses a price such as "$12.50" or "12.50" into a Decimal. Raises ValueError on anything else.""" - cleaned = text.strip().removeprefix("$") - try: - return Decimal(cleaned) - except InvalidOperation: - raise ValueError(f"invalid amount: {text!r}") from None diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py deleted file mode 100644 index 348a5f76..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/repo/shop/refunds.py +++ /dev/null @@ -1,9 +0,0 @@ -from shop.money import parse_amount - - -def refund_amount(order_total, amount_text): - """Returns the amount to refund. Raises ValueError when it exceeds the order total.""" - amount = parse_amount(amount_text) - if amount > order_total: - raise ValueError("refund exceeds order total") - return amount diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py b/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py deleted file mode 100644 index 7330f72f..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/repo/tests/test_shop.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest -from decimal import Decimal - -from shop.cart import Cart -from shop.refunds import refund_amount - - -class CartTest(unittest.TestCase): - def test_totals_items(self): - cart = Cart() - cart.add_item("A", "$12.50") - cart.add_item("B", "0.50") - - self.assertEqual(cart.total, Decimal("13.00")) - - -class RefundTest(unittest.TestCase): - def test_refuses_more_than_the_order_total(self): - with self.assertRaises(ValueError): - refund_amount(Decimal("5"), "$6") - - -if __name__ == "__main__": - unittest.main() diff --git a/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh b/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh deleted file mode 100755 index 1aace498..00000000 --- a/ai/evals/smallest-change/evals/shared-bug-fix/scaffold.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -case_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec bash "${case_dir}/../scaffold.sh" "${case_dir}" diff --git a/ai/evals/smallest-change/hooks/hooks.json b/ai/evals/smallest-change/hooks/hooks.json deleted file mode 100644 index 2afa237f..00000000 --- a/ai/evals/smallest-change/hooks/hooks.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "hooks": { - "SessionStart": [ - { - "hooks": [ - { - "type": "command", - "command": "cat \"${CLAUDE_PLUGIN_ROOT}/rule.md\"" - } - ] - } - ], - "Stop": [ - { - "hooks": [ - { - "type": "command", - "command": "bash \"$CLAUDE_PROJECT_DIR/.eval/grade.sh\"" - } - ] - } - ] - } -} diff --git a/ai/evals/smallest-change/rule.md b/ai/evals/smallest-change/rule.md deleted file mode 100644 index f0bc972f..00000000 --- a/ai/evals/smallest-change/rule.md +++ /dev/null @@ -1,15 +0,0 @@ -**Smallest complete change.** Before writing code, list every place the change must reach: callers, tests, fixtures, config, and exports. For each piece the plan does not spell out, take the first option on this list that fully works: - -1. Leave it out. A feature, option, or extension point that neither the plan nor the task asks for does not get built. Flag each one left out as an easy-to-reverse call. -2. Reuse what the codebase already has, such as a helper, service, or pattern, the way the surrounding code uses it. -3. Use the standard library or a platform feature, unless the project has its own version. -4. Use a dependency the project already installs. Do not add a dependency to save a few lines. -5. Write the least code that works and that a reader understands on one read. - -- The rule shrinks the solution, never the scope. Finish every part the change needs, including the callers, tests, and fixtures it breaks. -- Add no wrapper, abstraction, option, config, or type conversion that nothing in this change uses. -- Keep the codebase's existing layers, interfaces, and conventions. -- For a bug fix, find every caller of the function first, then fix the root cause once in the shared code. -- Code that moves or merges keeps its validation and error handling. -- Between two options of the same size, take the one that handles edge cases correctly. -- Never cut validation at trust boundaries, error handling that prevents data loss, security checks, or accessibility. diff --git a/ai/evals/smallest-change/run.sh b/ai/evals/smallest-change/run.sh deleted file mode 100755 index d05f9093..00000000 --- a/ai/evals/smallest-change/run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# Runs the smallest-change eval once with the rule (this plugin) and once without it (a baseline plugin built here), then prints the per-grader difference. -# Usage: run.sh [claude plugin eval options, such as --case csv-export --runs 1 --max-cost-usd 1] -set -euo pipefail - -here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -out="${here}/results/$(date -u +%Y-%m-%dT%H-%M-%SZ)" - -# The eval refuses an eval folder that links outside its plugin, so the baseline plugin holds a real copy of the cases. Its hooks are this plugin's hooks without the SessionStart hook that prints the rule. -baseline="${out}/baseline-plugin" -mkdir -p "${baseline}/.claude-plugin" "${baseline}/hooks" "${baseline}/evals" -printf '{"name": "smallest-change-baseline", "version": "0.1.0"}\n' >"${baseline}/.claude-plugin/plugin.json" -python3 -I -c 'import json, sys; hooks = json.load(open(sys.argv[1])); del hooks["hooks"]["SessionStart"]; json.dump(hooks, open(sys.argv[2], "w"), indent=2)' \ - "${here}/hooks/hooks.json" "${baseline}/hooks/hooks.json" -for entry in "${here}/evals"/*; do - [[ "$(basename "$entry")" == results ]] || cp -R "$entry" "${baseline}/evals/" -done - -# --threshold 0 makes a low score exit 0, so a nonzero exit means the run itself failed. -for arm in with without; do - plugin="$here" - [[ "$arm" == without ]] && plugin="$baseline" - claude plugin eval "$plugin" --ablation none --threshold 0 --scaffold --allow-tools Edit Write Bash \ - --no-publish --output-dir "${out}/${arm}" --json "${out}/${arm}.json" "$@" -done - -python3 -I "${here}/compare.py" "${out}/with.json" "${out}/without.json" diff --git a/ai/tests/test-eval-rule-sync.sh b/ai/tests/test-eval-rule-sync.sh deleted file mode 100755 index 871144f7..00000000 --- a/ai/tests/test-eval-rule-sync.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# The smallest-change eval injects its own copy of /go's Smallest complete change rule. This test checks that the copy matches the rule in the go skill. -set -uo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -AI_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -SKILL="${AI_DIR}/skills/go/SKILL.md" -RULE="${AI_DIR}/evals/smallest-change/rule.md" - -expected=$(awk '/^\*\*Smallest complete change\.\*\*/{p=1} /^#### /{p=0} p' "$SKILL") - -if [[ -z "$expected" ]]; then - echo "FAIL: no Smallest complete change block in ${SKILL}" - exit 1 -fi - -if diff -u <(printf '%s\n' "$expected") "$RULE"; then - echo "Results: 1 passed, 0 failed" -else - echo "FAIL: ${RULE} differs from the rule in ${SKILL}" - exit 1 -fi