diff --git a/docs/findings/0002-pack-cuts-search-loops.md b/docs/findings/0002-pack-cuts-search-loops.md new file mode 100644 index 00000000..d4459e1c --- /dev/null +++ b/docs/findings/0002-pack-cuts-search-loops.md @@ -0,0 +1,181 @@ +# Finding 0002 — An OCH pack cuts a coding agent's shell-hunting and file re-reads (search loops don't fire on fix tasks) + +- Status: **Preliminary — first live run, 2026-07-08.** Six SWE-bench Verified + tasks × 5 runs/arm on Claude Code / Sonnet 5 via Amazon Bedrock. **4 tasks + produced a valid two-arm comparison; 2 (the largest repos) overflowed the + model context** — itself a finding, see below. A signal, not a benchmark. +- Author: Bonk + Laith. +- Instrument: `codehub code-pack --variance-probe --insight` (Move 1), scoring + each run's tool-call trajectory against the TraceProbe (arXiv:2607.06184) + structural anti-pattern detectors, on real SWE-bench Verified tasks. +- Grounding: TraceProbe ran its 2,500 trajectories on **SWE-bench Verified**, so + these per-detector numbers are directly comparable to the paper's. + +## The headline + +Across the 4 tasks with a valid comparison, an OCH pack **suppressed every +anti-pattern that fired and introduced none:** + +- **Shell-over-Tool: 1.9 → 0.65 firings/run** (mean Δ +1.25) — the pack stops the + agent from shelling `grep`/`cat`/`find` to reconstruct structure it was handed. + It cut this on all 4 tasks. +- **Re-read Churn: 0.3 → 0.05 firings/run** (mean Δ +0.25) — fewer repeat reads of + the same file (fired on requests-1142). +- **Search Loop: 0 → 0.** Never fired in either arm. SWE-bench fix tasks give the + agent a specific bug, so it does not run the ≥10-action open-ended search + stretches TraceProbe's Search Loop detects. On these tasks the pack's win is + *shell-hunting and re-reads*, not loops — the honest, narrower claim. +- **Redundant Search: 0 → 0.** Same reason. + +This is the behavioral complement to Finding 0001's 2–4× token cut: the pack +doesn't just cost fewer tokens, it changes what the agent *does* — less shelling +out, fewer re-reads. + +## Second finding: a large-repo pack can overflow the model context + +The two pytest tasks **errored on every with-pack run** while their without-pack +arms ran clean. Cause: the assembled pack for `pytest` is **~5.2 MB (≈1.3M +tokens)** — past Claude's 1M window — so the model rejected it with "prompt is +too long." This is not a probe bug; it is a real product boundary. The default +`--budget` (100K tokens) bounds only the **AST-chunks** BOM item; `xrefs`, +`skeleton`, and `file-tree` are unbounded and dominate a large repo's pack. +**Action for OCH:** a whole-pack token ceiling (budget the assembled context, +not just the chunks), and the probe should detect an oversized pack and skip the +with-pack arm with a clear message rather than erroring all N runs. Tracked as a +follow-up; the without-pack pytest arms still gave clean baseline trajectories +(shell-over-tool 1.8–2.2/run, re-read churn up to 1.8/run — the behavior a +fitting pack would target). + +## The question + +Finding 0001 showed an OCH pack cuts a coding agent's **token usage 2–4×** by +letting it stop re-reading files to reconstruct structure. That is a *resource* +number. This finding asks the *behavioral* question underneath it: does the pack +change **what the agent does** — specifically, does it cut the anti-patterns +TraceProbe found are the most stable failure-associated clues, chiefly the +**search loop**? + +The hypothesis follows directly from OCH's thesis (hand the agent structure up +front, it stops hunting): with the pack in context, the agent should run fewer +long search/read stretches, re-read the same file less, and repeat fewer +searches. If it does not — if the pack cuts tokens but not loops — that is a real +result we publish too. **The number can come back null.** + +## What is measured + +For each task, the probe runs the agent N times with the pack and N times +without, holding commit / instruction / agent / model fixed (the same two-arm +design as Finding 0001). New in Move 1: every run's tool-call stream is captured +and normalized to TraceProbe's nine-type action taxonomy, then scored against +the four **structural** detectors — the ones whose predicates read only the +action list, no LLM labeler: + +| Detector | Fires when (frozen predicate) | +|---|---| +| **Search Loop** | ≥10 consecutive search/read actions with no write and no validation command between them | +| **Re-read Churn** | the same file is read ≥3× within a 10-action window with no intervening write | +| **Redundant Search** | the same normalized query recurs ≥2× within a 10-action window | +| **Shell-over-Tool** | a shell `cat`/`grep`/`rg`/`find` runs read/search work a structured tool covers | + +The report's headline is the **per-run `without − with` delta** for each +detector: positive means the pack suppressed the anti-pattern. + +### The four semantic detectors are deliberately out of scope + +TraceProbe defines eight structural + four semantic detectors. The semantic ones +(Phase Oscillation, and effect-labelled variants) need an LLM to label each +action's phase/effect — the same judge-oracle dependency that kept Finding +0001's headline on the directly-measured token number. Scoring them would import +labeler noise into a number we want to be a pure function of the trajectory. +They are a v2 concern. + +## Results + +Claude Code / Sonnet 5 on Bedrock, N=5 runs/arm. Per-run detector firings, +`without pack → with pack`: + +| Task | tokenOverhead | Search Loop | Re-read Churn | Redundant Search | Shell-over-Tool | +|---|---:|---|---|---|---| +| `pallets/flask-5014` | 11.0× | 0 → 0 | 0 → 0 | 0 → 0 | **1.6 → 0.4** | +| `psf/requests-1142` | 2.3× | 0 → 0 | **1.2 → 0.2** | 0 → 0 | **2.0 → 0.2** | +| `psf/requests-1724` | 7.3× | 0 → 0 | 0 → 0 | 0 → 0 | **3.0 → 2.0** | +| `psf/requests-1766` | 6.0× | 0 → 0 | 0 → 0 | 0 → 0 | **1.0 → 0.0** | +| **mean (4 valid)** | 6.6× | 0 → 0 | 0.3 → 0.05 | 0 → 0 | 1.9 → 0.65 | +| `pytest/pytest-10051` | — | \_context overflow\_ | | | (baseline 1.8/run) | +| `pytest/pytest-10081` | — | \_context overflow\_ | | | (baseline 2.2/run) | + +Positive delta = the pack suppressed the anti-pattern. All 4 valid tasks agree in +direction; the pack cut every anti-pattern that fired and introduced none. The +two pytest tasks are the ~1.3M-token overflow (see "Second finding" above) — +their without-pack baselines are shown for reference (the shell-hunting a fitting +pack would target). + +**Codex arm:** not run. This Bedrock account exposes only `openai.gpt-oss-*`, +not the `gpt-5.5` the Codex runner targets, so the run is Claude-only (Sonnet 5, +the default agentic tier). The Codex arm is deferred to an account with the +model. + +**Token overhead** ranged 2.3×–11×: the pack is injected fresh and uncached per +run, so a small repo (flask) with a large-relative pack shows high overhead. On +the `--cache-channel` path (Move 4), a stable pack prefix caches across runs on +opt-in providers — not exercised here. + +**Assertion pass-rate: 0 in both arms, uninformative.** The `/tmp` clones did +not have their Python deps installed, so the graded oracle could not run the +tests. Exactly the fidelity limit called out below. The token + trajectory +deltas are the trustworthy headline; the graded correctness number needs the +Docker-image path (v2). + +## What this is / is NOT + +- **A behavioral complement to 0001, on graded tasks.** SWE-bench instances ship + their own tests, so correctness here is *graded*, not the eyeball judgment + 0001 flagged. +- **NOT leaderboard resolve-rate.** v1 grades on a `/tmp` clone with deps + installed; the probe runner does not reset the checkout between the N runs, so + treat the **token + trajectory deltas as the trustworthy headline** and the + assertion pass-rate as indicative. Per-run isolation (or SWE-bench's official + per-instance Docker images) is a v2 upgrade. +- **NOT the four semantic detectors** (see above). +- **NOT a claim the pack makes agents *smarter*.** TraceProbe is explicit that + structure helps by making navigation reproducible and disciplined; this + measures exactly that — less wasted search — not higher capability. + +## Reproduce + +```bash +# 1. Get a SWE-bench Verified slice as instances.json (HF datasets), then: +node packages/eval/scripts/swebench-to-tasks.mjs \ + --instances /tmp/sb/instances.json --out-dir /tmp/sb/tasks \ + --clone-root /tmp/sb/repos --limit 8 + +# 2. Clone + install + analyze each repo (materializes the graph the pack needs): +bash packages/eval/scripts/swebench-prep.sh /tmp/sb/tasks + +# 3. Run the probe with --insight (Claude on Bedrock; Sonnet 5 lane): +CLAUDE_CODE_USE_BEDROCK=1 AWS_REGION=us-east-1 \ + codehub code-pack --variance-probe /tmp/sb/tasks/.task.json \ + --insight --runs 10 \ + --pack-tokenizer anthropic:claude-sonnet-5@2026-06-30 --json +``` + +The `--json` report carries per-arm `insight.total` / `insight.perRun` and the +per-harness `insightDelta` (positive = the pack suppressed the anti-pattern). + +## Next + +1. **Fix the pack context ceiling** (highest priority — surfaced by this run). + Budget the *whole assembled pack* to a token ceiling, not just the AST + chunks, so a large repo's pack fits the target window; and have the probe + detect an oversized pack and skip the with-pack arm with a clear message + rather than erroring all N runs. The two pytest tasks failed only for this. +2. **Widen the corpus.** Four valid tasks is a signal, not a benchmark. Add + tasks with genuinely open-ended exploration to test whether Search Loop + *ever* fires under a pack — the one detector still at zero. +3. **Install deps in the checkouts** (or use SWE-bench's official per-instance + Docker images) so the assertion pass-rate becomes informative and we can + report graded correctness alongside the trajectory deltas. +4. **Codex arm** on an account exposing a `gpt-5.x` Bedrock model, for the + agent-neutral claim. +5. Add the SWE-bench Pro slice (harder, contamination-resistant). +6. v2: the four semantic detectors behind an explicit judge opt-in. diff --git a/packages/cli/src/commands/variance-probe.ts b/packages/cli/src/commands/variance-probe.ts index ea042298..e1903273 100644 --- a/packages/cli/src/commands/variance-probe.ts +++ b/packages/cli/src/commands/variance-probe.ts @@ -73,6 +73,12 @@ export interface VarianceProbeArgs { * `auto` default. Defaults to `auto` when omitted. */ readonly cacheChannel?: CacheChannel; + /** + * Score each arm's captured trajectory against the TraceProbe structural + * anti-pattern detectors and report the per-run without−with delta (Move 1). + * Off by default; the report shape is unchanged unless set. + */ + readonly insight?: boolean; /** * Test seam — inject a fake pack-context assembler so unit tests don't need a * real analyzed repo + pack on disk. Receives the resolved tokenizer lane so @@ -203,6 +209,7 @@ export async function runVarianceProbe(args: VarianceProbeArgs): Promise) => { // Channel-aware cache-prefix enforcement (Move 4). Validated once here so // an unknown channel errors clearly before either path runs. Commander @@ -446,6 +453,7 @@ program ...(typeof opts["packTokenizer"] === "string" ? { packTokenizer: opts["packTokenizer"] } : {}), + ...(opts["insight"] === true ? { insight: true } : {}), cacheChannel, }); probeMod.printVarianceReport(report, opts["json"] === true); diff --git a/packages/eval/README.md b/packages/eval/README.md index 4c23e331..a3b20818 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -38,6 +38,36 @@ Claude Code via `CLAUDE_CODE_USE_BEDROCK=1` + a `us.`-prefixed `amazon-bedrock` provider. AWS credentials and region are inherited from the operator's environment. +## Trajectory scoring — INSIGHT anti-patterns (Move 1) + +Passing `--insight` also captures each run's **tool-call trajectory** (Claude +Code `--output-format stream-json`; Codex `--json`), normalizes it to +TraceProbe's ([arXiv:2607.06184](https://arxiv.org/abs/2607.06184)) nine-type +action taxonomy, and scores it against four **structural** anti-pattern +detectors — no LLM labeler: + +| Detector | Fires when | +|---|---| +| Search Loop | ≥10 consecutive search/read actions, no write or validation between | +| Re-read Churn | same file read ≥3× in a 10-action window, no intervening write | +| Redundant Search | same normalized query recurs ≥2× in a 10-action window | +| Shell-over-Tool | `cat`/`grep`/`rg`/`find` shelled while a structured tool covers it | + +The report adds a per-harness `insightDelta` — the per-run `without − with` of +each detector (positive = the pack suppressed the anti-pattern). The four +*semantic* TraceProbe detectors (Phase Oscillation, …) need an LLM labeler and +are deferred to v2 (the judge-oracle caveat). + +### Real tasks — SWE-bench Verified / Pro + +`scripts/swebench-to-tasks.mjs` turns a SWE-bench instances JSON into probe task +files: the `problem_statement` becomes the instruction, and an `assertion` +oracle applies the `test_patch` and runs the FAIL_TO_PASS + PASS_TO_PASS tests +(so correctness is *graded*, unlike Finding 0001). `scripts/swebench-prep.sh` +clones + installs + analyzes each repo. See `docs/findings/0002` for the full +protocol and fidelity limits (v1 grades on a `/tmp` clone without per-run +checkout isolation — token + trajectory deltas are the trustworthy headline). + ## Determinism of the probe's own output The agent runs are nondeterministic by nature — that nondeterminism is exactly diff --git a/packages/eval/scripts/swebench-prep.sh b/packages/eval/scripts/swebench-prep.sh new file mode 100755 index 00000000..25541402 --- /dev/null +++ b/packages/eval/scripts/swebench-prep.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Clone + install + analyze each SWE-bench repo from a tasks dir's clones.json +# (Move 1, Phase 0). Idempotent: skips a clone/analyze that already exists. +# +# Usage: bash packages/eval/scripts/swebench-prep.sh +# +# Reads /clones.json (produced by swebench-to-tasks.mjs), clones each +# repo at its base_commit into the clone dest, best-effort installs Python deps, +# then runs `codehub analyze` so the with-pack arm has a graph to pack from. +# +# ⚠️ Fidelity: this materializes ONE checkout per instance. The v1 probe runner +# does not reset that checkout between the N runs, so the graded assertion +# pass-rate is indicative; the token + trajectory deltas are the trustworthy +# headline. Per-run isolation (or SWE-bench's official Docker images) is a v2 +# upgrade — see docs/findings/0002. +set -euo pipefail + +TASKS_DIR="${1:?usage: swebench-prep.sh }" +CLONES="${TASKS_DIR}/clones.json" +[ -f "$CLONES" ] || { echo "no clones.json in $TASKS_DIR — run swebench-to-tasks.mjs first" >&2; exit 1; } + +# Resolve the codehub CLI: prefer a repo-local build, fall back to a global install. +CODEHUB="${CODEHUB:-codehub}" + +# Emit the clone manifest as plain tab-separated rows in ONE node call, using +# process.stdout.write (console.log can be ANSI-colorized by a shim, which would +# corrupt the parsed fields). Iterate with `while read` — no per-field node +# spawns, no arithmetic on a possibly-colorized count. +node -e ' +const rows = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); +for (const r of rows) process.stdout.write([r.instanceId, r.cloneUrl, r.baseCommit, r.dest].join("\t") + "\n"); +' "$CLONES" > "$TASKS_DIR/.clones.tsv" + +echo "Preparing $(wc -l < "$TASKS_DIR/.clones.tsv" | tr -d ' ') SWE-bench repo(s) from ${CLONES}" >&2 + +while IFS=$'\t' read -r id url commit dest; do + [ -n "$id" ] || continue + echo "── ${id}" >&2 + if [ ! -d "$dest/.git" ]; then + git clone --quiet "$url" "$dest" + fi + git -C "$dest" fetch --quiet origin "$commit" 2>/dev/null || true + git -C "$dest" checkout --quiet --force "$commit" + git -C "$dest" reset --hard --quiet "$commit" + + # Best-effort Python dep install (SWE-bench is ~94% Python). Non-fatal: a repo + # that needs a bespoke env still yields token/trajectory deltas; only its + # graded assertion needs the deps. Prefer uv, fall back to pip. + if [ -f "$dest/pyproject.toml" ] || [ -f "$dest/setup.py" ]; then + # Install the package + pytest into a per-repo uv venv so the assertion can + # import and test it. Best-effort — a repo needing a bespoke env still + # yields token/trajectory deltas; only its graded assertion needs deps. + ( cd "$dest" \ + && uv venv --quiet .venv 2>/dev/null \ + && uv pip install --python .venv --quiet -e . pytest 2>/dev/null ) &2 + fi + + # Analyze so the with-pack arm has a graph. --no-scan keeps it fast (findings + # aren't needed for the pack context); drop it if a task exercises SARIF. + # stdin from /dev/null so analyze never consumes the loop's manifest feed. + if [ ! -f "$dest/.codehub/store.sqlite" ]; then + "$CODEHUB" analyze "$dest" --no-scan >&2 &2 +echo " CLAUDE_CODE_USE_BEDROCK=1 AWS_REGION=us-east-1 \\" >&2 +echo " codehub code-pack --variance-probe ${TASKS_DIR}/.task.json --insight --runs 10 --json" >&2 diff --git a/packages/eval/scripts/swebench-to-tasks.mjs b/packages/eval/scripts/swebench-to-tasks.mjs new file mode 100755 index 00000000..fe4a9845 --- /dev/null +++ b/packages/eval/scripts/swebench-to-tasks.mjs @@ -0,0 +1,90 @@ +#!/usr/bin/env node +/** + * Generate variance-probe task files from a SWE-bench instances JSON (Move 1, + * Phase 0). Pure orchestration around the tested `instanceToTask` transform in + * `@opencodehub/eval` — this file only does I/O (read instances, write task + * files + test patches + a clone manifest); all task-shaping logic is the + * unit-tested `src/swebench.ts`. + * + * INPUT — a JSON array of SWE-bench instances. Get one with: + * # SWE-bench Verified (500 human-validated instances), via the HF datasets viewer or: + * uvx --from datasets python -c "import datasets,json; \ + * ds=datasets.load_dataset('princeton-nlp/SWE-bench_Verified', split='test'); \ + * json.dump([dict(r) for r in ds.select(range(8))], open('/tmp/sb/instances.json','w'))" + * + * USAGE: + * node packages/eval/scripts/swebench-to-tasks.mjs \ + * --instances /tmp/sb/instances.json \ + * --out-dir /tmp/sb/tasks \ + * --clone-root /tmp/sb/repos \ + * [--runner pytest|node] [--limit N] + * + * OUTPUT under --out-dir: + * .task.json — the OCH task (feed to `code-pack --variance-probe`) + * .patch — the test_patch the assertion applies + * clones.json — the clone manifest the prep script consumes + * + * Then run scripts/swebench-prep.sh to clone + install + analyze, and finally + * `codehub code-pack --variance-probe .task.json --insight --json`. + */ + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { instanceToTask } from "../dist/swebench.js"; + +function parseArgs(argv) { + const args = {}; + for (let i = 2; i < argv.length; i += 1) { + const a = argv[i]; + if (!a.startsWith("--")) continue; + const key = a.slice(2); + const next = argv[i + 1]; + if (next === undefined || next.startsWith("--")) { + args[key] = true; + } else { + args[key] = next; + i += 1; + } + } + return args; +} + +async function main() { + const args = parseArgs(process.argv); + const instancesPath = args["instances"]; + const outDir = args["out-dir"]; + const cloneRoot = args["clone-root"]; + const runner = args["runner"] === "node" ? "node" : "pytest"; + const limit = args["limit"] !== undefined ? Number.parseInt(String(args["limit"]), 10) : undefined; + + if (typeof instancesPath !== "string" || typeof outDir !== "string" || typeof cloneRoot !== "string") { + console.error( + "usage: swebench-to-tasks.mjs --instances --out-dir --clone-root [--runner pytest|node] [--limit N]", + ); + process.exit(2); + } + + const raw = await readFile(instancesPath, "utf8"); + const parsed = JSON.parse(raw); + const instances = Array.isArray(parsed) ? parsed : [parsed]; + const selected = limit !== undefined ? instances.slice(0, limit) : instances; + + await mkdir(outDir, { recursive: true }); + const clones = []; + for (const instance of selected) { + const testPatchPath = join(outDir, `${instance.instance_id}.patch`); + const gen = instanceToTask(instance, { cloneRoot, testPatchPath, runner }); + await writeFile(join(outDir, `${instance.instance_id}.task.json`), JSON.stringify(gen.task, null, 2)); + await writeFile(testPatchPath, gen.testPatch); + clones.push(gen.clone); + } + await writeFile(join(outDir, "clones.json"), JSON.stringify(clones, null, 2)); + + console.error(`Wrote ${selected.length} task(s) + patches + clones.json to ${outDir}`); + console.error(`Next: bash packages/eval/scripts/swebench-prep.sh ${outDir}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/eval/src/cli-runner.test.ts b/packages/eval/src/cli-runner.test.ts index c32ca375..997963c2 100644 --- a/packages/eval/src/cli-runner.test.ts +++ b/packages/eval/src/cli-runner.test.ts @@ -62,22 +62,26 @@ describe("buildAgentEnv — Bedrock wiring (§4a)", () => { }); describe("buildArgv", () => { - it("builds the headless claude command with JSON output + model", () => { - const { command, argv } = buildArgv({ harness: "claude" }, "PROMPT"); + it("builds the headless claude command with stream-json output + verbose + model (prompt on stdin, NOT argv)", () => { + const { command, argv } = buildArgv({ harness: "claude" }); assert.equal(command, "claude"); + // stream-json (not json) captures the tool-call trajectory; --verbose is + // mandatory with stream-json in -p mode. No positional prompt — Claude reads + // it from stdin (an ~1 MB pack overflows a single argv arg's 128 KB cap). assert.deepEqual(argv, [ "-p", - "PROMPT", "--output-format", - "json", + "stream-json", + "--verbose", "--model", DEFAULT_CLAUDE_MODEL, ]); }); - it("builds the codex exec command routed to the amazon-bedrock provider", () => { - const { command, argv } = buildArgv({ harness: "codex" }, "PROMPT"); + it("builds the codex exec command routed to amazon-bedrock, prompt via stdin (-)", () => { + const { command, argv } = buildArgv({ harness: "codex" }); assert.equal(command, "codex"); + // `-` as the prompt arg → Codex reads instructions from stdin. assert.deepEqual(argv, [ "exec", "--json", @@ -86,16 +90,16 @@ describe("buildArgv", () => { "-m", DEFAULT_CODEX_MODEL, "--skip-git-repo-check", - "PROMPT", + "-", ]); }); it("honors a per-harness model override (Bug-2 fix: codex gets a codex model)", () => { // The bug was that one global --model handed Claude's id to Codex. Each // harness must carry its own model into the argv. - const claude = buildArgv({ harness: "claude", model: "us.anthropic.claude-opus-4-8" }, "P"); + const claude = buildArgv({ harness: "claude", model: "us.anthropic.claude-opus-4-8" }); assert.equal(claude.argv[claude.argv.indexOf("--model") + 1], "us.anthropic.claude-opus-4-8"); - const codex = buildArgv({ harness: "codex", model: "openai.gpt-5.4" }, "P"); + const codex = buildArgv({ harness: "codex", model: "openai.gpt-5.4" }); assert.equal(codex.argv[codex.argv.indexOf("-m") + 1], "openai.gpt-5.4"); }); }); @@ -122,9 +126,26 @@ describe("composePrompt", () => { }); }); -describe("parseClaudeOutput", () => { - it("extracts result text + usage + cache + cost from the JSON result object", () => { - const stdout = JSON.stringify({ +describe("parseClaudeOutput (stream-json)", () => { + // A representative stream: two assistant events (a tool_use + text) and the + // final result event carrying usage + cost. + const stream = [ + JSON.stringify({ type: "system", subtype: "init", session_id: "s" }), + JSON.stringify({ + type: "assistant", + message: { + content: [ + { type: "tool_use", id: "toolu_1", name: "Grep", input: { pattern: "foo" } }, + { type: "tool_use", id: "toolu_2", name: "Read", input: { file_path: "/a.ts" } }, + ], + }, + }), + " ", // blank/whitespace line — tolerated + JSON.stringify({ + type: "assistant", + message: { content: [{ type: "text", text: "Done." }] }, + }), + JSON.stringify({ type: "result", subtype: "success", result: "Done — added the flag.", @@ -135,8 +156,11 @@ describe("parseClaudeOutput", () => { cache_read_input_tokens: 100, }, total_cost_usd: 0.0123, - }); - const { finalText, tokens } = parseClaudeOutput(stdout); + }), + ].join("\n"); + + it("extracts result text + usage + cache + cost from the final result event", () => { + const { finalText, tokens } = parseClaudeOutput(stream); assert.equal(finalText, "Done — added the flag."); assert.equal(tokens.inputTokens, 1234); assert.equal(tokens.outputTokens, 56); @@ -144,15 +168,34 @@ describe("parseClaudeOutput", () => { assert.equal(tokens.cacheTokens, 27506); assert.equal(tokens.costUsd, 0.0123); }); + + it("captures the normalized trajectory from assistant tool_use blocks", () => { + const { trajectory } = parseClaudeOutput(stream); + // Grep→search, Read→file_read, text→reason. + assert.deepEqual(trajectory, [ + { type: "search", query: "foo" }, + { type: "file_read", target: "/a.ts" }, + { type: "reason" }, + ]); + }); + it("tolerates a missing usage block (zeros, null cost)", () => { - const { tokens, finalText } = parseClaudeOutput(JSON.stringify({ result: "x" })); + const stdout = JSON.stringify({ type: "result", result: "x" }); + const { tokens, finalText } = parseClaudeOutput(stdout); assert.equal(finalText, "x"); assert.equal(tokens.inputTokens, 0); assert.equal(tokens.cacheTokens, 0); assert.equal(tokens.costUsd, null); }); - it("throws on unparseable stdout", () => { + + it("throws when the stream carries no result event", () => { + // Non-JSON, and a stream of only assistant events, both lack a result line. assert.throws(() => parseClaudeOutput("not json")); + assert.throws(() => + parseClaudeOutput( + JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: "hi" }] } }), + ), + ); }); }); @@ -197,15 +240,24 @@ describe("parseCodexOutput", () => { }); describe("CliAgentRunner.run (stubbed spawn)", () => { - it("returns a successful outcome from a claude run", async () => { + it("returns a successful outcome + trajectory from a claude run", async () => { const spawnFn: SpawnFn = async ({ command, argv }) => { assert.equal(command, "claude"); assert.ok(argv.includes("--output-format")); - return { - stdout: JSON.stringify({ result: "ok", usage: { input_tokens: 10, output_tokens: 2 } }), - stderr: "", - code: 0, - }; + const stdout = [ + JSON.stringify({ + type: "assistant", + message: { + content: [{ type: "tool_use", id: "t1", name: "Read", input: { file_path: "/x.ts" } }], + }, + }), + JSON.stringify({ + type: "result", + result: "ok", + usage: { input_tokens: 10, output_tokens: 2 }, + }), + ].join("\n"); + return { stdout, stderr: "", code: 0 }; }; const runner = new CliAgentRunner({ harness: "claude", _spawn: spawnFn, _baseEnv: {} }); const outcome = await runner.run({ task: TASK, harness: "claude", withPack: false }); @@ -213,6 +265,7 @@ describe("CliAgentRunner.run (stubbed spawn)", () => { assert.equal(outcome.finalText, "ok"); assert.equal(outcome.tokens.inputTokens, 10); assert.equal(outcome.checkoutPath, TASK.repo); + assert.deepEqual(outcome.trajectory, [{ type: "file_read", target: "/x.ts" }]); assert.equal(runner.name, "cli:claude"); }); @@ -231,19 +284,28 @@ describe("CliAgentRunner.run (stubbed spawn)", () => { assert.equal(outcome.errored, true); }); - it("injects the pack context into the prompt on the with-pack arm", async () => { - let seenPrompt = ""; - const spawnFn: SpawnFn = async ({ argv }) => { - // claude argv: ["-p", PROMPT, ...] - seenPrompt = argv[1] ?? ""; + it("injects the pack context into the prompt via STDIN on the with-pack arm", async () => { + let seenStdin = ""; + let seenArgv: readonly string[] = []; + const spawnFn: SpawnFn = async ({ argv, stdin }) => { + // The prompt is fed on stdin (not argv) so a ~1 MB pack can't overflow a + // single argv arg's 128 KB cap. argv must NOT carry the pack. + seenStdin = stdin; + seenArgv = argv; return { - stdout: JSON.stringify({ result: "ok", usage: { input_tokens: 1, output_tokens: 1 } }), + stdout: JSON.stringify({ + type: "result", + result: "ok", + usage: { input_tokens: 1, output_tokens: 1 }, + }), stderr: "", code: 0, }; }; const runner = new CliAgentRunner({ harness: "claude", _spawn: spawnFn, _baseEnv: {} }); await runner.run({ task: TASK, harness: "claude", withPack: true, packContext: "THE PACK" }); - assert.ok(seenPrompt.startsWith("THE PACK")); + assert.ok(seenStdin.startsWith("THE PACK"), "pack context is on stdin"); + assert.ok(seenStdin.includes(TASK.instruction), "instruction follows the pack on stdin"); + assert.ok(!seenArgv.includes("THE PACK"), "pack context is NOT on argv"); }); }); diff --git a/packages/eval/src/cli-runner.ts b/packages/eval/src/cli-runner.ts index 6765f294..c7a10911 100644 --- a/packages/eval/src/cli-runner.ts +++ b/packages/eval/src/cli-runner.ts @@ -26,6 +26,7 @@ import { spawn } from "node:child_process"; import type { AgentRunner, Harness, RunOutcome, RunRequest, RunTokens } from "./runner.js"; +import { type Action, actionsFromClaudeStreamJson, actionsFromCodexJsonl } from "./trajectory.js"; /** Default Claude Code Bedrock inference profile (us.-prefixed, §4a). */ export const DEFAULT_CLAUDE_MODEL = "us.anthropic.claude-sonnet-4-6"; @@ -107,27 +108,39 @@ export function composePrompt(request: RunRequest): string { return request.task.instruction; } -/** Build the argv for a harness invocation. Pure + exported for tests. */ -export function buildArgv( - config: CliRunnerConfig, - prompt: string, -): { command: string; argv: string[] } { +/** + * Build the argv for a harness invocation. Pure + exported for tests. + * + * The prompt is NOT passed on argv — an OCH pack context is ~1 MB, and a single + * argv argument is capped at `MAX_ARG_STRLEN` (128 KB on Linux, independent of + * the larger total `ARG_MAX`), so a with-pack prompt on argv fails the spawn + * with `E2BIG` / "argument list too long". Instead the prompt is fed on **stdin** + * (see {@link CliAgentRunner.run}): `claude -p` with no positional reads the + * prompt from stdin; `codex exec -` reads it from stdin. This keeps argv tiny + * regardless of pack size. + */ +export function buildArgv(config: CliRunnerConfig): { command: string; argv: string[] } { if (config.harness === "claude") { + // No positional prompt → Claude Code reads it from stdin. stream-json (not + // json) captures the tool-call trajectory; `--verbose` is mandatory with + // stream-json in -p mode. The final `type:"result"` line still carries + // usage + total_cost_usd, so token accounting is unchanged. return { command: "claude", argv: [ "-p", - prompt, "--output-format", - "json", + "stream-json", + "--verbose", "--model", config.model ?? DEFAULT_CLAUDE_MODEL, ], }; } - // Codex: route to the first-party Bedrock provider via -c, pin the model, - // emit JSONL with --json, and skip the git-repo guard so the probe can run - // the agent against an arbitrary checkout. + // Codex: `-` as the prompt arg reads instructions from stdin. Route to the + // first-party Bedrock provider via -c, pin the model, emit JSONL with --json, + // and skip the git-repo guard so the probe can run against an arbitrary + // checkout. return { command: "codex", argv: [ @@ -138,37 +151,76 @@ export function buildArgv( "-m", config.model ?? DEFAULT_CODEX_MODEL, "--skip-git-repo-check", - prompt, + "-", ], }; } +/** The final `type:"result"` event Claude Code emits in stream-json mode. */ +interface ClaudeResultEvent { + readonly result?: unknown; + readonly usage?: { + readonly input_tokens?: unknown; + readonly output_tokens?: unknown; + readonly cache_creation_input_tokens?: unknown; + readonly cache_read_input_tokens?: unknown; + }; + readonly total_cost_usd?: unknown; +} + /** - * Parse Claude Code's `--output-format json` single result object into a - * {@link RunOutcome} (minus checkout/errored, filled by the runner). Pure + - * exported. Throws on unparseable output so a malformed run is a hard error, - * not a silent zero-token success. + * Parse Claude Code's `--output-format stream-json --verbose` JSONL stream into + * the final answer, token accounting, and the normalized action trajectory. + * Pure + exported. + * + * The stream is one JSON object per line; the final `type:"result"` line + * carries `result` (final text), `usage.{input,output,cache_*}_tokens`, and + * `total_cost_usd` — the same fields the old single-object `json` format + * exposed, so token accounting is unchanged. The trajectory comes from the + * `assistant` events' `tool_use` blocks (see {@link actionsFromClaudeStreamJson}). + * + * Throws when the stream carries no `result` line, so a malformed / truncated + * run is a hard error rather than a silent zero-token success — matching the + * old parser's fail-fast contract. */ -export function parseClaudeOutput(stdout: string): { finalText: string; tokens: RunTokens } { - const doc = JSON.parse(stdout) as { - result?: unknown; - usage?: { - input_tokens?: unknown; - output_tokens?: unknown; - cache_creation_input_tokens?: unknown; - cache_read_input_tokens?: unknown; - }; - total_cost_usd?: unknown; - }; - const finalText = typeof doc.result === "string" ? doc.result : ""; - const inputTokens = num(doc.usage?.input_tokens); - const outputTokens = num(doc.usage?.output_tokens); +export function parseClaudeOutput(stdout: string): { + finalText: string; + tokens: RunTokens; + trajectory: Action[]; +} { + let result: ClaudeResultEvent | null = null; + + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + let evt: unknown; + try { + evt = JSON.parse(trimmed); + } catch { + continue; // partial / non-JSON line — skip + } + if (typeof evt === "object" && evt !== null && (evt as { type?: unknown }).type === "result") { + result = evt as ClaudeResultEvent; + } + } + + if (result === null) { + throw new Error("claude stream-json produced no result event"); + } + + const finalText = typeof result.result === "string" ? result.result : ""; + const inputTokens = num(result.usage?.input_tokens); + const outputTokens = num(result.usage?.output_tokens); // Claude Code injects a large cached system prompt per call; both the // creation and read halves are real token cost the overhead headline needs. const cacheTokens = - num(doc.usage?.cache_creation_input_tokens) + num(doc.usage?.cache_read_input_tokens); - const costUsd = typeof doc.total_cost_usd === "number" ? doc.total_cost_usd : null; - return { finalText, tokens: { inputTokens, outputTokens, cacheTokens, costUsd } }; + num(result.usage?.cache_creation_input_tokens) + num(result.usage?.cache_read_input_tokens); + const costUsd = typeof result.total_cost_usd === "number" ? result.total_cost_usd : null; + return { + finalText, + tokens: { inputTokens, outputTokens, cacheTokens, costUsd }, + trajectory: actionsFromClaudeStreamJson(stdout), + }; } /** @@ -178,7 +230,11 @@ export function parseClaudeOutput(stdout: string): { finalText: string; tokens: * exported. Tolerates interleaved non-JSON lines (progress noise) by skipping * them. */ -export function parseCodexOutput(stdout: string): { finalText: string; tokens: RunTokens } { +export function parseCodexOutput(stdout: string): { + finalText: string; + tokens: RunTokens; + trajectory: Action[]; +} { let finalText = ""; let inputTokens = 0; let outputTokens = 0; @@ -208,7 +264,11 @@ export function parseCodexOutput(stdout: string): { finalText: string; tokens: R } // Codex does not surface a per-invocation USD cost on the public event // schema, so cost is null (the report tolerates a null-cost arm). - return { finalText, tokens: { inputTokens, outputTokens, cacheTokens, costUsd: null } }; + return { + finalText, + tokens: { inputTokens, outputTokens, cacheTokens, costUsd: null }, + trajectory: actionsFromCodexJsonl(stdout), + }; } function num(v: unknown): number { @@ -257,15 +317,18 @@ export class CliAgentRunner implements AgentRunner { async run(request: RunRequest): Promise { const prompt = composePrompt(request); - const { command, argv } = buildArgv(this.config, prompt); + const { command, argv } = buildArgv(this.config); const env = buildAgentEnv(this.config); // The checkout the agent works in. v1 runs the agent against the task's // repo path directly; a future iteration can clone per-run for isolation. const cwd = request.task.repo; + // The prompt is fed on stdin, not argv: an OCH pack is ~1 MB and a single + // argv arg caps at MAX_ARG_STRLEN (128 KB), so a with-pack prompt on argv + // would fail the spawn with "argument list too long". stdin has no such cap. let result: SpawnResult; try { - result = await this.spawnFn({ command, argv, env, cwd, stdin: "" }); + result = await this.spawnFn({ command, argv, env, cwd, stdin: prompt }); } catch (err) { return erroredOutcome(cwd, `spawn failed: ${String(err)}`); } @@ -283,6 +346,7 @@ export class CliAgentRunner implements AgentRunner { finalText: parsed.finalText, diff: "", // neither CLI surfaces a structured diff in headless JSON; left empty tokens: parsed.tokens, + trajectory: parsed.trajectory, checkoutPath: cwd, errored: false, }; diff --git a/packages/eval/src/index.ts b/packages/eval/src/index.ts index 65efabd0..596ced94 100644 --- a/packages/eval/src/index.ts +++ b/packages/eval/src/index.ts @@ -27,6 +27,13 @@ export { mean, populationStddev, } from "./dispersion.js"; +export { + aggregateInsight, + breaksSearchLoop, + type InsightCounts, + scoreInsight, + ZERO_INSIGHT, +} from "./insight.js"; export { type JudgeScorer, type ScoreOptions, scoreArm } from "./oracle.js"; export { DEFAULT_RUNS, @@ -37,6 +44,7 @@ export { runProbe, } from "./probe.js"; export { + type ArmInsight, type ArmReport, type ArmTokens, buildHarnessReport, @@ -53,6 +61,15 @@ export type { RunRequest, RunTokens, } from "./runner.js"; +export { + buildAssertionCommand, + type GeneratedTask, + instanceToTask, + parseTestList, + type SweBenchInstance, + type TestRunner, + type ToTaskOptions, +} from "./swebench.js"; export { type AssertionOracle, type JudgeOracle, @@ -64,3 +81,13 @@ export { TaskSchema, TaskValidationError, } from "./task.js"; +export { + type Action, + type ActionType, + actionsFromClaudeStreamJson, + actionsFromCodexJsonl, + isShellReadSearch, + isValidationCommand, + normalizeQuery, + shellFirstWord, +} from "./trajectory.js"; diff --git a/packages/eval/src/insight.test.ts b/packages/eval/src/insight.test.ts new file mode 100644 index 00000000..b6ec06b5 --- /dev/null +++ b/packages/eval/src/insight.test.ts @@ -0,0 +1,165 @@ +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; +import { aggregateInsight, breaksSearchLoop, scoreInsight, ZERO_INSIGHT } from "./insight.js"; +import { type Action, normalizeQuery } from "./trajectory.js"; + +/* Small action builders keep the detector tables readable. `search` normalizes + * its query exactly as the capture normalizers do — the detector's contract is + * that `Action.query` is already normalized, so the builder must honor it. */ +const search = (query = "q"): Action => ({ type: "search", query: normalizeQuery(query) }); +const read = (target: string): Action => ({ type: "file_read", target }); +const write = (target: string): Action => ({ type: "file_write", target }); +const reason = (): Action => ({ type: "reason" }); +const cmd = (command: string): Action => ({ type: "command", command }); +/** N search actions in a row (distinct-enough queries to avoid redundant-search). */ +const searches = (n: number): Action[] => Array.from({ length: n }, (_, i) => search(`q${i}`)); +const reads = (target: string, n: number): Action[] => + Array.from({ length: n }, () => read(target)); + +describe("Search Loop", () => { + it("fires on ≥10 consecutive search/read with no write or validation", () => { + assert.equal(scoreInsight(searches(10)).searchLoop, 1); + assert.equal(scoreInsight(searches(25)).searchLoop, 1, "one maximal loop, not many"); + }); + + it("does NOT fire below the threshold of 10", () => { + assert.equal(scoreInsight(searches(9)).searchLoop, 0); + }); + + it("is broken by a file_write and by a validation command, splitting the run", () => { + // 6 reads, write, 6 reads → neither half reaches 10 → 0. + assert.equal(scoreInsight([...reads("/a", 6), write("/a"), ...reads("/b", 6)]).searchLoop, 0); + // 10 reads, validation cmd, 10 reads → two loops. + assert.equal( + scoreInsight([...reads("/a", 10), cmd("pnpm test"), ...reads("/b", 10)]).searchLoop, + 2, + ); + }); + + it("treats reason / non-validation commands as transparent (does not reset the run)", () => { + // reason blocks interleave nearly every tool call — they must be transparent + // or the detector would essentially never fire. + const withReasons: Action[] = []; + for (let i = 0; i < 10; i += 1) { + withReasons.push(read(`/f${i}`), reason()); + } + assert.equal(scoreInsight(withReasons).searchLoop, 1); + // a non-validation command (e.g. `ls`) is also transparent. + assert.equal(scoreInsight([...reads("/a", 5), cmd("ls -la"), ...reads("/b", 5)]).searchLoop, 1); + }); +}); + +describe("Re-read Churn", () => { + it("fires when the same file is read ≥3 times within a 10-action window", () => { + assert.equal(scoreInsight([read("/a"), read("/a"), read("/a")]).rereadChurn, 1); + }); + + it("does NOT fire at 2 reads", () => { + assert.equal(scoreInsight([read("/a"), read("/a")]).rereadChurn, 0); + }); + + it("is reset by an intervening write to that same path", () => { + assert.equal( + scoreInsight([read("/a"), read("/a"), write("/a"), read("/a")]).rereadChurn, + 0, + "the write resets the read tally for /a", + ); + }); + + it("counts distinct churning files separately, once each per window", () => { + const counts = scoreInsight([ + read("/a"), + read("/b"), + read("/a"), + read("/b"), + read("/a"), + read("/b"), + ]); + assert.equal(counts.rereadChurn, 2, "/a and /b each churn once in the window"); + }); + + it("scores a churn that completes inside a short (< window) trajectory", () => { + assert.equal(scoreInsight([read("/a"), read("/a"), read("/a")]).rereadChurn, 1); + }); +}); + +describe("Redundant Search", () => { + it("fires on a repeated normalized query within a 10-action window", () => { + assert.equal(scoreInsight([search("foo"), search("foo")]).redundantSearch, 1); + }); + + it("counts each repeat: three identical searches score 2", () => { + assert.equal(scoreInsight([search("foo"), search("foo"), search("foo")]).redundantSearch, 2); + }); + + it("treats whitespace-different but normalized-equal queries as the same", () => { + assert.equal(scoreInsight([search("foo bar"), search(" foo bar ")]).redundantSearch, 1); + }); + + it("does NOT fire for distinct queries", () => { + assert.equal(scoreInsight([search("foo"), search("bar")]).redundantSearch, 0); + }); + + it("does NOT fire when the repeat is outside the 10-action window", () => { + // search("foo"), then 10 filler actions, then search("foo") → 11 apart > window. + const actions = [search("foo"), ...reads("/x", 10), search("foo")]; + assert.equal(scoreInsight(actions).redundantSearch, 0); + }); +}); + +describe("Shell-over-Tool", () => { + it("fires per shell read/search command", () => { + const counts = scoreInsight([ + cmd("grep -rn foo ."), + cmd("cat f"), + cmd("/bin/zsh -lc 'find . -name x'"), + ]); + assert.equal(counts.shellOverTool, 3); + }); + + it("does NOT fire for builds/tests or non-read commands", () => { + assert.equal( + scoreInsight([cmd("pnpm test"), cmd("git status"), cmd("sed -i s/a/b/ f")]).shellOverTool, + 0, + ); + }); +}); + +describe("breaksSearchLoop", () => { + it("true for writes and validation commands, false otherwise", () => { + assert.equal(breaksSearchLoop(write("/a")), true); + assert.equal(breaksSearchLoop(cmd("pytest")), true); + assert.equal(breaksSearchLoop(cmd("ls")), false); + assert.equal(breaksSearchLoop(read("/a")), false); + assert.equal(breaksSearchLoop(reason()), false); + }); +}); + +describe("aggregateInsight", () => { + it("sums firings and divides per scored (trajectory-bearing) run", () => { + const loopTraj = searches(10); // searchLoop 1 + const cleanTraj = [read("/a")]; // all zero + const agg = aggregateInsight([loopTraj, cleanTraj]); + assert.ok(agg !== undefined); + assert.equal(agg.scored, 2); + assert.equal(agg.total.searchLoop, 1); + assert.equal(agg.perRun.searchLoop, 0.5); + }); + + it("excludes trajectory-less runs from scored (crash ≠ clean)", () => { + const agg = aggregateInsight([searches(10), undefined, undefined]); + assert.ok(agg !== undefined); + assert.equal(agg.scored, 1, "only the one trajectory-bearing run is scored"); + assert.equal(agg.perRun.searchLoop, 1); + }); + + it("returns undefined when no run carried a trajectory", () => { + assert.equal(aggregateInsight([undefined, undefined]), undefined); + }); +}); + +describe("empty / degenerate trajectories", () => { + it("an empty trajectory scores all zeros", () => { + assert.deepEqual(scoreInsight([]), ZERO_INSIGHT); + }); +}); diff --git a/packages/eval/src/insight.ts b/packages/eval/src/insight.ts new file mode 100644 index 00000000..134d74ae --- /dev/null +++ b/packages/eval/src/insight.ts @@ -0,0 +1,247 @@ +/** + * INSIGHT — structural single-trajectory anti-pattern detectors + * (Move 1 / arXiv:2607.06184 "TraceProbe", the INSIGHT module). + * + * Each detector is a deterministic, no-oracle predicate over a normalized + * {@link Action} list. TraceProbe defines eight structural detectors; v1 ships + * the four whose frozen predicates read only structural fields OCH already + * captures — action type, file target, normalized query, command first word — + * so no LLM labeler is required. This is the same discipline that made Finding + * 0001's headline the directly-measured token number rather than the + * judge-dependent one: publish what a rule can prove. + * + * The four (predicates quoted from the paper's frozen-detector table): + * - Search Loop — "≥10 consecutive SEARCH or FILE READ actions with no + * FILE WRITE and no validation COMMAND between them." + * - Re-read Churn — "same canonical file path is read ≥3 times within a + * 10-action window, with no intervening write to that file." + * - Redundant Search — "same exact-normalized SEARCH query recurs ≥2 times + * within a 10-action window." + * - Shell-over-Tool — "a shell command's first word is cat/head/tail/less/ + * more, grep-family, rg/ag, or find while structured + * read/search tools are exposed." + * + * Deliberately deferred to v2 (need a semantic labeler / effect labels — the + * judge-oracle caveat): Tool Oscillation, No Formal Tail Validation, Unsupported + * Completion Claim, Structured Plan Absence, and all four semantic detectors + * (Phase Oscillation, …). Counting them would import LLM-labeler noise into a + * number we want to be a pure function of the trajectory. + * + * Each detector returns a **count of firings** (windows/occurrences), not a + * boolean, so the with/without-pack delta is graded, not just present/absent. + * The functions are pure — identical action list → identical counts — so an + * `InsightReport` inherits the probe's byte-stable-report contract. + */ + +import { type Action, isShellReadSearch, isValidationCommand } from "./trajectory.js"; + +/** Per-detector firing counts for one trajectory. */ +export interface InsightCounts { + /** Number of maximal ≥10-long search/read runs with no write/validation. */ + readonly searchLoop: number; + /** Number of (file, window) re-read-churn firings. */ + readonly rereadChurn: number; + /** Number of redundant-search firings (a repeat within a 10-action window). */ + readonly redundantSearch: number; + /** Number of shell commands doing read/search work a structured tool covers. */ + readonly shellOverTool: number; +} + +/** Detector window width (TraceProbe's "10-action window"). */ +const WINDOW = 10; +/** Search Loop minimum run length. */ +const SEARCH_LOOP_MIN = 10; +/** Re-read Churn minimum reads of the same file within a window. */ +const REREAD_MIN = 3; +// Redundant Search's threshold ("recurs ≥2 times") is one prior identical query +// within the window — expressed directly by the first-match `break` in +// countRedundantSearch rather than a named minimum. + +/** + * Score all four structural detectors over one trajectory. Pure. + */ +export function scoreInsight(actions: readonly Action[]): InsightCounts { + return { + searchLoop: countSearchLoops(actions), + rereadChurn: countRereadChurn(actions), + redundantSearch: countRedundantSearch(actions), + shellOverTool: countShellOverTool(actions), + }; +} + +/** + * Search Loop: count maximal segments holding ≥{@link SEARCH_LOOP_MIN} SEARCH + * or FILE_READ actions with no FILE_WRITE and no validation COMMAND between + * them. Only those two — a write, or a validation command — break a segment; + * every other action type (`reason`, `plan`, `spawn`, `fetch`, `navigate`, and + * a non-validation `command`) is transparent, because the paper's predicate + * names FILE WRITE and validation COMMAND as the sole exclusions. This matters: + * agents emit `reason` blocks between nearly every tool call, so a detector + * that reset on `reason` would essentially never fire. A transparent action + * neither counts toward the 10 nor resets the tally. One maximal qualifying + * segment counts once — a 25-read hunt is one loop, not sixteen. + */ +function countSearchLoops(actions: readonly Action[]): number { + let count = 0; + let run = 0; + for (const a of actions) { + if (a.type === "search" || a.type === "file_read") { + run += 1; + } else if (breaksSearchLoop(a)) { + if (run >= SEARCH_LOOP_MIN) count += 1; + run = 0; + } + // else: transparent action — leave `run` untouched. + } + if (run >= SEARCH_LOOP_MIN) count += 1; + return count; +} + +/** + * Re-read Churn: for each sliding window of {@link WINDOW} actions, count a + * firing for every distinct file path read ≥{@link REREAD_MIN} times within + * that window with no intervening write *to that same path* inside the window. + * A file counted in one window can fire again in a later window (the churn + * persists), matching the paper's window-local definition; the same file is not + * double-counted within a single window. + */ +function countRereadChurn(actions: readonly Action[]): number { + let count = 0; + for (let start = 0; start + WINDOW <= actions.length; start += 1) { + const window = actions.slice(start, start + WINDOW); + count += churnFiringsInWindow(window); + } + // Windows shorter than WINDOW at the tail cannot reach a full-window count, + // but a churn can still complete inside them; scan the final partial window + // only when the trajectory is shorter than one full window (otherwise every + // qualifying churn already appeared in a full window above). + if (actions.length < WINDOW) { + count += churnFiringsInWindow(actions); + } + return count; +} + +/** Count distinct files read ≥REREAD_MIN times with no intervening write, in one window. */ +function churnFiringsInWindow(window: readonly Action[]): number { + const readsSinceLastWrite = new Map(); + const fired = new Set(); + let firings = 0; + for (const a of window) { + if (a.target === undefined) continue; + if (a.type === "file_write") { + readsSinceLastWrite.set(a.target, 0); // a write resets the read tally + } else if (a.type === "file_read") { + const n = (readsSinceLastWrite.get(a.target) ?? 0) + 1; + readsSinceLastWrite.set(a.target, n); + if (n >= REREAD_MIN && !fired.has(a.target)) { + fired.add(a.target); + firings += 1; + } + } + } + return firings; +} + +/** + * Redundant Search: count firings where the same normalized SEARCH query recurs + * ≥2 times within a {@link WINDOW}-action window. Counted + * per repeat occurrence: the second (and each later) appearance of a query + * already seen within the trailing window fires once. This makes three + * back-to-back identical searches score 2, matching "recurs ≥2 times" read as + * repeats beyond the first. + */ +function countRedundantSearch(actions: readonly Action[]): number { + let count = 0; + for (let i = 0; i < actions.length; i += 1) { + const a = actions[i]; + if (a?.type !== "search" || a.query === undefined) continue; + // Look back within the window for an identical normalized query. + const lo = Math.max(0, i - (WINDOW - 1)); + for (let j = i - 1; j >= lo; j -= 1) { + const prev = actions[j]; + if (prev?.type === "search" && prev.query === a.query) { + count += 1; + break; // one firing per redundant occurrence + } + } + } + return count; +} + +/** + * Shell-over-Tool: count COMMAND actions whose unwrapped program is a shell + * read/search utility (cat/head/tail/less/more, grep-family, rg/ag, find) that + * a structured Read/Grep/Glob tool covers. Each such command counts once. + */ +function countShellOverTool(actions: readonly Action[]): number { + let count = 0; + for (const a of actions) { + if (a.type === "command" && a.command !== undefined && isShellReadSearch(a.command)) { + count += 1; + } + } + return count; +} + +/** + * Whether a command action breaks a search loop (a validation command). Exposed + * for the detector's unit tests and any future detector that shares the notion. + */ +export function breaksSearchLoop(action: Action): boolean { + if (action.type === "file_write") return true; + if (action.type === "command" && action.command !== undefined) { + return isValidationCommand(action.command); + } + return false; +} + +/** Zero counts — the additive identity for aggregation. */ +export const ZERO_INSIGHT: InsightCounts = { + searchLoop: 0, + rereadChurn: 0, + redundantSearch: 0, + shellOverTool: 0, +}; + +/** Element-wise sum of two count records. */ +function addCounts(a: InsightCounts, b: InsightCounts): InsightCounts { + return { + searchLoop: a.searchLoop + b.searchLoop, + rereadChurn: a.rereadChurn + b.rereadChurn, + redundantSearch: a.redundantSearch + b.redundantSearch, + shellOverTool: a.shellOverTool + b.shellOverTool, + }; +} + +/** + * Aggregate an arm's per-run trajectories into summed + per-scored-run counts. + * Only runs that carried a trajectory are scored; a run whose `trajectory` is + * `undefined` (an errored run that emitted no stream, or a legacy runner) is + * excluded from `scored` rather than counted as a clean zero — a crash that + * produced no actions is absence of evidence, not evidence of a clean run. + * + * Returns `undefined` when no run carried a trajectory, so the report omits the + * insight block entirely rather than dividing by zero. + */ +export function aggregateInsight( + trajectories: readonly (readonly Action[] | undefined)[], +): { total: InsightCounts; perRun: InsightCounts; scored: number } | undefined { + let total = ZERO_INSIGHT; + let scored = 0; + for (const traj of trajectories) { + if (traj === undefined) continue; + total = addCounts(total, scoreInsight(traj)); + scored += 1; + } + if (scored === 0) return undefined; + return { + total, + perRun: { + searchLoop: total.searchLoop / scored, + rereadChurn: total.rereadChurn / scored, + redundantSearch: total.redundantSearch / scored, + shellOverTool: total.shellOverTool / scored, + }, + scored, + }; +} diff --git a/packages/eval/src/probe.ts b/packages/eval/src/probe.ts index 907cf3bc..698c7df0 100644 --- a/packages/eval/src/probe.ts +++ b/packages/eval/src/probe.ts @@ -14,6 +14,7 @@ * responsibility (the CLI runner spawns a new process each call). */ +import { aggregateInsight } from "./insight.js"; import { type ScoreOptions, scoreArm } from "./oracle.js"; import { type ArmReport, @@ -50,6 +51,15 @@ export interface ProbeOptions { readonly packTokenizerId?: string; /** Required only when the task's oracle is `judge`. */ readonly score?: ScoreOptions; + /** + * When true, score each arm's captured trajectories against the INSIGHT + * anti-pattern detectors (Move 1 / TraceProbe) and attach the per-arm + * {@link ArmInsight} + the harness `insightDelta`. Off by default so the + * dispersion+token report shape is unchanged unless the caller opts in with + * `--insight`. Requires a runner that captures trajectories (the CLI runner + * does); with a trajectory-less runner the arm insight is simply omitted. + */ + readonly insight?: boolean; /** * Per-run progress callback. Pure-side-channel — never affects the report. */ @@ -112,7 +122,17 @@ async function runArm( outcomes.push(outcome); } const dispersion = await scoreArm(task.oracle, outcomes, options.score ?? {}); - return { dispersion, tokens: sumTokens(outcomes) }; + // When --insight is on, score the arm's captured trajectories. aggregateInsight + // returns undefined if no run carried a trajectory (trajectory-less runner or + // all-errored arm), in which case the arm's insight is omitted rather than a + // misleading zero. + const insight = + options.insight === true ? aggregateInsight(outcomes.map((o) => o.trajectory)) : undefined; + return { + dispersion, + tokens: sumTokens(outcomes), + ...(insight !== undefined ? { insight } : {}), + }; } /** diff --git a/packages/eval/src/report.ts b/packages/eval/src/report.ts index 3a840682..f9078848 100644 --- a/packages/eval/src/report.ts +++ b/packages/eval/src/report.ts @@ -16,6 +16,7 @@ import { canonicalJson } from "@opencodehub/core-types"; import type { ArmDispersion } from "./dispersion.js"; import { dispersionScalar } from "./dispersion.js"; +import type { InsightCounts } from "./insight.js"; /** * Token-overhead guardrail (spec 010 §7.4). Above this ratio the report flags @@ -37,10 +38,33 @@ export interface ArmTokens { readonly costUsd: number | null; } +/** + * An arm's aggregated INSIGHT anti-pattern signal (Move 1 / TraceProbe). `total` + * sums each detector's firings across every run in the arm that carried a + * trajectory; `perRun` divides by {@link scored} (the count of trajectory-bearing + * runs), so arms with different run counts or different numbers of + * trajectory-less errored runs stay comparable. `scored` is surfaced so a reader + * can see how much of the arm the signal rests on. + */ +export interface ArmInsight { + /** Summed detector firings across the arm's scored runs. */ + readonly total: InsightCounts; + /** Per-scored-run mean of each detector (total / scored; 0 when scored is 0). */ + readonly perRun: InsightCounts; + /** How many of the arm's runs carried a trajectory to score. */ + readonly scored: number; +} + /** One arm's measured result (with-pack or without-pack). */ export interface ArmReport { readonly dispersion: ArmDispersion; readonly tokens: ArmTokens; + /** + * INSIGHT anti-pattern signal for the arm. Optional so pre-Move-1 captured + * reports and runners that emit no trajectory stay valid; absent means the + * probe was not run with `--insight` (or no run carried a trajectory). + */ + readonly insight?: ArmInsight; } /** The full per-harness probe result. */ @@ -61,6 +85,13 @@ export interface HarnessReport { readonly tokenOverhead: number; /** True when `tokenOverhead` exceeds {@link TOKEN_OVERHEAD_FLAG}. */ readonly tokenOverheadFlagged: boolean; + /** + * Per-run `without − with` delta for each INSIGHT detector (Move 1). Positive + * = the pack reduced that anti-pattern — the headline the move publishes + * ("the pack cuts search loops"). Present only when both arms carry an + * {@link ArmInsight}; absent when the probe ran without `--insight`. + */ + readonly insightDelta?: InsightCounts; } /** The top-level report the probe emits. */ @@ -105,6 +136,13 @@ export function buildHarnessReport(input: { // Overhead is undefined when the baseline arm spent no tokens; report 0 in // that degenerate case rather than Infinity/NaN, and never flag it. const tokenOverhead = withoutTotal === 0 ? 0 : withTotal / withoutTotal; + // Insight delta is the per-run without − with of each detector — positive + // means the pack suppressed the anti-pattern. Computed only when both arms + // carry insight (the probe ran with --insight and runs produced trajectories). + const insightDelta = + input.without.insight !== undefined && input.with.insight !== undefined + ? subtractCounts(input.without.insight.perRun, input.with.insight.perRun) + : undefined; return { harness: input.harness, runner: input.runner, @@ -114,6 +152,17 @@ export function buildHarnessReport(input: { dispersionDelta, tokenOverhead, tokenOverheadFlagged: tokenOverhead > TOKEN_OVERHEAD_FLAG, + ...(insightDelta !== undefined ? { insightDelta } : {}), + }; +} + +/** Per-detector `a − b` of two per-run insight-count records. */ +function subtractCounts(a: InsightCounts, b: InsightCounts): InsightCounts { + return { + searchLoop: a.searchLoop - b.searchLoop, + rereadChurn: a.rereadChurn - b.rereadChurn, + redundantSearch: a.redundantSearch - b.redundantSearch, + shellOverTool: a.shellOverTool - b.shellOverTool, }; } @@ -148,10 +197,27 @@ export function formatReport(report: VarianceReport): string { // tokens (the per-call system prompt) are part of the total, not hidden. lines.push(` tokens without: ${fmtTokens(h.without.tokens)}`); lines.push(` tokens with: ${fmtTokens(h.with.tokens)}`); + if (h.insightDelta !== undefined) { + // Per-run without − with; positive = the pack suppressed the anti-pattern. + const d = h.insightDelta; + const wScored = h.without.insight?.scored ?? 0; + const wPack = h.with.insight?.scored ?? 0; + lines.push(` insight Δ/run (without − with, scored ${wScored}/${wPack}):`); + lines.push(` search loops: ${fmtDelta(d.searchLoop)}`); + lines.push(` re-read churn: ${fmtDelta(d.rereadChurn)}`); + lines.push(` redundant search: ${fmtDelta(d.redundantSearch)}`); + lines.push(` shell-over-tool: ${fmtDelta(d.shellOverTool)}`); + } } return lines.join("\n"); } +/** Format a per-run insight delta with a leading sign (positive = pack helped). */ +function fmtDelta(n: number): string { + const s = n.toFixed(3); + return n > 0 ? `+${s}` : s; +} + /** Render an arm's token split: input + output + cache (the overhead inputs). */ function fmtTokens(t: ArmTokens): string { return `in ${t.inputTokens} + out ${t.outputTokens} + cache ${t.cacheTokens}`; diff --git a/packages/eval/src/runner.ts b/packages/eval/src/runner.ts index 1ff30e91..561da13b 100644 --- a/packages/eval/src/runner.ts +++ b/packages/eval/src/runner.ts @@ -14,6 +14,7 @@ */ import type { Task } from "./task.js"; +import type { Action } from "./trajectory.js"; /** Which coding agent a runner drives. */ export type Harness = "claude" | "codex"; @@ -82,6 +83,15 @@ export interface RunOutcome { * lower variance — but the oracle treats it as the worst outcome. */ readonly errored: boolean; + /** + * The normalized action sequence the agent took (Move 1 / TraceProbe). Present + * when the runner captured a tool-call stream (`--output-format stream-json` + * for Claude, `--json` for Codex); absent when the harness emitted no stream + * (older runners, or an errored run that produced no events). The INSIGHT + * detectors score this list; a run without a trajectory contributes no + * detector firings rather than a zero that could read as "clean". + */ + readonly trajectory?: readonly Action[]; } /** diff --git a/packages/eval/src/swebench.test.ts b/packages/eval/src/swebench.test.ts new file mode 100644 index 00000000..5c78e08b --- /dev/null +++ b/packages/eval/src/swebench.test.ts @@ -0,0 +1,106 @@ +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; +import { + buildAssertionCommand, + instanceToTask, + parseTestList, + type SweBenchInstance, +} from "./swebench.js"; + +const INSTANCE: SweBenchInstance = { + instance_id: "astropy__astropy-12907", + repo: "astropy/astropy", + base_commit: "d16bfe05a744909de4b27f5875fe0d4ed41ce607", + problem_statement: + "Modeling's `separability_matrix` does not compute correctly for nested CompoundModels.", + test_patch: "diff --git a/astropy/modeling/tests/test_separable.py ...", + FAIL_TO_PASS: '["astropy/modeling/tests/test_separable.py::test_nested"]', + PASS_TO_PASS: '["astropy/modeling/tests/test_separable.py::test_basic", "...::test_other"]', +}; + +describe("parseTestList", () => { + it("parses a JSON-string array", () => { + assert.deepEqual(parseTestList('["a::t1", "b::t2"]'), ["a::t1", "b::t2"]); + }); + it("passes through an already-parsed array", () => { + assert.deepEqual(parseTestList(["a", "b"]), ["a", "b"]); + }); + it("falls back to whitespace-split for a bare string", () => { + assert.deepEqual(parseTestList("a::t1 b::t2"), ["a::t1", "b::t2"]); + }); + it("returns [] for empty / malformed", () => { + assert.deepEqual(parseTestList(""), []); + assert.deepEqual(parseTestList("[not json"), ["[not", "json"]); + }); +}); + +describe("buildAssertionCommand", () => { + it("applies the test_patch then runs F2P+P2P under pytest -x", () => { + const cmd = buildAssertionCommand(INSTANCE, "pytest", "/tmp/p.patch"); + assert.match(cmd, /^git apply --3way '\/tmp\/p\.patch' && python -m pytest -x /); + assert.ok(cmd.includes("'astropy/modeling/tests/test_separable.py::test_nested'")); + assert.ok(cmd.includes("'astropy/modeling/tests/test_separable.py::test_basic'")); + // P2P tests are included alongside F2P. + assert.ok(cmd.includes("'...::test_other'")); + }); + + it("runs distinct test files (not node ids) under the node runner", () => { + const nodeInstance: SweBenchInstance = { + ...INSTANCE, + FAIL_TO_PASS: '["test/a.test.js::x", "test/a.test.js::y"]', + PASS_TO_PASS: '["test/b.test.js::z"]', + }; + const cmd = buildAssertionCommand(nodeInstance, "node", "/tmp/p.patch"); + assert.match(cmd, /node --test /); + // a.test.js appears once (deduped), b.test.js once. + assert.ok(cmd.includes("'test/a.test.js'")); + assert.ok(cmd.includes("'test/b.test.js'")); + assert.equal((cmd.match(/a\.test\.js/g) ?? []).length, 1, "file deduped across two node ids"); + }); + + it("shell-quotes node ids so :: and [param] pass verbatim", () => { + const paramInstance: SweBenchInstance = { + ...INSTANCE, + FAIL_TO_PASS: '["pkg/test_x.py::test_f[param-1]"]', + PASS_TO_PASS: "[]", + }; + const cmd = buildAssertionCommand(paramInstance, "pytest", "/tmp/p.patch"); + assert.ok(cmd.includes("'pkg/test_x.py::test_f[param-1]'")); + }); +}); + +describe("instanceToTask", () => { + it("maps an instance to an OCH assertion task + clone spec + patch", () => { + const gen = instanceToTask(INSTANCE, { + cloneRoot: "/tmp/swebench/", + testPatchPath: "/tmp/swebench/astropy__astropy-12907.patch", + }); + assert.equal(gen.task.id, "astropy__astropy-12907"); + assert.equal(gen.task.repo, "/tmp/swebench/astropy__astropy-12907"); + assert.equal(gen.task.commit, INSTANCE.base_commit); + assert.equal(gen.task.instruction, INSTANCE.problem_statement); + assert.equal(gen.task.oracle.type, "assertion"); + assert.equal(gen.task.oracle.timeoutMs, 600_000); + // clone spec + assert.equal(gen.clone.cloneUrl, "https://github.com/astropy/astropy.git"); + assert.equal(gen.clone.baseCommit, INSTANCE.base_commit); + assert.equal(gen.clone.dest, "/tmp/swebench/astropy__astropy-12907"); + // patch carried through + assert.equal(gen.testPatch, INSTANCE.test_patch); + }); + + it("honors a custom timeout and trailing-slash-normalizes cloneRoot", () => { + const gen = instanceToTask(INSTANCE, { + cloneRoot: "/tmp/sb///", + testPatchPath: "/tmp/p.patch", + timeoutMs: 120_000, + }); + assert.equal(gen.task.repo, "/tmp/sb/astropy__astropy-12907"); + assert.equal(gen.task.oracle.timeoutMs, 120_000); + }); + + it("normalizes an all-slash cloneRoot without ReDoS (linear strip)", () => { + const gen = instanceToTask(INSTANCE, { cloneRoot: "////", testPatchPath: "/tmp/p.patch" }); + assert.equal(gen.task.repo, "/astropy__astropy-12907"); + }); +}); diff --git a/packages/eval/src/swebench.ts b/packages/eval/src/swebench.ts new file mode 100644 index 00000000..c186fda4 --- /dev/null +++ b/packages/eval/src/swebench.ts @@ -0,0 +1,178 @@ +/** + * SWE-bench → variance-probe task conversion (Move 1, Phase 0). + * + * SWE-bench Verified (and Pro) ship each task as a fixed instance: a repo at a + * `base_commit`, a natural-language `problem_statement`, a `test_patch` that + * adds/updates the tests the fix must satisfy, and two test lists — + * `FAIL_TO_PASS` (must pass *after* the fix) and `PASS_TO_PASS` (must stay + * passing). That is exactly OCH's {@link Task} shape with an `assertion` oracle: + * the instruction is the problem statement, and the oracle applies the + * test_patch and runs the two test lists, exiting 0 iff all pass. + * + * Using real instances upgrades Finding 0001's honesty caveat — there, + * correctness was an eyeball judgment; here the F2P/P2P tests grade it. And + * because TraceProbe (arXiv:2607.06184) itself ran on SWE-bench Verified, the + * INSIGHT per-detector numbers this produces are comparable to the paper's. + * + * This module is the **pure transform** — instance JSON → task + on-disk + * artifacts descriptor. The clone / dependency-install / `codehub analyze` + * orchestration is inherently side-effectful and lives in the sibling + * `scripts/swebench-to-tasks.mjs` CLI; keeping the transform pure means it is + * unit-tested with no network, Docker, or filesystem. + * + * ⚠️ Fidelity limits (documented, not hidden — see the PR + Finding 0002): + * 1. **Per-run checkout isolation.** The v1 CLI runner runs the agent in the + * task's repo dir directly and does not reset between the N runs. Graded + * correctness needs a clean checkout per run; until the runner clones + * per-run, treat the *token + trajectory* deltas as the trustworthy + * headline and the assertion pass-rate as indicative. + * 2. **Environment.** Real repos need their deps installed to run tests. The + * prep script installs into a `/tmp` clone; for leaderboard-grade parity + * use SWE-bench's official per-instance Docker images (v2). + */ + +/** The subset of a SWE-bench instance this transform reads. */ +export interface SweBenchInstance { + readonly instance_id: string; + readonly repo: string; // "owner/name" + readonly base_commit: string; + readonly problem_statement: string; + readonly test_patch: string; + /** JSON-encoded array of test node ids, or an already-parsed array. */ + readonly FAIL_TO_PASS: string | readonly string[]; + readonly PASS_TO_PASS: string | readonly string[]; + /** Optional install/version hint carried by some instances. */ + readonly version?: string; +} + +/** The test runner a repo's F2P/P2P node ids are executed under. */ +export type TestRunner = "pytest" | "node"; + +export interface ToTaskOptions { + /** + * Absolute directory each instance's repo is cloned into by the prep script. + * The emitted task's `repo` is `${cloneRoot}/${instance_id}`. The generator + * and the prep script must agree on this. + */ + readonly cloneRoot: string; + /** + * Absolute path the instance's `test_patch` is written to (the assertion + * `git apply`s it before running tests). The generator writes the patch here. + */ + readonly testPatchPath: string; + /** Test runner for the F2P/P2P node ids. Defaults to `pytest` (SWE-bench is ~94% Python). */ + readonly runner?: TestRunner; + /** Per-command timeout (ms) for the assertion. Defaults to 600_000 (10 min). */ + readonly timeoutMs?: number; +} + +/** A generated OCH task plus the on-disk artifacts the generator must write. */ +export interface GeneratedTask { + /** The OCH task document (serialize to `.task.json`). */ + readonly task: { + readonly id: string; + readonly repo: string; + readonly commit: string; + readonly instruction: string; + readonly oracle: { + readonly type: "assertion"; + readonly command: string; + readonly timeoutMs: number; + }; + }; + /** The clone spec the prep script consumes for this instance. */ + readonly clone: { + readonly instanceId: string; + readonly cloneUrl: string; + readonly baseCommit: string; + readonly dest: string; + }; + /** The test_patch text to write to `testPatchPath` (empty when the instance carries none). */ + readonly testPatch: string; +} + +/** Parse a FAIL_TO_PASS / PASS_TO_PASS field that may be a JSON string or an array. */ +export function parseTestList(value: string | readonly string[]): string[] { + if (Array.isArray(value)) return value.filter((t): t is string => typeof t === "string"); + if (typeof value === "string") { + try { + const parsed: unknown = JSON.parse(value); + if (Array.isArray(parsed)) return parsed.filter((t): t is string => typeof t === "string"); + } catch { + // Not JSON — treat as a single whitespace-separated list. + return value.trim().length > 0 ? value.trim().split(/\s+/) : []; + } + } + return []; +} + +/** + * Build the assertion command that grades an instance: apply the test_patch, + * then run the union of F2P + P2P tests, exiting 0 iff all pass. `-x` (pytest) + * / bail (node) stops at the first failure so a broken run fails fast. + * + * The test_patch is applied with `git apply --3way`; the agent's fix is already + * in the working tree (the runner ran the agent in this checkout), so we apply + * only the tests on top. Shell-quoted so node ids with `::` and `[param]` are + * passed verbatim. + */ +export function buildAssertionCommand( + instance: SweBenchInstance, + runner: TestRunner, + testPatchPath: string, +): string { + const tests = [...parseTestList(instance.FAIL_TO_PASS), ...parseTestList(instance.PASS_TO_PASS)]; + const applyPatch = `git apply --3way ${shquote(testPatchPath)}`; + if (runner === "node") { + // node --test takes files, not node ids; run the distinct test files. + const files = [...new Set(tests.map((t) => t.split("::")[0] ?? t).filter((f) => f.length > 0))]; + const fileArgs = files.map(shquote).join(" "); + return `${applyPatch} && node --test ${fileArgs}`; + } + // pytest: pass the node ids directly; -x bails on first failure. + const idArgs = tests.map(shquote).join(" "); + return `${applyPatch} && python -m pytest -x ${idArgs}`; +} + +/** Convert one SWE-bench instance into a generated task + its artifacts. */ +export function instanceToTask(instance: SweBenchInstance, options: ToTaskOptions): GeneratedTask { + const runner = options.runner ?? "pytest"; + const timeoutMs = options.timeoutMs ?? 600_000; + const dest = `${stripTrailingSlashes(options.cloneRoot)}/${instance.instance_id}`; + return { + task: { + id: instance.instance_id, + repo: dest, + commit: instance.base_commit, + instruction: instance.problem_statement, + oracle: { + type: "assertion", + command: buildAssertionCommand(instance, runner, options.testPatchPath), + timeoutMs, + }, + }, + clone: { + instanceId: instance.instance_id, + cloneUrl: `https://github.com/${instance.repo}.git`, + baseCommit: instance.base_commit, + dest, + }, + testPatch: instance.test_patch, + }; +} + +/** Minimal single-quote shell quoting for a path / test node id. */ +function shquote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'`; +} + +/** + * Strip trailing `/` from a directory path. A linear char scan rather than a + * `/\/+$/` regex, which backtracks polynomially on an all-slash string + * (CodeQL js/polynomial-redos). + */ +function stripTrailingSlashes(path: string): string { + let end = path.length; + while (end > 0 && path[end - 1] === "/") end -= 1; + return path.slice(0, end); +} diff --git a/packages/eval/src/trajectory.test.ts b/packages/eval/src/trajectory.test.ts new file mode 100644 index 00000000..5d63506a --- /dev/null +++ b/packages/eval/src/trajectory.test.ts @@ -0,0 +1,225 @@ +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; +import { + type Action, + actionsFromClaudeStreamJson, + actionsFromCodexJsonl, + isShellReadSearch, + isValidationCommand, + normalizeQuery, + shellFirstWord, +} from "./trajectory.js"; + +describe("normalizeQuery", () => { + it("trims and collapses internal whitespace, preserving case", () => { + assert.equal(normalizeQuery(" foo bar\tbaz "), "foo bar baz"); + assert.equal(normalizeQuery("Foo"), "Foo"); + assert.notEqual(normalizeQuery("Foo"), normalizeQuery("foo")); + }); +}); + +describe("shellFirstWord", () => { + it("unwraps a shell -c wrapper (Codex's /bin/zsh -lc form)", () => { + assert.equal(shellFirstWord("/bin/zsh -lc 'cat data.txt'"), "cat"); + assert.equal(shellFirstWord("bash -c \"find . -name '*.ts'\""), "find"); + }); + it("takes the basename of an absolute program path", () => { + assert.equal(shellFirstWord("/usr/bin/rg pattern src/"), "rg"); + }); + it("skips leading VAR=value assignments and sudo/env/time", () => { + assert.equal(shellFirstWord("FOO=1 BAR=2 grep -rn foo"), "grep"); + assert.equal(shellFirstWord("sudo find / -name x"), "find"); + assert.equal(shellFirstWord("env RUST_LOG=debug cargo test"), "cargo"); + }); + it("lower-cases and handles a bare command", () => { + assert.equal(shellFirstWord("GREP foo"), "grep"); + assert.equal(shellFirstWord(""), ""); + }); + + it("unwraps a tab-separated shell wrapper (ReDoS-safe path)", () => { + // The old regex backtracked polynomially on 'sh\t-c\t…'; the tokenized + // unwrap handles tabs the same as spaces in linear time. + assert.equal(shellFirstWord("sh\t-c\t'cat f'"), "cat"); + }); + + it("does not unwrap a non-shell program that merely has a -c-looking arg", () => { + // `gcc -c foo.c` is a compile, not a shell wrapper — the first word stands. + assert.equal(shellFirstWord("gcc -c foo.c"), "gcc"); + }); +}); + +describe("isShellReadSearch (Shell-over-Tool set)", () => { + it("flags the frozen read/search program set", () => { + for (const cmd of [ + "cat file", + "head -n5 f", + "tail f", + "less f", + "more f", + "grep -rn foo .", + "egrep x f", + "rg pattern", + "ag pattern", + "find . -name '*.ts'", + "/bin/zsh -lc 'grep -rn TODO src/'", + ]) { + assert.equal(isShellReadSearch(cmd), true, cmd); + } + }); + it("does NOT flag builds, tests, writes, or other commands", () => { + for (const cmd of [ + "pnpm test", + "node --test x.js", + "python app.py", + "git status", + "sed -i s/a/b/ f", + ]) { + assert.equal(isShellReadSearch(cmd), false, cmd); + } + }); +}); + +describe("isValidationCommand (Search Loop breaker)", () => { + it("recognizes test/build/lint across ecosystems", () => { + for (const cmd of [ + "pytest tests/", + "python -m pytest -k foo", + "python3 -m unittest", + "tox", + "pnpm run test", + "npm test", + "node --test ./dist/x.test.js", + "vitest run", + "tsc -b", + "ruff check .", + "cargo test", + "go test ./...", + "make check", + "mvn test", + "/bin/zsh -lc 'pnpm run build'", + ]) { + assert.equal(isValidationCommand(cmd), true, cmd); + } + }); + it("does NOT treat reads/searches/greps as validation", () => { + for (const cmd of ["cat f", "grep -rn foo .", "ls -la", "echo hi", "git diff"]) { + assert.equal(isValidationCommand(cmd), false, cmd); + } + }); +}); + +describe("actionsFromClaudeStreamJson", () => { + it("maps every built-in tool to its canonical action type", () => { + const stream = [ + evt("Glob", { pattern: "**/*.ts" }), + evt("Grep", { pattern: " handleAuth " }), + evt("Read", { file_path: "/src/a.ts" }), + evt("Edit", { file_path: "/src/a.ts" }), + evt("Write", { file_path: "/src/b.ts" }), + evt("Bash", { command: "pnpm test" }), + evt("Task", { subagent_type: "x" }), + evt("TodoWrite", { todos: [] }), + evt("WebFetch", { url: "https://x" }), + evt("mcp__codehub__query", { query: " Foo bar " }), + evt("mcp__codehub__impact", {}), + ].join("\n"); + const actions = actionsFromClaudeStreamJson(stream); + assert.deepEqual(actions, [ + { type: "search", query: "**/*.ts" }, + { type: "search", query: "handleAuth" }, // normalized + { type: "file_read", target: "/src/a.ts" }, + { type: "file_write", target: "/src/a.ts" }, + { type: "file_write", target: "/src/b.ts" }, + { type: "command", command: "pnpm test" }, + { type: "spawn" }, + { type: "plan" }, + { type: "fetch", target: "https://x" }, + { type: "search", query: "Foo bar" }, // mcp query→search by name heuristic + { type: "navigate" }, // unknown mcp tool → navigate (no detector reads it) + ]); + }); + + it("maps assistant text/thinking blocks to reason and skips user/system/result events", () => { + const stream = [ + JSON.stringify({ type: "system", subtype: "init" }), + JSON.stringify({ + type: "assistant", + message: { + content: [ + { type: "thinking", thinking: "hmm" }, + { type: "text", text: "go" }, + ], + }, + }), + JSON.stringify({ + type: "user", + message: { content: [{ type: "tool_result", tool_use_id: "t" }] }, + }), + JSON.stringify({ type: "result", result: "done" }), + ].join("\n"); + assert.deepEqual(actionsFromClaudeStreamJson(stream), [{ type: "reason" }, { type: "reason" }]); + }); + + it("tolerates non-JSON / blank lines", () => { + const stream = ["", " ", "not json", evt("Read", { file_path: "/x" })].join("\n"); + assert.deepEqual(actionsFromClaudeStreamJson(stream), [{ type: "file_read", target: "/x" }]); + }); +}); + +describe("actionsFromCodexJsonl", () => { + it("maps command_execution, file_change, web_search, reasoning; skips agent_message", () => { + const stream = [ + JSON.stringify({ + type: "item.completed", + item: { type: "command_execution", command: "cat f" }, + }), + JSON.stringify({ + type: "item.completed", + item: { + type: "file_change", + changes: [ + { path: "/a", kind: "update" }, + { path: "/b", kind: "add" }, + ], + }, + }), + JSON.stringify({ type: "item.completed", item: { type: "web_search", query: "how to x" } }), + JSON.stringify({ type: "item.completed", item: { type: "reasoning", text: "..." } }), + JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "final" } }), + JSON.stringify({ type: "turn.completed", usage: { input_tokens: 1 } }), + ].join("\n"); + assert.deepEqual(actionsFromCodexJsonl(stream), [ + { type: "command", command: "cat f" }, + { type: "file_write", target: "/a" }, + { type: "file_write", target: "/b" }, + { type: "search", query: "how to x" }, + { type: "reason" }, + ]); + }); + + it("ignores item.started twins (counts each item once at completion)", () => { + const stream = [ + JSON.stringify({ + type: "item.started", + item: { type: "command_execution", command: "cat f" }, + }), + JSON.stringify({ + type: "item.completed", + item: { type: "command_execution", command: "cat f" }, + }), + ].join("\n"); + assert.deepEqual(actionsFromCodexJsonl(stream), [{ type: "command", command: "cat f" }]); + }); +}); + +/** Build one Claude assistant tool_use event line. */ +function evt(name: string, input: Record): string { + return JSON.stringify({ + type: "assistant", + message: { content: [{ type: "tool_use", id: "t", name, input }] }, + }); +} + +// Type-only guard: Action is structurally what the detectors consume. +const _sample: Action = { type: "search", query: "x" }; +void _sample; diff --git a/packages/eval/src/trajectory.ts b/packages/eval/src/trajectory.ts new file mode 100644 index 00000000..2d4598eb --- /dev/null +++ b/packages/eval/src/trajectory.ts @@ -0,0 +1,434 @@ +/** + * Trajectory capture + normalization (Move 1 / arXiv:2607.06184 "TraceProbe"). + * + * The variance probe captures each agent run's *outcome* (final text, tokens). + * TraceProbe's insight is that the *trajectory* — the ordered action sequence — + * carries diagnostic signal a pass/fail label hides: search loops, re-read + * churn, redundant search. To score those, we first normalize every harness's + * raw event stream into one canonical action list. + * + * The taxonomy is TraceProbe's **nine canonical action types** (§ "canonical + * actions from a nine-type taxonomy: file read, file write, search, command, + * sub-agent spawn, plan, navigate, fetch, and reason"). We keep the four + * *structural* fields the deterministic INSIGHT detectors need — action type, + * file target, normalized search query, and the raw command string — and drop + * the semantic effect labels (failed / reverted), which need an LLM labeler and + * are a v2 concern (see `insight.ts`). + * + * Faithfulness note: the detectors' "10-action window" is defined over this + * full canonical list *including* `reason` actions, exactly as the paper does, + * so our per-detector numbers are comparable to TraceProbe's own figures rather + * than a bespoke tool-only variant. + * + * Both normalizers are pure functions of the captured stdout — no clock, no + * filesystem — so a trajectory is a deterministic function of the run's bytes. + */ + +/** TraceProbe's nine canonical action types. */ +export type ActionType = + | "file_read" + | "file_write" + | "search" + | "command" + | "spawn" + | "plan" + | "navigate" + | "fetch" + | "reason"; + +/** + * One normalized action in a run's trajectory. Optional fields carry only the + * signal the structural detectors read: + * - `target` — canonical file path for `file_read`/`file_write`; URL for `fetch`. + * - `query` — normalized search query for `search`. + * - `command` — raw command line for `command` (the shell-first-word and + * validation classifiers derive from it). + */ +export interface Action { + readonly type: ActionType; + readonly target?: string; + readonly query?: string; + readonly command?: string; +} + +/** + * Normalize a search query for equality comparison (Redundant Search): trim and + * collapse internal whitespace runs to a single space. Case is preserved — + * agent search patterns are case-sensitive regexes, so `Foo` and `foo` are + * genuinely different queries. + */ +export function normalizeQuery(raw: string): string { + return raw.trim().replace(/\s+/g, " "); +} + +/** + * Recover the real program name from a command line, unwrapping the shell + * invocation harnesses wrap commands in (Codex emits `/bin/zsh -lc ''`; + * Claude Bash runs bare). Strips a leading shell wrapper, then leading + * `sudo` / `command` / `env` and `VAR=value` assignments, then returns the + * basename of the program token, lower-cased. Returns "" when empty. + * + * Examples: + * "/bin/zsh -lc 'cat data.txt'" → "cat" + * "grep -rn foo src/" → "grep" + * "FOO=1 /usr/bin/rg pattern" → "rg" + * "bash -c \"find . -name '*.ts'\"" → "find" + */ +export function shellFirstWord(command: string): string { + const inner = unwrapShell(command).trim(); + const tokens = inner.split(/\s+/); + let i = 0; + // Skip leading VAR=value assignments and trivial prefixes. + while (i < tokens.length) { + const t = tokens[i]; + if (t === undefined) break; + if (t === "sudo" || t === "command" || t === "env" || t === "time") { + i += 1; + continue; + } + if (/^[A-Za-z_][A-Za-z0-9_]*=/.test(t)) { + i += 1; + continue; + } + break; + } + const program = tokens[i] ?? ""; + const base = program.split("/").pop() ?? program; + return base.toLowerCase(); +} + +/** Shell program basenames that take a `-c`-style inline-command flag. */ +const SHELL_PROGRAMS: ReadonlySet = new Set(["sh", "bash", "zsh", "dash", "ksh"]); + +/** + * Unwrap one layer of `sh -c ''` / `/bin/zsh -lc ""`, else return + * the command unchanged. Tokenized (not regex-matched) to stay linear-time — a + * backtracking regex over library-influenced command strings is a ReDoS risk + * (CodeQL js/polynomial-redos). We split off at most the first two tokens: a + * shell program whose basename is in {@link SHELL_PROGRAMS} followed by a + * `-…c` flag; the inner command is the remainder after that flag. + */ +function unwrapShell(command: string): string { + const trimmed = trimStart(command); + const firstSp = indexOfWhitespace(trimmed); + if (firstSp < 0) return command; + const prog = trimmed.slice(0, firstSp); + const progBase = (prog.split("/").pop() ?? prog).toLowerCase(); + if (!SHELL_PROGRAMS.has(progBase)) return command; + + const afterProg = trimStart(trimmed.slice(firstSp)); + const flagEnd = indexOfWhitespace(afterProg); + if (flagEnd < 0) return command; + const flag = afterProg.slice(0, flagEnd); + // Flag must start with '-' and end in 'c' (e.g. -c, -lc, -ic) and hold only + // letters between — a simple char scan, no regex. + if (!isDashCFlag(flag)) return command; + + const inner = trimStart(afterProg.slice(flagEnd)); + return stripOuterQuotes(inner); +} + +/** True for a `-[A-Za-z]*c` flag, checked by linear char scan (no regex). */ +function isDashCFlag(flag: string): boolean { + if (flag.length < 2 || flag[0] !== "-" || flag[flag.length - 1] !== "c") return false; + for (let i = 1; i < flag.length - 1; i += 1) { + const c = flag.charCodeAt(i); + const isAlpha = (c >= 65 && c <= 90) || (c >= 97 && c <= 122); + if (!isAlpha) return false; + } + return true; +} + +/** Index of the first ASCII-whitespace char, or -1. Linear, no regex. */ +function indexOfWhitespace(s: string): number { + for (let i = 0; i < s.length; i += 1) { + const c = s.charCodeAt(i); + if (c === 32 || c === 9 || c === 10 || c === 13 || c === 11 || c === 12) return i; + } + return -1; +} + +/** Strip leading ASCII whitespace. Linear, no regex. */ +function trimStart(s: string): string { + let i = 0; + while (i < s.length) { + const c = s.charCodeAt(i); + if (c === 32 || c === 9 || c === 10 || c === 13 || c === 11 || c === 12) i += 1; + else break; + } + return i === 0 ? s : s.slice(i); +} + +/** Strip one matched layer of surrounding single or double quotes. */ +function stripOuterQuotes(s: string): string { + if (s.length >= 2) { + const first = s[0]; + const last = s[s.length - 1]; + if ((first === "'" || first === '"') && last === first) { + return s.slice(1, -1); + } + } + return s; +} + +/** + * Programs whose use is a "read" or "search" done through the shell instead of + * a structured tool — the Shell-over-Tool anti-pattern's frozen set (grep + * family, rg/ag, find, and the pager/cat readers). + */ +const SHELL_READ_SEARCH_PROGRAMS: ReadonlySet = new Set([ + "cat", + "head", + "tail", + "less", + "more", + "grep", + "egrep", + "fgrep", + "rg", + "ag", + "find", +]); + +/** True when a command's unwrapped program is in the shell read/search set. */ +export function isShellReadSearch(command: string): boolean { + return SHELL_READ_SEARCH_PROGRAMS.has(shellFirstWord(command)); +} + +/** + * Heuristic validation-command classifier (Search Loop reset condition): does + * this command run tests / a build / a linter? A run of search+read actions is + * "broken" by a validation command — the agent stopped hunting and checked its + * work. Matched against the unwrapped command line, so wrapper-nested test runs + * still classify. Deliberately broad across the common ecosystems; it is a + * documented heuristic, not an exhaustive oracle. + */ +export function isValidationCommand(command: string): boolean { + const inner = unwrapShell(command); + return VALIDATION_PATTERNS.some((re) => re.test(inner)); +} + +const VALIDATION_PATTERNS: readonly RegExp[] = [ + /\b(pytest|py\.test)\b/, + /\bpython[0-9.]*\s+-m\s+(pytest|unittest|tox|nose2?)\b/, + /\b(tox|nox)\b/, + /\bunittest\b/, + /\b(npm|pnpm|yarn|bun)\s+(run\s+)?(test|build|lint|typecheck|check)\b/, + /\bnode\s+--test\b/, + /\b(vitest|jest|mocha|ava|tap)\b/, + /\b(tsc|biome|eslint|ruff|mypy|pyright)\b/, + /\bcargo\s+(test|build|check|clippy|nextest)\b/, + /\bgo\s+(test|build|vet)\b/, + /\b(make|ctest|cmake)\b/, + /\b(mvn|gradle|gradlew)\b/, + /\b(phpunit|rspec|rake\s+test)\b/, + /\bbazel\s+(test|build)\b/, + /\bdotnet\s+test\b/, +]; + +/* ────────────────────────────── Claude Code ────────────────────────────── */ + +/** + * Normalize Claude Code's `--output-format stream-json --verbose` JSONL into a + * canonical action list. Each `assistant` event's `message.content[]` blocks + * become actions in order: `thinking`/`text` → `reason`, `tool_use` → the + * mapped action. `user` (tool_result) and `system`/`result` events carry no + * agent action and are skipped. Non-JSON / partial lines are tolerated. + * + * Tool → action mapping (built-in tool `input` shapes are grounded against + * Claude Code 2.1.x): Read→file_read, Write/Edit/NotebookEdit→file_write, + * Grep/Glob→search, Bash→command, Task/Agent→spawn, TodoWrite/TaskCreate/ + * TaskUpdate→plan, WebFetch/WebSearch→fetch. Unknown tools (including OCH's own + * `mcp__…` graph tools) map by a name heuristic, defaulting to `navigate` so an + * unclassified tool never masquerades as a read/search/write/command and skews + * a detector. + */ +export function actionsFromClaudeStreamJson(stdout: string): Action[] { + const actions: Action[] = []; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + let evt: unknown; + try { + evt = JSON.parse(trimmed); + } catch { + continue; + } + if (typeof evt !== "object" || evt === null) continue; + const e = evt as { type?: unknown; message?: { content?: unknown } }; + if (e.type !== "assistant") continue; + const content = e.message?.content; + if (!Array.isArray(content)) continue; + for (const block of content) { + const action = claudeBlockToAction(block); + if (action !== undefined) actions.push(action); + } + } + return actions; +} + +function claudeBlockToAction(block: unknown): Action | undefined { + if (typeof block !== "object" || block === null) return undefined; + const b = block as { type?: unknown; name?: unknown; input?: unknown }; + if (b.type === "thinking" || b.type === "text") return { type: "reason" }; + if (b.type !== "tool_use" || typeof b.name !== "string") return undefined; + const input = (typeof b.input === "object" && b.input !== null ? b.input : {}) as Record< + string, + unknown + >; + return claudeToolToAction(b.name, input); +} + +function claudeToolToAction(name: string, input: Record): Action { + const filePath = str(input["file_path"]); + switch (name) { + case "Read": + return filePath !== undefined + ? { type: "file_read", target: filePath } + : { type: "file_read" }; + case "Write": + case "Edit": + case "MultiEdit": + case "NotebookEdit": + return filePath !== undefined + ? { type: "file_write", target: filePath } + : { type: "file_write" }; + case "Grep": + case "Glob": { + const pattern = str(input["pattern"]); + return pattern !== undefined + ? { type: "search", query: normalizeQuery(pattern) } + : { type: "search" }; + } + case "Bash": { + const command = str(input["command"]); + return command !== undefined ? { type: "command", command } : { type: "command" }; + } + case "Task": + case "Agent": + return { type: "spawn" }; + case "TodoWrite": + case "TaskCreate": + case "TaskUpdate": + case "ExitPlanMode": + return { type: "plan" }; + case "WebFetch": { + const url = str(input["url"]); + return url !== undefined ? { type: "fetch", target: url } : { type: "fetch" }; + } + case "WebSearch": { + const query = str(input["query"]); + return query !== undefined + ? { type: "search", query: normalizeQuery(query) } + : { type: "search" }; + } + default: + return unknownToolToAction(name, input); + } +} + +/** + * Map an unrecognized tool (an MCP tool such as OCH's `mcp__…query` / `impact`, + * or a newly-added built-in) by a conservative name heuristic. Search-ish and + * read-ish MCP tools are the only ones promoted into a detector-relevant type; + * everything else lands in `navigate`, which no structural detector reads. + */ +function unknownToolToAction(name: string, input: Record): Action { + const lower = name.toLowerCase(); + if (/search|query|grep|find|list_findings|dead_code/.test(lower)) { + const q = str(input["query"]) ?? str(input["pattern"]); + return q !== undefined ? { type: "search", query: normalizeQuery(q) } : { type: "search" }; + } + if (/fetch|http|url|web/.test(lower)) return { type: "fetch" }; + return { type: "navigate" }; +} + +/* ─────────────────────────────────── Codex ─────────────────────────────── */ + +/** + * Normalize Codex's `exec --json` JSONL into a canonical action list. Codex + * reports work as `item.completed` events; we read each item once at + * completion (its `item.started` twin is ignored to avoid double-counting). + * + * Item → action mapping (grounded against codex-cli 0.143.x + * `exec_events.rs`): `command_execution`→command (Codex does reads/greps + * through the shell, so those surface here and are caught by Shell-over-Tool), + * `file_change`→one `file_write` per changed path, `web_search`→search, + * `reasoning`→reason. `agent_message` is the final answer, not an action, and + * is skipped. `mcp_tool_call` maps by the same name heuristic as Claude. + */ +export function actionsFromCodexJsonl(stdout: string): Action[] { + const actions: Action[] = []; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + let evt: unknown; + try { + evt = JSON.parse(trimmed); + } catch { + continue; + } + if (typeof evt !== "object" || evt === null) continue; + const e = evt as { type?: unknown; item?: unknown }; + if (e.type !== "item.completed") continue; + pushCodexItem(actions, e.item); + } + return actions; +} + +function pushCodexItem(actions: Action[], item: unknown): void { + if (typeof item !== "object" || item === null) return; + const it = item as { + type?: unknown; + command?: unknown; + changes?: unknown; + query?: unknown; + server?: unknown; + tool?: unknown; + arguments?: unknown; + }; + switch (it.type) { + case "command_execution": { + const command = str(it.command); + actions.push(command !== undefined ? { type: "command", command } : { type: "command" }); + return; + } + case "file_change": { + if (Array.isArray(it.changes)) { + for (const ch of it.changes) { + const path = str((ch as { path?: unknown } | null)?.path); + actions.push( + path !== undefined ? { type: "file_write", target: path } : { type: "file_write" }, + ); + } + } else { + actions.push({ type: "file_write" }); + } + return; + } + case "web_search": { + const query = str(it.query); + actions.push( + query !== undefined ? { type: "search", query: normalizeQuery(query) } : { type: "search" }, + ); + return; + } + case "reasoning": + actions.push({ type: "reason" }); + return; + case "mcp_tool_call": { + const tool = str(it.tool) ?? ""; + actions.push(unknownToolToAction(tool, {})); + return; + } + // agent_message (final answer), todo_list, and unknown item types carry no + // detector-relevant action in v1. + default: + return; + } +} + +/** Narrow an unknown JSON value to a non-empty string, else undefined. */ +function str(v: unknown): string | undefined { + return typeof v === "string" && v.length > 0 ? v : undefined; +}