diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..213bbe3796 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,172 @@ +name: eve Benchmarks + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - "packages/eve-bench/**" + - research/eve-bench-runner.md + workflow_dispatch: + inputs: + model: + description: Model to benchmark + required: false + default: zai/glm-5.2 + type: string + cohort: + description: Task cohort to run + required: false + default: smoke + type: string + base_job_artifact: + description: Optional artifact containing a base job to compare + required: false + type: string + +concurrency: + group: eve-bench-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + oracle-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + # The typecheck and the tool and agent tests import eve's built exports. + - name: Build eve package + run: pnpm --filter eve run build + working-directory: ${{ github.workspace }} + + - name: Run tests + run: pnpm run test + + # Real containers: timeouts, bad images, abort/resume, concurrency, and trial validity. + - name: Run Docker stress tests + run: pnpm run test:stress + + - name: Sync benchmark tasks + run: pnpm bench tasks sync --json + + - name: Run oracle smoke benchmark + run: pnpm bench run --harness oracle --cohort smoke --concurrency 5 --job oracle-smoke --format junit > junit.xml + + - name: Print oracle smoke report and require valid attempts + run: pnpm bench report oracle-smoke --fail-on-invalid + + - name: Upload oracle smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-oracle-smoke + path: | + packages/eve-bench/.generated/jobs/oracle-smoke + packages/eve-bench/junit.xml + include-hidden-files: true + if-no-files-found: warn + + - name: Require every oracle attempt to resolve + run: | + node -e ' + const result = require("./.generated/jobs/oracle-smoke/result.json"); + const { attempts, resolved } = result.summary; + if (resolved < attempts) { + console.error(`Oracle resolved ${resolved} of ${attempts} attempts`); + process.exit(1); + } + ' + + # AI_GATEWAY_API_KEY is available to pull requests from branches in this repository, not forks. + eve-smoke: + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bench:eve') }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + - name: Build eve package + run: pnpm --filter eve run build + working-directory: ${{ github.workspace }} + + - name: Run eve smoke benchmark + env: + MODEL: ${{ inputs.model || 'zai/glm-5.2' }} + COHORT: ${{ inputs.cohort || 'smoke' }} + run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke + + # Scores are only meaningful when every attempt reached the model and the verifier. + - name: Print eve smoke report and require valid attempts + run: pnpm bench report eve-smoke --fail-on-invalid + + - name: Download base job artifact + if: ${{ inputs.base_job_artifact }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.base_job_artifact }} + path: packages/eve-bench/.generated/base-job + + - name: Compare with base job + if: ${{ inputs.base_job_artifact }} + run: | + pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt + { + echo "## Benchmark diff" + echo '```text' + cat diff.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload eve smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-eve-smoke + path: packages/eve-bench/.generated/jobs/eve-smoke + include-hidden-files: true + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index b8d7025889..c05d3acf3b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ coverage/ .pnpm-store/ .bun/ .npm/ +__pycache__/ # Test artifacts .eve-bench-runner/ @@ -27,6 +28,7 @@ apps/benchmarks/evals/*/.eve-authoring-bootstrap.json packages/eve/.workflow-vitest/ packages/eve/.generated/ packages/eve/src/extensions/code/extension/ +packages/eve-bench/.generated/ playwright-report/ test-results/ @@ -39,6 +41,7 @@ packages/eve-buzz-acp-adapter/NOTICE # Logs *.log +/jobs/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md new file mode 100644 index 0000000000..048c72442c --- /dev/null +++ b/packages/eve-bench/README.md @@ -0,0 +1,183 @@ +# @eve/bench + +Terminal-Bench harness benchmarks for eve, run by a zero-dependency Node runner. + +The package owns the harness under evaluation (`agent/`), pinned dataset locks +and task cohorts (`datasets/`), and the runner (`src/`). Tasks keep the +Terminal-Bench layout and reward contract, so results stay comparable with +published runs. See `research/eve-bench-runner.md` for the design. + +## Prerequisites + +- Node.js 24 and pnpm +- Docker +- model-provider credentials in the environment, such as `AI_GATEWAY_API_KEY` +- a built local eve package when using `--eve local`; run `pnpm build` from the + repository root first + +Everything below runs from this directory with `pnpm bench` (alias for +`node src/cli.ts`). + +## Run + +```bash +pnpm bench tasks sync # fetch pinned datasets into .generated/datasets +pnpm bench tasks list --cohort smoke # print a cohort's task names + +AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8 +pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model +pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release +pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task +pnpm bench prepare --harness eve --model zai/glm-5.2 # build and cache a bundle without calling a model + +pnpm bench report --format junit --out junit.xml +pnpm bench diff +``` + +Every command accepts `--json` and never prompts. Jobs live in +`.generated/jobs//`. Re-running the same job resumes only when the model, +harness, bundle provenance, dataset, tasks, and attempt count match. Invalid or +mismatched trial records fail closed instead of being overwritten. Ctrl-C +aborts in-flight trials and removes their containers. + +Pass `--task-dir ` more than once to run one or more local Terminal-Bench +task directories. A local-only run does not sync a dataset and records its +dataset as `local`. + +## How a trial runs + +1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). +2. Start one container, upload the harness bundle to `/installed-agent`, and + run the harness command under the task's agent timeout. +3. Upload `tests/` and run `tests/test.sh` under the verifier timeout. The + verifier always runs, so the reward reflects what the agent left behind. +4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` + to the trial directory, remove the container, and write `trial.json`. + +## Reading results + +A trial is scored only when it measured the harness. Otherwise `trial.json` +records `invalid` with a phase and reason, and the trial is excluded from +`resolved`, `meanReward`, and `diff`: + +- `environment`: runner-owned work failed, such as the image pull or build, + container start, uploads, or log download. +- `harness`: the harness ran but completed no model call (zero or missing token + usage). This usually means broken credentials, adapter configuration, or + startup, not a weak harness. +- `verifier`: the verifier wrote no reward, including verifier timeouts. + +After one model call completes, crashes, agent timeouts, and wrong answers +count against the harness. Reports list every invalid attempt, JUnit marks them +as errors instead of failures, and `diff` shows invalid counts next to each +task. Resolve invalid attempts before comparing jobs; `pnpm bench report +--fail-on-invalid` exits 1 when any remain, and CI applies it to every smoke job. + +Usage is read on the host from each harness's own event log (`events.ndjson` +for eve, `.jsonl` for pi, OpenCode, and Codex), so it survives agent +timeouts. Input tokens include cache reads and output tokens include reasoning. +`costUsd` is null when the harness reports no cost; the CLI competitors report +none for custom Gateway providers, so compare tokens rather than cost across +harnesses. + +## Environment parity + +Every harness's tool commands run with the task container's own environment: +the image's `HOME`, `PATH`, and `ENV`, starting in the task working directory. +Harness state lives only in harness-owned paths under `/installed-agent` +(`CODEX_HOME`, `PI_CODING_AGENT_DIR`, `OPENCODE_CONFIG`), so what an agent +installs or configures in the task survives into verification. The eve runner +restores the container's own `PORT`, `HOST`, and Nitro variables for agent +commands. + +Each runner-owned log (`container.log`, `agent.log`, and `verifier.log`) is +capped at 64 MiB. The runner appends a truncation notice when a log exceeds the +cap. + +Containers carry the labels `eve-bench=1`, `eve-bench.job=`, and +`eve-bench.task=`. Find any leftovers with: + +```bash +docker ps -a --filter label=eve-bench=1 +``` + +At the start of a job, the runner removes stale containers with the same +`eve-bench.job` label before starting new trials. + +## Stress tests + +Run the Docker-backed stress suite separately from the default test suite: + +```bash +pnpm run test:stress +``` + +The suite requires Docker and takes about 30 seconds. It covers the happy path, +agent and verifier timeouts, a bad image, SIGINT abort and resume, and concurrent +trials. + +## Harnesses + +- `eve` (default): builds `agent/` against the local checkout (`--eve local`) + or an exact published version, then packages the Nitro output and a small + runner that drives the server over HTTP. Build eve from the repository root + with `pnpm build` before using `--eve local`. For cache lookup, `prepare()` + derives the local eve identity from `packages/eve/dist` instead of packing + first. The cache lives in `.generated/cache/`. Each trial uploads the bundle + and only the Linux Node binary matching the task container's architecture, + for about 128 MB total. + + The agent's tools operate on the task working directory through + `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass + eve's nested sandbox on purpose. + +- `pi`, `opencode`, and `codex`: prepare explicitly versioned CLI bundles on the + host and route full provider-qualified model IDs through an HTTPS API base. + Pass an exact `--version`. The default base is Vercel AI Gateway. Codex uses + the Responses protocol; pi and OpenCode use chat completions. Hermes fails + explicitly until a pinned Python/uv bundle and verified custom-provider + routing are implemented. + +- `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner + and as the reward ceiling. + +Adding a harness means implementing `src/core/harness.ts` (prepare a +directory, return a command and env) and registering it in `src/cli.ts`. + +## Agent tools + +`src/tools/` exposes `run_bench`, `get_bench_result`, `diff_bench`, and +`list_bench_tasks` as eve `defineTool` definitions over the same core used by +the CLI. Their results use the same `JobResult`, `TaskDelta`, and task-list JSON +as the CLI. Add a file to an agent's `tools/` directory that re-exports a +definition by relative path, or copy the definition there. + +## Datasets and cohorts + +`datasets/-.json` pins a git URL, commit, and the task paths in +it. `datasets/cohorts/.json` names a reviewed task subset and the reason +it exists. Compare base and candidate runs with the same cohort, model, +attempt count, and eve source. + +## CI + +`.github/workflows/bench.yml` runs the oracle smoke cohort on pull requests that +touch this package or its design document and requires every task attempt to +resolve. It uploads the job directory and `junit.xml` as artifacts. + +The real-model `eve-smoke` job runs through `workflow_dispatch` or when the +`bench:eve` label is added to a pull request. It accepts `model` and `cohort` +inputs for manual runs and requires `AI_GATEWAY_API_KEY`, which is unavailable +to pull requests from forks. It uploads its job directory and can optionally +diff the result against a downloaded base-job artifact. + +## Layout + +- `agent/`: harness behavior under evaluation (model binding, instructions, tools) +- `src/core/`: runner library with no dependency on eve or third parties +- `src/harnesses/`: `eve`, CLI competitors, and `oracle` +- `src/tools/`: eve tool definitions over the runner core +- `src/options.ts`: run options shared by the CLI and tools +- `src/cli.ts`: `eve-bench` CLI +- `datasets/`: locks and cohorts +- `.generated/`: datasets, bundle cache, and job output; never committed diff --git a/packages/eve-bench/agent/agent.ts b/packages/eve-bench/agent/agent.ts new file mode 100644 index 0000000000..595843cf27 --- /dev/null +++ b/packages/eve-bench/agent/agent.ts @@ -0,0 +1,10 @@ +import { defineAgent } from "eve"; + +const model = process.env.EVE_BENCH_MODEL; +if (model === undefined || model.length === 0) { + throw new Error("EVE_BENCH_MODEL must be set by the eve-bench harness."); +} + +export default defineAgent({ + model, +}); diff --git a/packages/eve-bench/agent/instructions.md b/packages/eve-bench/agent/instructions.md new file mode 100644 index 0000000000..75ea47d355 --- /dev/null +++ b/packages/eve-bench/agent/instructions.md @@ -0,0 +1,11 @@ +You are an autonomous terminal task agent. + +Work until the requested task is complete or conclusively cannot be completed. + +- Inspect relevant files before modifying them. +- Use the available terminal and file tools to work directly in the task environment. +- Make the smallest correct changes that satisfy the task. +- Run relevant validation after making changes. +- Do not ask the user questions. +- Do not use network access unless the task explicitly requires it. +- Finish with a concise description of the completed work and validation. diff --git a/packages/eve-bench/agent/task-path.ts b/packages/eve-bench/agent/task-path.ts new file mode 100644 index 0000000000..b27ed33c6b --- /dev/null +++ b/packages/eve-bench/agent/task-path.ts @@ -0,0 +1,28 @@ +import { isAbsolute, resolve } from "node:path"; + +export function taskRoot(): string { + const root = process.env.EVE_BENCH_TASK_WORKDIR; + if (root === undefined || root.length === 0) { + throw new Error("EVE_BENCH_TASK_WORKDIR must be set by the eve-bench harness."); + } + return root; +} + +export function resolveTaskPath(path = "."): string { + return isAbsolute(path) ? path : resolve(taskRoot(), path); +} + +/** + * The container environment as the task sees it: the runner's server-only + * overrides (such as PORT) are restored so task commands behave as they would + * for any other harness. + */ +export function taskEnv(): NodeJS.ProcessEnv { + const { EVE_BENCH_TASK_ENV_RESTORE: restore, ...env } = process.env; + const original = JSON.parse(restore ?? "{}") as Record; + for (const [key, value] of Object.entries(original)) { + if (value === null) delete env[key]; + else env[key] = value; + } + return env; +} diff --git a/packages/eve-bench/agent/tools/ask_question.ts b/packages/eve-bench/agent/tools/ask_question.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/ask_question.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/bash.ts b/packages/eve-bench/agent/tools/bash.ts new file mode 100644 index 0000000000..9979744b41 --- /dev/null +++ b/packages/eve-bench/agent/tools/bash.ts @@ -0,0 +1,91 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:os"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { taskEnv, taskRoot } from "../task-path.ts"; + +const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; +const DEFAULT_TIMEOUT_SECONDS = 120; +const MAX_TIMEOUT_SECONDS = 1800; + +export default defineTool({ + approval: never(), + description: `Execute a shell command in the task environment. Commands start in the task working directory. A command that runs longer than timeoutSeconds (default ${DEFAULT_TIMEOUT_SECONDS}) is stopped with its child processes and returns exit code 124 with the output captured so far; raise timeoutSeconds for long builds, and start servers in the background.`, + inputSchema: z.object({ + command: z.string().describe("The shell command to execute."), + timeoutSeconds: z + .number() + .int() + .positive() + .max(MAX_TIMEOUT_SECONDS) + .optional() + .describe(`Seconds before the command is stopped. Default ${DEFAULT_TIMEOUT_SECONDS}.`), + }), + execute({ command, timeoutSeconds = DEFAULT_TIMEOUT_SECONDS }, ctx) { + return new Promise((resolve) => { + // A separate process group lets a timeout stop everything the command started. + const child = spawn("sh", ["-c", command], { + cwd: taskRoot(), + env: taskEnv(), + detached: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const output = { stdout: "", stderr: "" }; + let bytes = 0; + let truncated = false; + let timedOut = false; + const collect = (stream: "stdout" | "stderr") => (chunk: Buffer) => { + const kept = chunk.subarray(0, Math.max(0, MAX_OUTPUT_BYTES - bytes)); + bytes += kept.length; + output[stream] += kept.toString("utf8"); + if (kept.length < chunk.length) truncated = true; + }; + child.stdout.on("data", collect("stdout")); + child.stderr.on("data", collect("stderr")); + const stop = () => { + try { + process.kill(-child.pid!, "SIGKILL"); + } catch { + // The group already exited. + } + }; + const timer = setTimeout(() => { + timedOut = true; + stop(); + }, timeoutSeconds * 1000); + ctx.abortSignal?.addEventListener("abort", stop, { once: true }); + let done = false; + const finish = (exitCode: number, note?: string) => { + if (done) return; + done = true; + clearTimeout(timer); + child.stdout.destroy(); + child.stderr.destroy(); + ctx.abortSignal?.removeEventListener("abort", stop); + const notes = [ + note, + timedOut ? `Command stopped after ${timeoutSeconds} seconds.` : undefined, + truncated ? `Output truncated at ${MAX_OUTPUT_BYTES} bytes.` : undefined, + ].filter(Boolean); + resolve({ + exitCode: timedOut ? 124 : exitCode, + stdout: output.stdout, + stderr: + notes.length > 0 ? `${output.stderr}\n[${notes.join(" ")}]`.trimStart() : output.stderr, + }); + }; + const exitCode = (code: number | null, signal: NodeJS.Signals | null) => + code ?? 128 + (signal ? (constants.signals[signal] ?? 0) : 0); + child.on("error", (error) => finish(1, error.message)); + child.on("close", (code, signal) => finish(exitCode(code, signal))); + // Background jobs can keep the pipes open after the shell exits; return with + // what was captured instead of waiting for them. + child.on("exit", (code, signal) => { + setTimeout(() => finish(exitCode(code, signal)), 250); + }); + }); + }, +}); diff --git a/packages/eve-bench/agent/tools/glob.ts b/packages/eve-bench/agent/tools/glob.ts new file mode 100644 index 0000000000..b9fc371db1 --- /dev/null +++ b/packages/eve-bench/agent/tools/glob.ts @@ -0,0 +1,22 @@ +import { glob } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskRoot } from "../task-path.ts"; + +export default defineTool({ + approval: never(), + description: "Find task-environment files whose paths match a glob pattern.", + inputSchema: z.object({ + pattern: z.string(), + path: z.string().optional().describe("Search root relative to the task working directory."), + }), + async execute({ pattern, path }) { + const cwd = path === undefined ? taskRoot() : resolveTaskPath(path); + const matches: string[] = []; + for await (const match of glob(pattern, { cwd })) matches.push(match); + return { matches: matches.sort() }; + }, +}); diff --git a/packages/eve-bench/agent/tools/grep.ts b/packages/eve-bench/agent/tools/grep.ts new file mode 100644 index 0000000000..6cbdc0bd74 --- /dev/null +++ b/packages/eve-bench/agent/tools/grep.ts @@ -0,0 +1,52 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskEnv, taskRoot } from "../task-path.ts"; + +const execFileAsync = promisify(execFile); + +export default defineTool({ + approval: never(), + description: + "Search task-environment text files with ripgrep and return matching lines with file names and line numbers.", + inputSchema: z.object({ + pattern: z.string().describe("Regular expression accepted by ripgrep."), + path: z + .string() + .optional() + .describe("File or directory relative to the task working directory."), + include: z.string().optional().describe("Optional glob restricting searched files."), + }), + async execute({ pattern, path, include }, ctx) { + const args = ["--line-number", "--no-heading", "--color=never"]; + if (include !== undefined) args.push("--glob", include); + args.push("--", pattern, path === undefined ? "." : resolveTaskPath(path)); + try { + const { stdout, stderr } = await execFileAsync("rg", args, { + cwd: taskRoot(), + env: taskEnv(), + maxBuffer: 10 * 1024 * 1024, + signal: ctx.abortSignal, + }); + return { exitCode: 0, stderr, stdout }; + } catch (error) { + const result = error as Error & { code?: number | string; stderr?: string; stdout?: string }; + // Otherwise a missing binary reads exactly like "no matches". + if (result.code === "ENOENT") { + throw new Error( + "ripgrep (rg) is not installed in this task environment; search with the bash tool (for example grep -rn) instead.", + ); + } + const exitCode = typeof result.code === "number" ? result.code : 1; + return { + exitCode, + stderr: result.stderr ?? (exitCode === 1 ? "" : result.message), + stdout: result.stdout ?? "", + }; + } + }, +}); diff --git a/packages/eve-bench/agent/tools/read_file.ts b/packages/eve-bench/agent/tools/read_file.ts new file mode 100644 index 0000000000..bcaa4ad7b2 --- /dev/null +++ b/packages/eve-bench/agent/tools/read_file.ts @@ -0,0 +1,27 @@ +import { readFile } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.ts"; + +export default defineTool({ + approval: never(), + description: + "Read a UTF-8 text file from the task environment, optionally selecting an inclusive 1-based line range.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + startLine: z.number().int().positive().optional(), + endLine: z.number().int().positive().optional(), + }), + async execute({ path, startLine, endLine }) { + if (startLine !== undefined && endLine !== undefined && startLine > endLine) { + throw new Error("startLine must not exceed endLine."); + } + const content = await readFile(resolveTaskPath(path), "utf8"); + if (startLine === undefined && endLine === undefined) return { content }; + const lines = content.split(/(?<=\n)/u); + return { content: lines.slice((startLine ?? 1) - 1, endLine).join("") }; + }, +}); diff --git a/packages/eve-bench/agent/tools/todo.ts b/packages/eve-bench/agent/tools/todo.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/todo.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_fetch.ts b/packages/eve-bench/agent/tools/web_fetch.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_fetch.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_search.ts b/packages/eve-bench/agent/tools/web_search.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_search.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/write_file.ts b/packages/eve-bench/agent/tools/write_file.ts new file mode 100644 index 0000000000..a31e52a732 --- /dev/null +++ b/packages/eve-bench/agent/tools/write_file.ts @@ -0,0 +1,24 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.ts"; + +export default defineTool({ + approval: never(), + description: + "Create or replace a UTF-8 text file in the task environment. Parent directories are created automatically.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + content: z.string(), + }), + async execute({ path, content }) { + const resolved = resolveTaskPath(path); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile(resolved, content, "utf8"); + return { bytesWritten: Buffer.byteLength(content), path: resolved }; + }, +}); diff --git a/packages/eve-bench/datasets/cohorts/smoke.json b/packages/eve-bench/datasets/cohorts/smoke.json new file mode 100644 index 0000000000..fe2d0a37f9 --- /dev/null +++ b/packages/eve-bench/datasets/cohorts/smoke.json @@ -0,0 +1,11 @@ +{ + "dataset": "terminal-bench-2.0", + "description": "Short, deterministic tasks with small images that exercise shell, file editing, and git tooling. Use for harness changes before running the full dataset.", + "tasks": [ + "fix-git", + "openssl-selfsigned-cert", + "regex-log", + "sqlite-db-truncate", + "log-summary-date-ranges" + ] +} diff --git a/packages/eve-bench/datasets/terminal-bench-2.0.json b/packages/eve-bench/datasets/terminal-bench-2.0.json new file mode 100644 index 0000000000..c13b5eaf35 --- /dev/null +++ b/packages/eve-bench/datasets/terminal-bench-2.0.json @@ -0,0 +1,97 @@ +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": [ + "adaptive-rejection-sampler", + "bn-fit-modify", + "break-filter-js-from-html", + "build-cython-ext", + "build-pmars", + "build-pov-ray", + "caffe-cifar-10", + "cancel-async-tasks", + "chess-best-move", + "circuit-fibsqrt", + "cobol-modernization", + "code-from-image", + "compile-compcert", + "configure-git-webserver", + "constraints-scheduling", + "count-dataset-tokens", + "crack-7z-hash", + "custom-memory-heap-crash", + "db-wal-recovery", + "distribution-search", + "dna-assembly", + "dna-insert", + "extract-elf", + "extract-moves-from-video", + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis", + "filter-js-from-html", + "financial-document-processor", + "fix-code-vulnerability", + "fix-git", + "fix-ocaml-gc", + "gcode-to-text", + "git-leak-recovery", + "git-multibranch", + "gpt2-codegolf", + "headless-terminal", + "hf-model-inference", + "install-windows-3.11", + "kv-store-grpc", + "large-scale-text-editing", + "largest-eigenval", + "llm-inference-batching-scheduler", + "log-summary-date-ranges", + "mailman", + "make-doom-for-mips", + "make-mips-interpreter", + "mcmc-sampling-stan", + "merge-diff-arc-agi-task", + "model-extraction-relu-logits", + "modernize-scientific-stack", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "nginx-request-logging", + "openssl-selfsigned-cert", + "overfull-hbox", + "password-recovery", + "path-tracing", + "path-tracing-reverse", + "polyglot-c-py", + "polyglot-rust-c", + "portfolio-optimization", + "protein-assembly", + "prove-plus-comm", + "pypi-server", + "pytorch-model-cli", + "pytorch-model-recovery", + "qemu-alpine-ssh", + "qemu-startup", + "query-optimize", + "raman-fitting", + "regex-chess", + "regex-log", + "reshard-c4-data", + "rstan-to-pystan", + "sam-cell-seg", + "sanitize-git-repo", + "schemelike-metacircular-eval", + "sparql-university", + "sqlite-db-truncate", + "sqlite-with-gcov", + "torch-pipeline-parallelism", + "torch-tensor-parallelism", + "train-fasttext", + "tune-mjcf", + "video-processing", + "vulnerable-secret", + "winning-avg-corewars", + "write-compressor" + ] +} diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json new file mode 100644 index 0000000000..eb052a852d --- /dev/null +++ b/packages/eve-bench/package.json @@ -0,0 +1,30 @@ +{ + "name": "@eve/bench", + "version": "0.0.0", + "private": true, + "description": "Terminal-Bench harness benchmarks for eve.", + "bin": { + "eve-bench": "./src/cli.ts" + }, + "type": "module", + "scripts": { + "bench": "node src/cli.ts", + "test": "pnpm run typecheck && pnpm run test:unit && pnpm run test:integration", + "test:unit": "node --test src/core/*.test.ts src/harnesses/*/*.test.ts", + "test:integration": "node test/posix-only.ts test/scenarios/*.test.ts src/tools/*.test.ts", + "test:scenario": "node test/posix-only.ts test/scenarios/*.test.ts src/tools/*.test.ts", + "test:stress": "node --test test/stress/*.test.ts", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "eve": "workspace:*", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + }, + "engines": { + "node": ">=24" + } +} diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts new file mode 100644 index 0000000000..b15c73de12 --- /dev/null +++ b/packages/eve-bench/src/cli.ts @@ -0,0 +1,204 @@ +#!/usr/bin/env node +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; + +import { syncDataset, taskDirs } from "./core/dataset.ts"; +import { readJobResult, runJob } from "./core/job.ts"; +import { diffJobs, formatDiff, formatReport, type ReportFormat } from "./core/report.ts"; +import { loadTask } from "./core/task.ts"; +import { + allLocks, + defaultJobName, + forwardedEnv, + jobDir, + lock, + paths, + selectHarness, + selectTasks, +} from "./options.ts"; + +const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve + + eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets + eve-bench tasks list [--cohort ] print task names + eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] + [--concurrency N] [--harness eve|oracle|pi|opencode|codex] [--job ] + [--eve local|] [--version ] + [--reasoning ] [--base-url ] [--format console|json|junit] + eve-bench prepare --harness --model [harness options] build/cache without model calls + eve-bench report [--format console|json|junit] [--fail-on-invalid] + eve-bench diff [--json] + +Jobs live in .generated/jobs/; re-running with the same --job resumes it. +Provider credentials are read from the environment and forwarded into the task container. +`; + +const [command, subcommand, ...rest] = process.argv.slice(2); +try { + switch (command) { + case "tasks": + await tasks(subcommand, rest); + break; + case "prepare": + case "run": + await run(subcommand === undefined ? [] : [subcommand, ...rest], command === "prepare"); + break; + case "report": + await report(subcommand, rest); + break; + case "diff": + await diff(subcommand, rest); + break; + default: + process.stdout.write(USAGE); + process.exitCode = command === undefined || command === "--help" || command === "-h" ? 0 : 2; + } +} catch (error) { + process.stderr.write(`eve-bench: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} + +async function tasks(sub: string | undefined, args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { dataset: { type: "string" }, cohort: { type: "string" }, json: { type: "boolean" } }, + }); + if (sub === "sync") { + const locks = values.dataset ? [await lock(values.dataset)] : await allLocks(); + for (const entry of locks) { + const dir = await syncDataset(join(paths.generatedRoot, "datasets"), entry); + process.stdout.write( + values.json + ? `${JSON.stringify({ dataset: entry.name, version: entry.version, dir })}\n` + : `${entry.name}@${entry.version} -> ${dir}\n`, + ); + } + return; + } + if (sub === "list") { + const selection = await selectTasks(values); + process.stdout.write( + values.json ? `${JSON.stringify(selection.tasks)}\n` : `${selection.tasks.join("\n")}\n`, + ); + return; + } + throw new Error(`unknown tasks subcommand: ${sub ?? "(none)"}\n${USAGE}`); +} + +async function run(args: string[], prepareOnly = false): Promise { + const { values } = parseArgs({ + args, + options: { + model: { type: "string" }, + cohort: { type: "string" }, + dataset: { type: "string" }, + task: { type: "string", multiple: true }, + "task-dir": { type: "string", multiple: true }, + attempts: { type: "string", default: "1" }, + concurrency: { type: "string", default: "4" }, + eve: { type: "string", default: "local" }, + harness: { type: "string", default: "eve" }, + version: { type: "string" }, + "base-url": { type: "string" }, + reasoning: { type: "string" }, + job: { type: "string" }, + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + }, + }); + const harness = selectHarness(values.harness, values.eve, { + ...values, + baseUrl: values["base-url"], + }); + const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); + if (!model) throw new Error("--model is required"); + if (prepareOnly) { + const bundle = await harness.prepare({ model, cacheDir: join(paths.generatedRoot, "cache") }); + process.stdout.write(`${JSON.stringify(bundle, null, 2)}\n`); + return; + } + const forwardEnv = forwardedEnv(harness); + const selection = await selectTasks({ ...values, taskDir: values["task-dir"] }); + const datasetTaskDirs = selection.tasks.length + ? await taskDirs( + await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock), + selection.tasks, + ) + : []; + const loaded = await Promise.all( + [...datasetTaskDirs, ...selection.taskDirs].map((dir) => loadTask(dir)), + ); + const name = values.job ?? defaultJobName(`${harness.name}-${model}`); + const controller = new AbortController(); + const stop = () => { + process.stderr.write("\neve-bench: aborting, cleaning up containers...\n"); + controller.abort(); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + const format = reportFormat(values.json ? "json" : values.format); + const result = await runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks: loaded, + harness, + model, + attempts: Number(values.attempts), + concurrency: Number(values.concurrency), + forwardEnv, + signal: controller.signal, + onLog: (message) => process.stderr.write(`${message}\n`), + onTrial: (trial, resumed) => { + if (format !== "console") return; + const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); + process.stderr.write( + `${resumed ? "resume" : "trial "} ${trial.task}#${trial.attempt} reward=${reward} agent=${trial.agent.status} verifier=${trial.verifier.status}${trial.invalid ? ` INVALID(${trial.invalid.phase}): ${trial.invalid.reason}` : ""}\n`, + ); + }, + }); + process.stdout.write(formatReport(result, format)); + if (controller.signal.aborted) process.exitCode = 130; +} + +async function report(job: string | undefined, args: string[]): Promise { + if (!job) throw new Error("report requires a job name"); + const { values } = parseArgs({ + args, + options: { + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + out: { type: "string" }, + "fail-on-invalid": { type: "boolean" }, + }, + }); + const result = await readJobResult(join(paths.generatedRoot, "jobs", job)); + const output = formatReport(result, reportFormat(values.json ? "json" : values.format)); + if (values.out) await writeFile(values.out, output); + else process.stdout.write(output); + const invalid = Object.values(result.summary.invalid).reduce((sum, count) => sum + count, 0); + if (values["fail-on-invalid"] && invalid > 0) { + process.stderr.write( + `${invalid} of ${result.summary.attempts} attempt(s) are invalid and did not measure the harness.\n`, + ); + process.exitCode = 1; + } +} + +async function diff(base: string | undefined, args: string[]): Promise { + const [candidate, ...restArgs] = args; + if (!base || !candidate) throw new Error("diff requires "); + const { values } = parseArgs({ args: restArgs, options: { json: { type: "boolean" } } }); + const deltas = diffJobs( + await readJobResult(jobDir(base)), + await readJobResult(jobDir(candidate)), + ); + process.stdout.write(values.json ? `${JSON.stringify(deltas, null, 2)}\n` : formatDiff(deltas)); +} + +function reportFormat(value: string | undefined): ReportFormat { + if (value === "console" || value === "json" || value === "junit") return value; + throw new Error(`unknown format: ${value}`); +} diff --git a/packages/eve-bench/src/core/dataset.test.ts b/packages/eve-bench/src/core/dataset.test.ts new file mode 100644 index 0000000000..c199c11519 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.test.ts @@ -0,0 +1,103 @@ +// syncDataset is intentionally untested because it performs network and git operations. + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { datasetDir, readCohort, readDatasetLock, taskDirs } from "./dataset.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-dataset-")); +after(() => rm(root, { recursive: true, force: true })); + +const VALID_LOCK = { + name: "terminal-bench", + version: "2.0", + gitUrl: "https://github.com/laude-institute/terminal-bench-2.git", + commit: "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + tasks: ["adaptive-rejection-sampler", "fix-git"], +}; + +test("parses a valid dataset lock", async () => { + const path = join(root, "valid-lock.json"); + await writeFile(path, JSON.stringify(VALID_LOCK)); + + const lock = await readDatasetLock(path); + + assert.deepEqual(lock, VALID_LOCK); +}); + +for (const field of ["name", "version", "gitUrl", "commit"] as const) { + test(`rejects a dataset lock missing ${field}`, async () => { + const path = join(root, `missing-${field}.json`); + const lock: Record = { ...VALID_LOCK }; + delete lock[field]; + await writeFile(path, JSON.stringify(lock)); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); + + test(`rejects a dataset lock with an empty ${field}`, async () => { + const path = join(root, `empty-${field}.json`); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, [field]: "" })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); +} + +test("rejects a dataset lock with an empty tasks array", async () => { + const path = join(root, "empty-lock-tasks.json"); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, tasks: [] })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("rejects a cohort missing dataset", async () => { + const path = join(root, "missing-cohort-dataset.json"); + await writeFile(path, JSON.stringify({ tasks: ["fix-git"] })); + + await assert.rejects(readCohort(path), { + message: `${path}: missing "dataset"`, + }); +}); + +test("rejects a cohort with an empty tasks array", async () => { + const path = join(root, "empty-cohort-tasks.json"); + await writeFile(path, JSON.stringify({ dataset: "terminal-bench-2.0", tasks: [] })); + + await assert.rejects(readCohort(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("builds the versioned dataset directory path", () => { + assert.equal(datasetDir(root, VALID_LOCK), join(root, "terminal-bench-2.0")); +}); + +test("resolves task directories containing task.toml", async () => { + const dir = join(root, "tasks-valid"); + await mkdir(join(dir, "task-a"), { recursive: true }); + await mkdir(join(dir, "nested", "task-b"), { recursive: true }); + await writeFile(join(dir, "task-a", "task.toml"), ""); + await writeFile(join(dir, "nested", "task-b", "task.toml"), ""); + + const dirs = await taskDirs(dir, ["task-a", "nested/task-b"]); + + assert.deepEqual(dirs, [join(dir, "task-a"), join(dir, "nested", "task-b")]); +}); + +test("rejects when any task directory lacks task.toml", async () => { + const dir = join(root, "tasks-missing-toml"); + await mkdir(join(dir, "present"), { recursive: true }); + await mkdir(join(dir, "missing"), { recursive: true }); + await writeFile(join(dir, "present", "task.toml"), ""); + + await assert.rejects(taskDirs(dir, ["present", "missing"]), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/dataset.ts b/packages/eve-bench/src/core/dataset.ts new file mode 100644 index 0000000000..beaa588dd5 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.ts @@ -0,0 +1,77 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface DatasetLock { + readonly name: string; + readonly version: string; + readonly gitUrl: string; + readonly commit: string; + readonly tasks: readonly string[]; +} + +export interface Cohort { + readonly dataset: string; + readonly description?: string; + readonly tasks: readonly string[]; +} + +export async function readDatasetLock(path: string): Promise { + const lock = JSON.parse(await readFile(path, "utf8")) as DatasetLock; + for (const key of ["name", "version", "gitUrl", "commit"] as const) { + if (typeof lock[key] !== "string" || lock[key].length === 0) { + throw new Error(`${path}: missing "${key}"`); + } + } + if (!Array.isArray(lock.tasks) || lock.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return lock; +} + +export async function readCohort(path: string): Promise { + const cohort = JSON.parse(await readFile(path, "utf8")) as Cohort; + if (typeof cohort.dataset !== "string") throw new Error(`${path}: missing "dataset"`); + if (!Array.isArray(cohort.tasks) || cohort.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return cohort; +} + +export function datasetDir(root: string, lock: DatasetLock): string { + return join(root, `${lock.name}-${lock.version}`); +} + +/** Fetches the pinned commit with a sparse, blobless checkout of the locked tasks. Idempotent. */ +export async function syncDataset(root: string, lock: DatasetLock): Promise { + const dir = datasetDir(root, lock); + if (await isSynced(dir, lock)) return dir; + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + const git = (...args: string[]) => + execFileAsync("git", ["-C", dir, ...args], { maxBuffer: 64 * 1024 * 1024 }); + await git("init", "--quiet"); + await git("remote", "add", "origin", lock.gitUrl); + await git("sparse-checkout", "set", "--cone", ...lock.tasks); + await git("fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", lock.commit); + await git("checkout", "--quiet", lock.commit); + return dir; +} + +async function isSynced(dir: string, lock: DatasetLock): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", dir, "rev-parse", "HEAD"]); + if (stdout.trim() !== lock.commit) return false; + const present = new Set(await readdir(dir)); + return lock.tasks.every((task) => present.has(task)); + } catch { + return false; + } +} + +export async function taskDirs(dir: string, tasks: readonly string[]): Promise { + const dirs = tasks.map((task) => join(dir, task)); + await Promise.all(dirs.map((task) => access(join(task, "task.toml")))); + return dirs; +} diff --git a/packages/eve-bench/src/core/docker.test.ts b/packages/eve-bench/src/core/docker.test.ts new file mode 100644 index 0000000000..e5221dc165 --- /dev/null +++ b/packages/eve-bench/src/core/docker.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { PassThrough } from "node:stream"; +import { finished } from "node:stream/promises"; + +import { createBoundedWritable } from "./docker.ts"; + +const TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; + +test("bounded writable keeps the head and appends one truncation line", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.write("abc"); + log.write("def"); + log.write("ignored"); + log.end(); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), `abcde${TRUNCATED_LINE}`); +}); + +test("bounded writable does not mark a log within the limit", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.end("abcde"); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), "abcde"); +}); diff --git a/packages/eve-bench/src/core/docker.ts b/packages/eve-bench/src/core/docker.ts new file mode 100644 index 0000000000..bb5937b0c2 --- /dev/null +++ b/packages/eve-bench/src/core/docker.ts @@ -0,0 +1,298 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { Writable } from "node:stream"; + +const CAPTURE_LIMIT_BYTES = 16 * 1024 * 1024; +const LOG_LIMIT_BYTES = 64 * 1024 * 1024; +const LOG_TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; + +export interface ProcessResult { + readonly exitCode: number | null; + readonly timedOut: boolean; + readonly aborted: boolean; + readonly stdout: string; + readonly stderr: string; +} + +export interface ExecOptions { + readonly cwd?: string; + readonly env?: Readonly>; + readonly timeoutMs?: number; + readonly signal?: AbortSignal; + readonly log?: Writable; +} + +export interface ContainerOptions { + readonly image: string; + readonly name: string; + readonly job: string; + readonly task: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly network: boolean; +} + +export interface RunnerContainer { + readonly id: string; + readonly name: string; + readonly job: string; + readonly task: string; + readonly createdAt: string; +} + +export class Container { + readonly name: string; + + private constructor(name: string) { + this.name = name; + } + + static async start(options: ContainerOptions, signal?: AbortSignal): Promise { + const args = [ + "run", + "--detach", + "--name", + options.name, + "--label", + "eve-bench=1", + "--label", + `eve-bench.job=${options.job}`, + "--label", + `eve-bench.task=${options.task}`, + "--entrypoint", + "sh", + ]; + if (options.cpus !== undefined) args.push("--cpus", String(options.cpus)); + if (options.memoryMb !== undefined) args.push("--memory", `${options.memoryMb}m`); + if (!options.network) args.push("--network", "none"); + args.push(options.image, "-c", "sleep infinity"); + await dockerOk(args, { signal }); + return new Container(options.name); + } + + /** + * Runs `command` through `sh -c` inside the container. A `docker exec` + * process is its own session and process-group leader, so recording its + * pid lets timeouts and aborts kill the whole group in-container; killing + * the host-side client alone would leave the workload running. + */ + async exec(command: string, options: ExecOptions = {}): Promise { + const pidFile = `/tmp/.eve-bench-${randomUUID()}.pid`; + const wrapped = `echo $$ > ${pidFile}; exec sh -c "$0"`; + const args = ["exec"]; + if (options.cwd) args.push("--workdir", options.cwd); + for (const [key, value] of Object.entries(options.env ?? {})) + args.push("--env", `${key}=${value}`); + args.push(this.name, "sh", "-c", wrapped, command); + const result = await docker(args, options); + if (result.timedOut || result.aborted) { + await docker( + [ + "exec", + this.name, + "sh", + "-c", + `pid=$(cat ${pidFile} 2>/dev/null); [ -n "$pid" ] && { kill -KILL -- -$pid 2>/dev/null; kill -KILL $pid 2>/dev/null; }; true`, + ], + { timeoutMs: 10_000 }, + ); + } + return result; + } + + async upload(hostPath: string, containerPath: string, signal?: AbortSignal): Promise { + await dockerOk(["cp", hostPath, `${this.name}:${containerPath}`], { signal }); + } + + async download(containerPath: string, hostPath: string, signal?: AbortSignal): Promise { + const result = await docker(["cp", `${this.name}:${containerPath}`, hostPath], { signal }); + return result.exitCode === 0; + } + + async remove(): Promise { + await docker(["rm", "--force", "--volumes", this.name], { timeoutMs: 60_000 }); + } +} + +export async function assertDockerAvailable(): Promise { + try { + await dockerOk(["version", "--format", "{{.Server.Version}}"], { timeoutMs: 10_000 }); + } catch { + throw new Error("Docker is unavailable; start the Docker daemon and retry."); + } +} + +export async function listRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const args = [ + "ps", + "-a", + "--filter", + "label=eve-bench=1", + ...(filter.job ? ["--filter", `label=eve-bench.job=${filter.job}`] : []), + "--format", + "{{json .}}", + ]; + const result = await dockerOk(args, { timeoutMs: 30_000 }); + return result.stdout + .split("\n") + .filter(Boolean) + .map((line) => { + const row = JSON.parse(line) as Record; + return { + id: row.ID ?? "", + name: row.Names ?? "", + job: labelValue(row.Labels, "eve-bench.job"), + task: labelValue(row.Labels, "eve-bench.task"), + createdAt: row.CreatedAt ?? "", + }; + }); +} + +export async function removeRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const containers = await listRunnerContainers(filter); + if (containers.length > 0) { + await dockerOk(["rm", "--force", "--volumes", ...containers.map((container) => container.id)], { + timeoutMs: 60_000, + }); + } + return containers; +} + +export function createBoundedLog(path: string): Writable { + return createBoundedWritable(createWriteStream(path), LOG_LIMIT_BYTES); +} + +export function createBoundedWritable(output: Writable, limitBytes: number): Writable { + let written = 0; + let truncated = false; + return new Writable({ + write(chunk: Buffer | string, encoding, callback) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); + const remaining = Math.max(0, limitBytes - written); + const kept = bytes.subarray(0, remaining); + written += kept.length; + if (kept.length < bytes.length) truncated = true; + if (kept.length === 0) callback(); + else output.write(kept, callback); + }, + final(callback) { + output.end(truncated ? LOG_TRUNCATED_LINE : undefined, callback); + }, + }); +} + +export async function imageExists(tag: string): Promise { + const result = await docker(["image", "inspect", tag], { timeoutMs: 30_000 }); + return result.exitCode === 0; +} + +export async function pullImage(image: string, options: ExecOptions = {}): Promise { + if (await imageExists(image)) return; + await dockerOk(["pull", image], options); +} + +export async function buildImage( + tag: string, + contextDir: string, + options: ExecOptions = {}, +): Promise { + if (await imageExists(tag)) return; + await dockerOk(["build", "--tag", tag, contextDir], options); +} + +export async function containerArch(container: Container): Promise<"x64" | "arm64"> { + const result = await container.exec("uname -m", { timeoutMs: 10_000 }); + const machine = result.stdout.trim(); + if (machine === "x86_64") return "x64"; + if (machine === "aarch64") return "arm64"; + throw new Error(`Unsupported container architecture: ${machine || result.stderr.trim()}`); +} + +async function dockerOk(args: readonly string[], options: ExecOptions): Promise { + const result = await docker(args, options); + if (result.exitCode !== 0) { + const reason = result.timedOut + ? "timed out" + : result.aborted + ? "aborted" + : `exit ${result.exitCode}`; + throw new Error(`docker ${args[0]} ${reason}: ${result.stderr.trim() || result.stdout.trim()}`); + } + return result; +} + +function docker(args: readonly string[], options: ExecOptions): Promise { + return new Promise((resolve, reject) => { + const child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let stdoutBytes = 0; + let stderrBytes = 0; + let timedOut = false; + let aborted = false; + const timer = + options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.timeoutMs); + const onAbort = () => { + aborted = true; + child.kill("SIGKILL"); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + child.stdout.on("data", (chunk: Buffer) => { + stdoutBytes = appendHead(stdout, stdoutBytes, chunk, CAPTURE_LIMIT_BYTES); + options.log?.write(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderrBytes = appendHead(stderr, stderrBytes, chunk, CAPTURE_LIMIT_BYTES); + options.log?.write(chunk); + }); + child.once("error", (error) => { + cleanup(); + reject(error); + }); + child.once("close", (code) => { + cleanup(); + resolve({ + exitCode: code, + timedOut, + aborted, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + function cleanup() { + if (timer) clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + } + }); +} + +function appendHead(chunks: Buffer[], size: number, chunk: Buffer, limit: number): number { + const kept = chunk.subarray(0, Math.max(0, limit - size)); + if (kept.length > 0) chunks.push(kept); + return size + kept.length; +} + +function labelValue(labels: string | undefined, key: string): string { + const prefix = `${key}=`; + return ( + labels + ?.split(",") + .find((label) => label.startsWith(prefix)) + ?.slice(prefix.length) ?? "" + ); +} diff --git a/packages/eve-bench/src/core/harness.ts b/packages/eve-bench/src/core/harness.ts new file mode 100644 index 0000000000..d36bf6b260 --- /dev/null +++ b/packages/eve-bench/src/core/harness.ts @@ -0,0 +1,45 @@ +import type { Usage } from "./result.ts"; + +/** + * A harness is anything that can be placed in a task container and told to + * solve an instruction. The runner uploads `bundleDir` to `/installed-agent`, + * then runs `command` through `sh -c` in the task working directory. + */ +export interface Harness { + readonly name: string; + /** Host environment names allowed to cross into this harness process. */ + readonly credentials?: readonly string[]; + /** Set when the harness makes no model calls, so missing usage does not invalidate a trial. */ + readonly modelFree?: boolean; + prepare(ctx: HarnessPrepareContext): Promise; + /** Optional per-task host paths uploaded into the install directory before `command` runs. */ + stage?(ctx: { readonly taskDir: string }): readonly string[]; + command(ctx: HarnessRunContext): string; + env(ctx: HarnessRunContext): Record; + /** + * Reads root-session usage from the downloaded agent logs on the host. It runs + * after timeouts too, so harnesses should log usage incrementally. + */ + readUsage?(agentLogsDir: string): Promise; +} + +export interface HarnessPrepareContext { + readonly model: string; + readonly cacheDir: string; +} + +export interface HarnessBundle { + readonly dir: string; + /** Architecture-specific host directories uploaded to `${installDir}/arch`. */ + readonly arch?: Readonly>; + /** Recorded in results so a run can be traced back to the exact harness build. */ + readonly provenance?: Record; +} + +export interface HarnessRunContext { + readonly model: string; + readonly installDir: string; + readonly instructionPath: string; + readonly taskWorkdir: string; + readonly logsDir: string; +} diff --git a/packages/eve-bench/src/core/id.test.ts b/packages/eve-bench/src/core/id.test.ts new file mode 100644 index 0000000000..cccf1ed792 --- /dev/null +++ b/packages/eve-bench/src/core/id.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertSafeId } from "./id.ts"; + +test("accepts bounded path- and Docker-safe identifiers", () => { + for (const value of ["job", "task-1", "a.b_c", `a${"x".repeat(127)}`]) { + assert.equal(assertSafeId(value, "job"), value); + } +}); + +test("rejects traversal, separators, labels, whitespace, and oversized identifiers", () => { + for (const value of ["", ".", "..", "../job", "a/b", "a:b", "two words", `a${"x".repeat(128)}`]) { + assert.throws(() => assertSafeId(value, "task"), /path- and Docker-safe/); + } +}); diff --git a/packages/eve-bench/src/core/id.ts b/packages/eve-bench/src/core/id.ts new file mode 100644 index 0000000000..54bc39ecc5 --- /dev/null +++ b/packages/eve-bench/src/core/id.ts @@ -0,0 +1,8 @@ +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; + +export function assertSafeId(value: string, kind: "job" | "task"): string { + if (!SAFE_ID.test(value) || value === "." || value === "..") { + throw new Error(`${kind} name must be 1-128 path- and Docker-safe characters`); + } + return value; +} diff --git a/packages/eve-bench/src/core/job-identity.test.ts b/packages/eve-bench/src/core/job-identity.test.ts new file mode 100644 index 0000000000..dd61bd45dd --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { assertJobIdentity, createJobIdentity, identityHash } from "./job-identity.ts"; + +const input = { + name: "resume", + harness: "eve@local", + model: "provider/model", + attempts: 2, + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a", "b"], + }, + tasks: [ + { name: "a", hash: "aaa" }, + { name: "b", hash: "bbb" }, + ], + provenance: { key: "bundle", build: { version: "1", sha: "abc" } }, +}; + +test("identity is stable across object key and selected task order, and JSON round trips", () => { + const expected = createJobIdentity(input); + const reordered = createJobIdentity({ + ...input, + tasks: [...input.tasks].reverse(), + provenance: { build: { sha: "abc", version: "1" }, key: "bundle" }, + }); + assert.deepEqual(reordered, expected); + assertJobIdentity(JSON.parse(JSON.stringify(expected)), reordered); + assert.equal( + identityHash({ b: [1, { z: 2, a: 3 }], a: true }), + identityHash({ a: true, b: [1, { a: 3, z: 2 }] }), + ); + assert.notEqual(identityHash([1, 2]), identityHash([2, 1])); + assert.notEqual(identityHash(["ab", "c"]), identityHash(["a", "bc"])); +}); + +for (const [field, changes] of [ + ["job", { name: "different" }], + ["harness", { harness: "oracle" }], + ["model", { model: "provider/other" }], + ["attempts", { attempts: 3 }], + ["tasks", { tasks: [{ name: "renamed", hash: "aaa" }, input.tasks[1]!] }], + ["tasks", { tasks: [{ name: "a", hash: "changed" }, input.tasks[1]!] }], + ["tasks", { tasks: input.tasks.slice(0, 1) }], + ["provenance", { provenance: { key: "bundle", build: { version: "1", sha: "changed" } } }], + ...(["name", "version", "gitUrl", "commit"] as const).map( + (key) => ["dataset", { dataset: { ...input.dataset, [key]: "changed" } }] as const, + ), +] as const) { + test(`changed ${field} refuses resume with an actionable error`, () => { + assert.throws( + () => + assertJobIdentity(createJobIdentity(input), createJobIdentity({ ...input, ...changes })), + new RegExp(`differs \\(.*${field}.*\\).*new job name`), + ); + }); +} + +test("unselected dataset task inventory does not change selected content identity", () => { + assert.deepEqual( + createJobIdentity({ ...input, dataset: { ...input.dataset, tasks: ["a", "b", "unselected"] } }), + createJobIdentity(input), + ); +}); + +test("legacy, unsupported, and incomplete identities fail closed", () => { + const expected = createJobIdentity(input); + for (const saved of [undefined, null, {}, "bad", { ...expected, version: 2 }, { version: 1 }]) { + assert.throws(() => assertJobIdentity(saved, expected), /new job name/); + } +}); + +test("duplicate task names cannot share trial directories", () => { + assert.throws( + () => createJobIdentity({ ...input, tasks: [input.tasks[0]!, input.tasks[0]!] }), + /unique/, + ); +}); + +test("absent provenance is stable and distinct from recorded provenance", () => { + const { provenance: _provenance, ...without } = input; + assert.deepEqual(createJobIdentity(without), createJobIdentity({ ...without, provenance: {} })); + assert.throws( + () => assertJobIdentity(createJobIdentity(without), createJobIdentity(input)), + /provenance/, + ); +}); diff --git a/packages/eve-bench/src/core/job-identity.ts b/packages/eve-bench/src/core/job-identity.ts new file mode 100644 index 0000000000..c4b0001dc0 --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.ts @@ -0,0 +1,129 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, opendir } from "node:fs/promises"; +import { isAbsolute, join, relative, sep } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import type { Task } from "./task.ts"; + +export interface JobIdentity { + readonly version: 1; + readonly job: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: string; + readonly tasks: string; + readonly provenance: string; +} + +export function identityHash(value: unknown): string { + const json = JSON.stringify(value, (_key, item: unknown) => { + if (item !== null && typeof item === "object" && !Array.isArray(item)) { + return Object.fromEntries( + Object.entries(item).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + } + return item; + }); + if (json === undefined) throw new Error("Job identity must be JSON serializable."); + return createHash("sha256").update(json).digest("hex"); +} + +export function createJobIdentity(input: { + readonly name: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: DatasetLock; + readonly tasks: readonly { readonly name: string; readonly hash: string }[]; + readonly provenance?: Record; +}): JobIdentity { + const { name, version, gitUrl, commit } = input.dataset; + const tasks = [...input.tasks].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + if (new Set(tasks.map((task) => task.name)).size !== tasks.length) { + throw new Error("Selected task names must be unique to resume safely."); + } + return { + version: 1, + job: input.name, + harness: input.harness, + model: input.model, + attempts: input.attempts, + dataset: identityHash({ name, version, gitUrl, commit }), + tasks: identityHash(tasks), + provenance: identityHash(input.provenance ?? {}), + }; +} + +export function assertJobIdentity(saved: unknown, expected: JobIdentity): void { + if (!saved || typeof saved !== "object" || !("version" in saved) || saved.version !== 1) { + throw new Error( + `Job ${expected.job} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + const fields = Object.keys(expected) as (keyof JobIdentity)[]; + const changed = fields.filter( + (key) => !(key in saved) || Reflect.get(saved, key) !== expected[key], + ); + if (changed.length > 0) { + throw new Error( + `Job ${expected.job} resume identity differs (${changed.join(", ")}). Use a new job name for changed inputs.`, + ); + } +} + +/** Hash only the selected tree, independent of its location, mtimes, and traversal order. */ +export async function hashTask(task: Task): Promise { + const dockerfileDir = relative(task.dir, task.environment.dockerfileDir); + if (isAbsolute(dockerfileDir) || dockerfileDir === ".." || dockerfileDir.startsWith(`..${sep}`)) { + throw new Error(`Task ${task.name}: environment must be inside the selected task tree.`); + } + const entries: { path: string; kind: string; executable: number; hash?: string }[] = []; + let bytes = 0; + let count = 0; + async function visit(path: string, depth: number): Promise { + if (depth > 64 || ++count > 100_000) { + throw new Error(`Task ${task.name}: identity scan exceeds 64 levels or 100,000 entries.`); + } + const absolute = join(task.dir, path); + const stat = await lstat(absolute); + // Do not follow links into unselected tasks (or hash a link but execute its changed target). + if (!stat.isDirectory() && !stat.isFile()) { + throw new Error( + `Task ${task.name}: identity requires regular files/directories, not ${path || "."}.`, + ); + } + const entry = { + path: path.split(sep).join("/"), + kind: stat.isDirectory() ? "directory" : "file", + executable: stat.mode & 0o111, + }; + if (stat.isDirectory()) { + entries.push(entry); + for await (const child of await opendir(absolute)) { + await visit(join(path, child.name), depth + 1); + } + } else { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(absolute)) { + bytes += chunk.length; + if (bytes > 1024 * 1024 * 1024) { + throw new Error(`Task ${task.name}: identity scan exceeds 1 GiB.`); + } + hash.update(chunk); + } + entries.push({ ...entry, hash: hash.digest("hex") }); + } + } + await visit("", 0); + entries.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + return identityHash({ + entries, + instruction: task.instruction, + agentTimeoutMs: task.agentTimeoutMs, + verifierTimeoutMs: task.verifierTimeoutMs, + buildTimeoutMs: task.buildTimeoutMs, + environment: { ...task.environment, dockerfileDir: dockerfileDir.split(sep).join("/") }, + }); +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts new file mode 100644 index 0000000000..ab35dd7f82 --- /dev/null +++ b/packages/eve-bench/src/core/job.ts @@ -0,0 +1,218 @@ +import { mkdir, open, readFile, readdir, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import { assertDockerAvailable, listRunnerContainers, removeRunnerContainers } from "./docker.ts"; +import type { Harness } from "./harness.ts"; +import { assertSafeId } from "./id.ts"; +import { assertJobIdentity, createJobIdentity, hashTask, identityHash } from "./job-identity.ts"; +import { assertTrialResult, summarize, type JobResult, type TrialResult } from "./result.ts"; +import type { Task } from "./task.ts"; +import { runTrial } from "./trial.ts"; + +export interface JobInput { + readonly name: string; + readonly dir: string; + readonly cacheDir: string; + readonly dataset: DatasetLock; + readonly tasks: readonly Task[]; + readonly harness: Harness; + readonly model: string; + readonly attempts: number; + readonly concurrency: number; + readonly forwardEnv: Readonly>; + readonly signal: AbortSignal; + readonly onLog?: (message: string) => void; + readonly onTrial?: (trial: TrialResult, resumed: boolean) => void; +} + +export async function runJob(input: JobInput): Promise { + assertSafeId(input.name, "job"); + for (const task of input.tasks) assertSafeId(task.name, "task"); + // Containers are labeled by name, so different output directories must share the lock. + const lockPath = join(tmpdir(), `eve-bench-job-${identityHash(input.name)}.lock`); + const lock = await open(lockPath, "wx", 0o600).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + throw new Error( + `Job ${input.name} is locked. Wait for the active run or use a new job name. If the owner crashed, confirm it has stopped before removing ${lockPath}.`, + ); + }); + try { + await lock.writeFile(`${JSON.stringify({ pid: process.pid, job: input.name })}\n`); + return await runLockedJob(input); + } finally { + await lock.close(); + await unlink(lockPath); + } +} + +async function runLockedJob(input: JobInput): Promise { + await mkdir(input.dir, { recursive: true }); + const savedIdentity = await readSavedIdentity(input.dir, input.name); + const taskHashes = []; + for (const task of input.tasks) taskHashes.push({ name: task.name, hash: await hashTask(task) }); + const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); + const identity = createJobIdentity({ + ...input, + harness: input.harness.name, + tasks: taskHashes, + provenance: bundle.provenance, + }); + if (savedIdentity !== undefined) assertJobIdentity(savedIdentity, identity); + + await assertDockerAvailable(); + const leftovers = await listRunnerContainers({ job: input.name }); + if (savedIdentity === undefined && leftovers.length > 0) { + throw new Error( + `Job ${input.name} has containers but no resume identity here. Use a new job name or resume from the original job directory.`, + ); + } + if (savedIdentity === undefined) { + await writeFile( + join(input.dir, "job.json"), + `${JSON.stringify( + { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + attempts: input.attempts, + tasks: input.tasks.map((task) => task.name), + bundle: bundle.provenance ?? {}, + identity, + startedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + { flag: "wx" }, + ); + } + if (leftovers.length > 0) { + await removeRunnerContainers({ job: input.name }); + for (const container of leftovers) { + input.onLog?.(`Removed leftover container ${container.name} from job ${input.name}`); + } + } + + const queue = input.tasks.flatMap((task) => + Array.from({ length: input.attempts }, (_, index) => ({ task, attempt: index + 1 })), + ); + const trials: TrialResult[] = []; + const workers = Array.from({ length: Math.max(1, input.concurrency) }, async () => { + for (;;) { + const next = queue.shift(); + if (!next || input.signal.aborted) return; + const dir = join(input.dir, next.task.name, String(next.attempt)); + const existing = await readTrial(dir, { + task: next.task.name, + attempt: next.attempt, + harness: input.harness.name, + model: input.model, + }); + if (existing) { + trials.push(existing); + input.onTrial?.(existing, true); + continue; + } + const trial = await runTrial({ + task: next.task, + attempt: next.attempt, + harness: input.harness, + bundle, + model: input.model, + forwardEnv: input.forwardEnv, + job: input.name, + dir, + signal: input.signal, + }); + trials.push(trial); + input.onTrial?.(trial, false); + } + }); + const outcomes = await Promise.allSettled(workers); + if (input.signal.aborted) await removeRunnerContainers({ job: input.name }); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } + + trials.sort((a, b) => a.task.localeCompare(b.task) || a.attempt - b.attempt); + const result: JobResult = { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + trials, + summary: summarize(trials), + }; + await writeFile(join(input.dir, "result.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +export async function readJobResult(dir: string): Promise { + return JSON.parse(await readFile(join(dir, "result.json"), "utf8")) as JobResult; +} + +async function readSavedIdentity(dir: string, name: string): Promise { + let text: string; + try { + text = await readFile(join(dir, "job.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if ((await readdir(dir)).length === 0) return undefined; + throw new Error( + `Job ${name} has existing files but no job.json. Use a new job name to run safely.`, + ); + } + let saved; + try { + saved = JSON.parse(text); + } catch { + throw new Error(`Job ${name} has invalid job.json. Use a new job name to run safely.`); + } + if (saved?.identity?.version !== 1) { + throw new Error( + `Job ${name} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + return saved.identity; +} + +async function readTrial( + dir: string, + expected: Pick, +): Promise { + let text: string; + try { + text = await readFile(join(dir, "trial.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error( + `Trial ${expected.task}#${expected.attempt} has invalid JSON; remove the job or use a new job name.`, + ); + } + assertTrialResult(parsed); + for (const key of ["task", "attempt", "harness", "model"] as const) { + if (parsed[key] !== expected[key]) { + throw new Error( + `Trial ${expected.task}#${expected.attempt} has mismatched ${key}; use a new job name.`, + ); + } + } + return parsed; +} diff --git a/packages/eve-bench/src/core/report.test.ts b/packages/eve-bench/src/core/report.test.ts new file mode 100644 index 0000000000..5e03733735 --- /dev/null +++ b/packages/eve-bench/src/core/report.test.ts @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { diffJobs, formatDiff, formatReport } from "./report.ts"; +import { classifyTrial, summarize, type JobResult, type TrialResult } from "./result.ts"; + +const USED = { inputTokens: 100, outputTokens: 10, cachedTokens: 0, costUsd: null }; + +function trial( + task: string, + attempt: number, + reward: number | null, + extra: Partial = {}, +): TrialResult { + return { + task, + attempt, + harness: "eve@local", + model: "test/model", + reward, + agent: { status: "completed", exitCode: 0, durationMs: 1000 }, + verifier: { status: "completed", exitCode: 0, durationMs: 500 }, + usage: USED, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.500Z", + ...extra, + }; +} + +function job(name: string, trials: TrialResult[]): JobResult { + return { + job: name, + dataset: { name: "terminal-bench", version: "2.0", commit: "abc" }, + harness: "eve@local", + model: "test/model", + trials, + summary: summarize(trials), + }; +} + +const harnessInvalid = { phase: "harness", reason: "no model call completed" } as const; + +test("classify separates environment, harness, and verifier failures from scored outcomes", () => { + const base = trial("a", 1, 0); + assert.equal(classifyTrial(base, false), undefined); + assert.equal( + classifyTrial({ ...base, agent: { ...base.agent, status: "timeout" } }, false), + undefined, + ); + + const setup = classifyTrial( + { + ...base, + error: "docker pull failed", + agent: { status: "skipped", exitCode: null, durationMs: 0 }, + }, + false, + ); + assert.equal(setup?.phase, "environment"); + assert.match(setup!.reason, /before the harness started: docker pull failed/); + assert.equal( + classifyTrial({ ...base, error: "could not download /logs/agent" }, false)?.phase, + "environment", + ); + + // A 401 or broken adapter leaves zero usage; a missing log leaves none. + const zero = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: null }; + assert.equal(classifyTrial({ ...base, usage: zero }, false)?.phase, "harness"); + assert.equal(classifyTrial({ ...base, usage: undefined, reward: 1 }, false)?.phase, "harness"); + assert.equal(classifyTrial({ ...base, usage: undefined }, true), undefined); + + const verifier = classifyTrial( + { ...base, reward: null, verifier: { status: "timeout", exitCode: null, durationMs: 1 } }, + false, + ); + assert.deepEqual(verifier, { + phase: "verifier", + reason: "verifier timed out without writing a reward", + }); +}); + +test("summarize scores only valid attempts and counts invalid ones by phase", () => { + const summary = summarize([ + trial("a", 1, 1), + trial("a", 2, 0), + trial("b", 1, 0, { invalid: harnessInvalid }), + trial("b", 2, null, { invalid: { phase: "verifier", reason: "verifier wrote no reward" } }), + ]); + assert.deepEqual(summary, { + tasks: 2, + attempts: 4, + scored: 2, + invalid: { environment: 0, harness: 1, verifier: 1 }, + resolved: 1, + meanReward: 0.5, + }); + assert.equal(summarize([trial("a", 1, 0, { invalid: harnessInvalid })]).meanReward, null); +}); + +test("junit reports invalid attempts as errors, not failures", () => { + const output = formatReport( + job("j", [trial("a", 1, 1), trial("b", 1, 0), trial("c", 1, 0, { invalid: harnessInvalid })]), + "junit", + ); + assert.match(output, /tests="3" failures="1" errors="1"/u); + assert.match(output, //u); + assert.match(output, //u); + assert.match(output, //u); +}); + +test("console report lists every invalid attempt with its reason", () => { + const output = formatReport( + job("j", [trial("a", 1, 1), trial("a", 2, 0, { invalid: harnessInvalid })]), + "console", + ); + assert.match(output, /1\.00 x\s+#2 invalid\(harness\)/u); + assert.match(output, /scored 1 {2}resolved 1 {2}mean reward 1\.000/u); + assert.match(output, /invalid attempts excluded from scoring: harness 1/u); + assert.match(output, /a #2 \[harness\] no model call completed/u); +}); + +test("diff compares scored attempts and reports invalid counts", () => { + const base = job("base", [trial("a", 1, 0), trial("a", 2, 1), trial("b", 1, 1)]); + const candidate = job("cand", [ + trial("a", 1, 1), + trial("a", 2, 0, { invalid: harnessInvalid }), + trial("c", 1, 0), + ]); + const deltas = diffJobs(base, candidate); + assert.deepEqual(deltas, [ + { task: "a", base: 0.5, candidate: 1, delta: 0.5, baseInvalid: 0, candidateInvalid: 1 }, + { task: "b", base: 1, candidate: null, delta: null, baseInvalid: 0, candidateInvalid: 0 }, + { task: "c", base: null, candidate: 0, delta: null, baseInvalid: 0, candidateInvalid: 0 }, + ]); + const text = formatDiff(deltas); + assert.match(text, /1 task\(s\) changed, net \+0.500/u); + assert.match(text, /invalid 0 -> 1/u); + assert.match(text, /1 invalid attempt\(s\) excluded/u); +}); diff --git a/packages/eve-bench/src/core/report.ts b/packages/eve-bench/src/core/report.ts new file mode 100644 index 0000000000..79e3ccb10a --- /dev/null +++ b/packages/eve-bench/src/core/report.ts @@ -0,0 +1,170 @@ +import type { JobResult, TrialResult } from "./result.ts"; + +export type ReportFormat = "console" | "json" | "junit"; + +export function formatReport(result: JobResult, format: ReportFormat): string { + switch (format) { + case "json": + return `${JSON.stringify(result, null, 2)}\n`; + case "junit": + return junit(result); + case "console": + return console_(result); + } +} + +function console_(result: JobResult): string { + const lines = [ + `${result.job} ${result.harness} / ${result.model} ${result.dataset.name}@${result.dataset.version}`, + "", + ]; + const byTask = groupByTask(result.trials); + for (const [task, trials] of byTask) { + const rewards = trials.map((trial) => + trial.invalid ? "x" : trial.reward === null ? "-" : trial.reward.toFixed(2), + ); + const invalid = trials.filter((trial) => trial.invalid); + const timeouts = trials.filter((trial) => !trial.invalid && trial.agent.status === "timeout"); + const notes = [ + ...invalid.map((trial) => `#${trial.attempt} invalid(${trial.invalid!.phase})`), + ...(timeouts.length > 0 ? [`${timeouts.length} agent timeout(s)`] : []), + ]; + lines.push(`${task.padEnd(44)} ${rewards.join(" ").padEnd(16)} ${notes.join(", ")}`.trimEnd()); + } + const { summary } = result; + const invalid = Object.entries(summary.invalid) + .filter(([, count]) => count > 0) + .map(([phase, count]) => `${phase} ${count}`); + const mean = summary.meanReward === null ? "-" : summary.meanReward.toFixed(3); + lines.push( + "", + `tasks ${summary.tasks} attempts ${summary.attempts} scored ${summary.scored} resolved ${summary.resolved} mean reward ${mean}`, + ); + if (invalid.length > 0) { + lines.push(`invalid attempts excluded from scoring: ${invalid.join(", ")}`); + for (const trial of result.trials.filter((trial) => trial.invalid)) { + lines.push( + ` ${trial.task} #${trial.attempt} [${trial.invalid!.phase}] ${trial.invalid!.reason}`, + ); + } + } + return `${lines.join("\n")}\n`; +} + +function junit(result: JobResult): string { + const invalid = result.trials.filter((trial) => trial.invalid).length; + const failures = result.trials.filter( + (trial) => !trial.invalid && (trial.reward ?? 0) < 1, + ).length; + const cases = result.trials.map((trial) => { + const time = ((trial.agent.durationMs + trial.verifier.durationMs) / 1000).toFixed(3); + const name = escape(`${trial.task} #${trial.attempt}`); + if (trial.invalid) { + const message = escape(`invalid (${trial.invalid.phase}): ${trial.invalid.reason}`); + return ` \n \n `; + } + if ((trial.reward ?? 0) >= 1) return ` `; + const message = escape( + `reward ${trial.reward ?? "none"}; agent ${trial.agent.status}; verifier ${trial.verifier.status}`, + ); + return ` \n \n `; + }); + return [ + ``, + ``, + ` `, + ...cases, + ` `, + ``, + "", + ].join("\n"); +} + +export interface TaskDelta { + readonly task: string; + /** Mean reward over scored attempts; null when the job has none for this task. */ + readonly base: number | null; + readonly candidate: number | null; + readonly delta: number | null; + readonly baseInvalid: number; + readonly candidateInvalid: number; +} + +export function diffJobs(base: JobResult, candidate: JobResult): TaskDelta[] { + const baseStats = statsByTask(base.trials); + const candidateStats = statsByTask(candidate.trials); + const tasks = [...new Set([...baseStats.keys(), ...candidateStats.keys()])].sort(); + return tasks.map((task) => { + const b = baseStats.get(task); + const c = candidateStats.get(task); + const baseMean = b?.mean ?? null; + const candidateMean = c?.mean ?? null; + return { + task, + base: baseMean, + candidate: candidateMean, + delta: baseMean === null || candidateMean === null ? null : candidateMean - baseMean, + baseInvalid: b?.invalid ?? 0, + candidateInvalid: c?.invalid ?? 0, + }; + }); +} + +export function formatDiff(deltas: readonly TaskDelta[]): string { + const lines = deltas.map((row) => { + const fmt = (value: number | null) => + value === null ? " - " : value.toFixed(3).padStart(6); + const delta = + row.delta === null + ? "" + : row.delta === 0 + ? "" + : row.delta > 0 + ? `+${row.delta.toFixed(3)}` + : row.delta.toFixed(3); + const invalid = + row.baseInvalid + row.candidateInvalid > 0 + ? `invalid ${row.baseInvalid} -> ${row.candidateInvalid}` + : ""; + return `${row.task.padEnd(44)} ${fmt(row.base)} -> ${fmt(row.candidate)} ${delta.padEnd(7)} ${invalid}`.trimEnd(); + }); + const changed = deltas.filter((row) => row.delta !== null && row.delta !== 0); + const net = changed.reduce((sum, row) => sum + (row.delta ?? 0), 0); + const invalid = deltas.reduce((sum, row) => sum + row.baseInvalid + row.candidateInvalid, 0); + lines.push("", `${changed.length} task(s) changed, net ${net >= 0 ? "+" : ""}${net.toFixed(3)}`); + if (invalid > 0) { + lines.push(`${invalid} invalid attempt(s) excluded; compare only after resolving them`); + } + return `${lines.join("\n")}\n`; +} + +function groupByTask(trials: readonly TrialResult[]): Map { + const groups = new Map(); + for (const trial of trials) { + const list = groups.get(trial.task) ?? []; + list.push(trial); + groups.set(trial.task, list); + } + return groups; +} + +function statsByTask( + trials: readonly TrialResult[], +): Map { + const stats = new Map(); + for (const [task, group] of groupByTask(trials)) { + const scored = group.filter((trial) => !trial.invalid); + stats.set(task, { + mean: + scored.length === 0 + ? null + : scored.reduce((sum, trial) => sum + (trial.reward ?? 0), 0) / scored.length, + invalid: group.length - scored.length, + }); + } + return stats; +} + +function escape(text: string): string { + return text.replaceAll("&", "&").replaceAll("<", "<").replaceAll('"', """); +} diff --git a/packages/eve-bench/src/core/result.ts b/packages/eve-bench/src/core/result.ts new file mode 100644 index 0000000000..62933ac171 --- /dev/null +++ b/packages/eve-bench/src/core/result.ts @@ -0,0 +1,161 @@ +export type StepStatus = "completed" | "timeout" | "failed" | "skipped"; + +export interface StepResult { + readonly status: StepStatus; + readonly exitCode: number | null; + readonly durationMs: number; +} + +/** + * Root-session model usage as reported by the harness. `inputTokens` includes + * cache reads and `outputTokens` includes reasoning, so harnesses that report + * those separately are normalized to the same totals. + */ +export interface Usage { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cachedTokens: number; + /** Null when the harness reported no cost (for example, no pricing data for a custom provider). */ + readonly costUsd: number | null; +} + +/** + * Where a trial broke before it could measure the harness: + * - `environment`: runner-owned work (image, container, uploads, log download) failed. + * - `harness`: the harness ran but completed no model call, so its adapter, + * credentials, or launch are broken rather than its task-solving. + * - `verifier`: the verifier produced no reward. + */ +export type InvalidPhase = "environment" | "harness" | "verifier"; + +export interface InvalidTrial { + readonly phase: InvalidPhase; + readonly reason: string; +} + +export interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: Usage; + readonly error?: string; + /** Present when the trial is excluded from scoring. */ + readonly invalid?: InvalidTrial; + readonly startedAt: string; + readonly finishedAt: string; +} + +export interface JobResult { + readonly job: string; + readonly dataset: { readonly name: string; readonly version: string; readonly commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: JobSummary; +} + +export interface JobSummary { + readonly tasks: number; + readonly attempts: number; + /** Attempts that measured the harness; invalid attempts are excluded. */ + readonly scored: number; + readonly invalid: Readonly>; + readonly resolved: number; + /** Mean reward over scored attempts; null when none were scored. */ + readonly meanReward: number | null; +} + +const PHASES: readonly InvalidPhase[] = ["environment", "harness", "verifier"]; + +export function classifyTrial( + trial: Pick, + modelFree: boolean, +): InvalidTrial | undefined { + if (trial.error !== undefined) { + return { + phase: "environment", + reason: + trial.agent.status === "skipped" + ? `trial setup failed before the harness started: ${trial.error}` + : `runner failed after the harness started: ${trial.error}`, + }; + } + if (!modelFree && !reachedModel(trial.usage)) { + return { + phase: "harness", + reason: trial.usage + ? "harness reported zero token usage; no model call completed" + : "harness reported no model usage; no model call completed", + }; + } + if (trial.reward === null) { + return { + phase: "verifier", + reason: + trial.verifier.status === "timeout" + ? "verifier timed out without writing a reward" + : "verifier wrote no reward", + }; + } + return undefined; +} + +function reachedModel(usage: Usage | undefined): boolean { + return usage !== undefined && (usage.inputTokens > 0 || usage.outputTokens > 0); +} + +export function assertTrialResult(value: unknown): asserts value is TrialResult { + if (!value || typeof value !== "object") throw new Error("trial record must be an object"); + const trial = value as Partial; + const step = (value: unknown) => { + const item = value as Partial | undefined; + return ( + item !== undefined && + ["completed", "timeout", "failed", "skipped"].includes(item.status ?? "") && + (item.exitCode === null || Number.isInteger(item.exitCode)) && + typeof item.durationMs === "number" && + Number.isFinite(item.durationMs) + ); + }; + const invalid = trial.invalid as Partial | undefined; + if ( + typeof trial.task !== "string" || + !Number.isInteger(trial.attempt) || + typeof trial.harness !== "string" || + typeof trial.model !== "string" || + !( + trial.reward === null || + (typeof trial.reward === "number" && Number.isFinite(trial.reward)) + ) || + !step(trial.agent) || + !step(trial.verifier) || + (invalid !== undefined && + (!PHASES.includes(invalid.phase as InvalidPhase) || typeof invalid.reason !== "string")) || + typeof trial.startedAt !== "string" || + typeof trial.finishedAt !== "string" + ) { + throw new Error("trial record has an invalid shape"); + } +} + +export function summarize(trials: readonly TrialResult[]): JobSummary { + const tasks = new Set(trials.map((trial) => trial.task)).size; + const invalid = { environment: 0, harness: 0, verifier: 0 }; + const scored: number[] = []; + for (const trial of trials) { + if (trial.invalid) invalid[trial.invalid.phase]++; + else scored.push(trial.reward ?? 0); + } + return { + tasks, + attempts: trials.length, + scored: scored.length, + invalid, + resolved: scored.filter((reward) => reward >= 1).length, + meanReward: scored.length === 0 ? null : scored.reduce((a, b) => a + b, 0) / scored.length, + }; +} diff --git a/packages/eve-bench/src/core/task.test.ts b/packages/eve-bench/src/core/task.test.ts new file mode 100644 index 0000000000..7767b05c41 --- /dev/null +++ b/packages/eve-bench/src/core/task.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { loadTask } from "./task.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-task-")); +after(() => rm(root, { recursive: true, force: true })); + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[agent] +timeout_sec = 1.234 + +[verifier] +timeout_sec = 2.789 + +[environment] +build_timeout_sec = 3.456 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1.5 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +async function writeTask(dir: string, toml = TASK_TOML): Promise { + await mkdir(join(dir, "tests"), { recursive: true }); + await writeFile(join(dir, "task.toml"), toml); + await writeFile(join(dir, "instruction.md"), "Solve the task.\n"); + await writeFile(join(dir, "tests", "test.sh"), "#!/bin/sh\n"); +} + +test("loads Terminal-Bench task fields and converts timeout seconds to milliseconds", async () => { + const dir = join(root, "complete"); + await writeTask(dir); + + const task = await loadTask(dir); + + assert.equal(task.name, "adaptive-rejection-sampler"); + assert.equal(task.dir, dir); + assert.equal(task.instruction, "Solve the task.\n"); + assert.equal(task.agentTimeoutMs, 1234); + assert.equal(task.verifierTimeoutMs, 2789); + assert.equal(task.buildTimeoutMs, 3456); + assert.equal(task.environment.dockerImage, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(task.environment.cpus, 1.5); + assert.equal(task.environment.memoryMb, 2048); + assert.equal(task.environment.allowInternet, true); +}); + +test('converts older-schema memory = "2G" to 2048 MB', async () => { + const dir = join(root, "memory-2g"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 2048); +}); + +test('converts older-schema memory = "512M" to 512 MB', async () => { + const dir = join(root, "memory-512m"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "512M"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 512); +}); + +test("prefers memory_mb when both memory fields are present", async () => { + const dir = join(root, "memory-precedence"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory_mb = 768\nmemory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 768); +}); + +test("uses 15-minute defaults when agent and verifier timeouts are missing", async () => { + const dir = join(root, "default-timeouts"); + const toml = TASK_TOML.replace("[agent]\ntimeout_sec = 1.234\n\n", "").replace( + "[verifier]\ntimeout_sec = 2.789\n\n", + "", + ); + await writeTask(dir, toml); + + const task = await loadTask(dir); + + assert.equal(task.agentTimeoutMs, 900_000); + assert.equal(task.verifierTimeoutMs, 900_000); +}); + +test("allows internet when allow_internet is absent", async () => { + const dir = join(root, "internet-default"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true\n", "")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, true); +}); + +test("disallows internet when allow_internet is false", async () => { + const dir = join(root, "internet-false"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true", "allow_internet = false")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, false); +}); + +test("rejects a task with neither docker_image nor environment/Dockerfile", async () => { + const dir = join(root, "missing-image"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); + +test("loads environment/Dockerfile when docker_image is absent", async () => { + const dir = join(root, "dockerfile"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + await mkdir(join(dir, "environment")); + await writeFile(join(dir, "environment", "Dockerfile"), "FROM scratch\n"); + + const task = await loadTask(dir); + + assert.equal(task.environment.dockerImage, undefined); + assert.equal(task.environment.dockerfileDir, join(dir, "environment")); +}); + +test("rejects a task missing tests/test.sh", async () => { + const dir = join(root, "missing-test"); + await writeTask(dir); + await rm(join(dir, "tests", "test.sh")); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/task.ts b/packages/eve-bench/src/core/task.ts new file mode 100644 index 0000000000..d2a2b61859 --- /dev/null +++ b/packages/eve-bench/src/core/task.ts @@ -0,0 +1,83 @@ +import { access, readFile } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import { assertSafeId } from "./id.ts"; +import { parseToml, type TomlTable } from "./toml.ts"; + +export interface Task { + readonly name: string; + readonly dir: string; + readonly instruction: string; + readonly agentTimeoutMs: number; + readonly verifierTimeoutMs: number; + readonly buildTimeoutMs: number; + readonly environment: { + readonly dockerImage?: string; + readonly dockerfileDir: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly allowInternet: boolean; + }; +} + +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; + +export async function loadTask(dir: string): Promise { + const toml = parseToml(await readFile(join(dir, "task.toml"), "utf8")); + const instruction = await readFile(join(dir, "instruction.md"), "utf8"); + await access(join(dir, "tests", "test.sh")); + const task = table(toml, "task"); + const agent = table(toml, "agent"); + const verifier = table(toml, "verifier"); + const environment = table(toml, "environment"); + const dockerImage = str(environment, "docker_image"); + const dockerfileDir = join(dir, "environment"); + if (dockerImage === undefined) await access(join(dockerfileDir, "Dockerfile")); + return { + name: assertSafeId(shortName(str(task, "name") ?? basename(dir)), "task"), + dir, + instruction, + agentTimeoutMs: seconds(agent, "timeout_sec"), + verifierTimeoutMs: seconds(verifier, "timeout_sec"), + buildTimeoutMs: seconds(environment, "build_timeout_sec"), + environment: { + dockerImage, + dockerfileDir, + cpus: num(environment, "cpus"), + memoryMb: num(environment, "memory_mb") ?? memoryMb(str(environment, "memory")), + allowInternet: environment.allow_internet !== false, + }, + }; +} + +function shortName(name: string): string { + return name.includes("/") ? name.slice(name.lastIndexOf("/") + 1) : name; +} + +function table(root: TomlTable, key: string): TomlTable { + const value = root[key]; + return typeof value === "object" && value !== null && !Array.isArray(value) ? value : {}; +} + +function str(table: TomlTable, key: string): string | undefined { + const value = table[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function num(table: TomlTable, key: string): number | undefined { + const value = table[key]; + return typeof value === "number" ? value : undefined; +} + +/** Older tasks declare `memory = "2G"` instead of `memory_mb`. */ +function memoryMb(value: string | undefined): number | undefined { + const match = value?.match(/^(\d+(?:\.\d+)?)\s*([GgMm])/u); + if (!match) return undefined; + const amount = Number(match[1]); + return Math.round(match[2]!.toLowerCase() === "g" ? amount * 1024 : amount); +} + +function seconds(table: TomlTable, key: string): number { + const value = num(table, key); + return value === undefined ? DEFAULT_TIMEOUT_MS : Math.round(value * 1000); +} diff --git a/packages/eve-bench/src/core/toml.test.ts b/packages/eve-bench/src/core/toml.test.ts new file mode 100644 index 0000000000..a2d31ddc6b --- /dev/null +++ b/packages/eve-bench/src/core/toml.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { parseToml } from "./toml.ts"; + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] # trailing comment +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[metadata] +expert_time_estimate_min = 180.0 + +[verifier] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +test("parses the Terminal-Bench task.toml subset", () => { + const parsed = parseToml(TASK_TOML); + assert.equal(parsed.schema_version, "1.1"); + assert.deepEqual(parsed.artifacts, []); + const task = parsed.task as Record; + assert.equal(task.name, "terminal-bench/adaptive-rejection-sampler"); + assert.deepEqual(task.keywords, ["applied-statistics", "simulation"]); + assert.deepEqual(task.authors, [{ name: "jvpoulos", email: "poulos@berkeley.edu" }]); + const environment = parsed.environment as Record; + assert.equal(environment.docker_image, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(environment.cpus, 1); + assert.equal(environment.allow_internet, true); + assert.deepEqual(environment.env, {}); + assert.deepEqual((parsed.verifier as Record).timeout_sec, 900); +}); + +test("handles dotted tables, escapes, and nested arrays", () => { + const parsed = parseToml(` +[a.b] +text = "line\\nbreak \\"quoted\\" # not a comment" +nested = [[1, 2], ["x, y"], []] +literal = 'C:\\path' +`); + const table = (parsed.a as Record).b as Record; + assert.equal(table.text, 'line\nbreak "quoted" # not a comment'); + assert.deepEqual(table.nested, [[1, 2], ["x, y"], []]); + assert.equal(table.literal, "C:\\path"); +}); + +test("rejects unsupported syntax with the line number", () => { + assert.throws(() => parseToml(`ok = 1\nbad = { inline = true }`), /line 2: inline tables/u); + assert.throws(() => parseToml(`ok = 1\nbad = 2024-01-01`), /line 2: unsupported value/u); + assert.throws(() => parseToml(`no equals sign`), /line 1: expected key = value/u); +}); diff --git a/packages/eve-bench/src/core/toml.ts b/packages/eve-bench/src/core/toml.ts new file mode 100644 index 0000000000..a17f39f06d --- /dev/null +++ b/packages/eve-bench/src/core/toml.ts @@ -0,0 +1,159 @@ +/** + * Minimal TOML reader for Terminal-Bench `task.toml` files. Supports tables, + * array-of-tables headers, bare/quoted keys, strings, numbers, booleans, and + * arrays. Anything else fails with the offending line so a dataset change + * surfaces immediately instead of being silently misread. + */ +export type TomlValue = string | number | boolean | TomlValue[] | TomlTable; +export interface TomlTable { + [key: string]: TomlValue; +} + +export function parseToml(source: string): TomlTable { + const root: TomlTable = {}; + let current = root; + const lines = source.split(/\r?\n/u); + for (let index = 0; index < lines.length; index++) { + const line = stripComment(lines[index] ?? "").trim(); + if (line.length === 0) continue; + const lineNumber = index + 1; + if (line.startsWith("[[") && line.endsWith("]]")) { + const path = parseKeyPath(line.slice(2, -2), lineNumber); + const parent = descend(root, path.slice(0, -1)); + const last = path.at(-1)!; + const existing = parent[last]; + const list = Array.isArray(existing) ? existing : []; + parent[last] = list; + current = {}; + list.push(current); + continue; + } + if (line.startsWith("[") && line.endsWith("]")) { + current = descend(root, parseKeyPath(line.slice(1, -1), lineNumber)); + continue; + } + const eq = findUnquoted(line, "="); + if (eq === -1) throw new TomlError(`expected key = value`, lineNumber); + const path = parseKeyPath(line.slice(0, eq), lineNumber); + const raw = line.slice(eq + 1).trim(); + const target = descend(current, path.slice(0, -1)); + target[path.at(-1)!] = parseValue(raw, lineNumber); + } + return root; +} + +export class TomlError extends Error { + constructor(message: string, line: number) { + super(`task.toml line ${line}: ${message}`); + } +} + +function descend(table: TomlTable, path: readonly string[]): TomlTable { + let node = table; + for (const segment of path) { + const next = node[segment]; + if (next === undefined) { + const created: TomlTable = {}; + node[segment] = created; + node = created; + } else if (Array.isArray(next)) { + const last = next.at(-1); + if (!isTable(last)) throw new Error(`cannot descend into array at ${segment}`); + node = last; + } else if (isTable(next)) { + node = next; + } else { + throw new Error(`key ${segment} is not a table`); + } + } + return node; +} + +function isTable(value: unknown): value is TomlTable { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseKeyPath(text: string, line: number): string[] { + const parts = text + .split(".") + .map((part) => part.trim()) + .map((part) => (part.startsWith('"') ? parseString(part, line) : part)); + if (parts.some((part) => part.length === 0)) throw new TomlError(`invalid key ${text}`, line); + return parts; +} + +function parseValue(raw: string, line: number): TomlValue { + if (raw.length === 0) throw new TomlError("missing value", line); + if (raw.startsWith('"') || raw.startsWith("'")) return parseString(raw, line); + if (raw === "true") return true; + if (raw === "false") return false; + if (raw.startsWith("[")) return parseArray(raw, line); + if (raw.startsWith("{")) throw new TomlError("inline tables are not supported", line); + const number = Number(raw.replaceAll("_", "")); + if (Number.isFinite(number)) return number; + throw new TomlError(`unsupported value ${raw}`, line); +} + +function parseString(raw: string, line: number): string { + const quote = raw[0]; + if (quote !== '"' && quote !== "'") throw new TomlError(`expected string`, line); + if (raw.length < 2 || raw.at(-1) !== quote) throw new TomlError(`unterminated string`, line); + const body = raw.slice(1, -1); + if (quote === "'") return body; + return body.replaceAll(/\\(["\\nrt])/gu, (_, code: string) => + code === "n" ? "\n" : code === "r" ? "\r" : code === "t" ? "\t" : code, + ); +} + +function parseArray(raw: string, line: number): TomlValue[] { + if (!raw.endsWith("]")) throw new TomlError("unterminated array", line); + const inner = raw.slice(1, -1).trim(); + if (inner.length === 0) return []; + return splitTopLevel(inner, line).map((item) => parseValue(item.trim(), line)); +} + +function splitTopLevel(text: string, line: number): string[] { + const items: string[] = []; + let depth = 0; + let quote: string | null = null; + let start = 0; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === "[") depth++; + else if (char === "]") depth--; + else if (char === "," && depth === 0) { + items.push(text.slice(start, index)); + start = index + 1; + } + } + if (quote !== null || depth !== 0) throw new TomlError("unbalanced array", line); + const tail = text.slice(start).trim(); + if (tail.length > 0) items.push(tail); + return items; +} + +function findUnquoted(text: string, needle: string): number { + let quote: string | null = null; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === needle) return index; + } + return -1; +} + +function stripComment(line: string): string { + const hash = findUnquoted(line, "#"); + return hash === -1 ? line : line.slice(0, hash); +} diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts new file mode 100644 index 0000000000..87489b3f96 --- /dev/null +++ b/packages/eve-bench/src/core/trial.ts @@ -0,0 +1,219 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Writable } from "node:stream"; +import { finished } from "node:stream/promises"; + +import { + buildImage, + Container, + containerArch, + createBoundedLog, + pullImage, + type ProcessResult, +} from "./docker.ts"; +import type { Harness, HarnessBundle } from "./harness.ts"; +import { + classifyTrial, + type InvalidTrial, + type StepResult, + type TrialResult, + type Usage, +} from "./result.ts"; +import type { Task } from "./task.ts"; + +export interface TrialInput { + readonly task: Task; + readonly attempt: number; + readonly harness: Harness; + readonly bundle: HarnessBundle; + readonly model: string; + /** Host env forwarded to the harness exec only (provider credentials). */ + readonly forwardEnv: Readonly>; + readonly job: string; + readonly dir: string; + readonly signal: AbortSignal; +} + +const INSTALL_DIR = "/installed-agent"; +const LOGS_DIR = "/logs"; +const TESTS_DIR = "/tests"; + +export async function runTrial(input: TrialInput): Promise { + const { task, signal } = input; + const startedAt = new Date().toISOString(); + await mkdir(input.dir, { recursive: true }); + const containerLog = createBoundedLog(join(input.dir, "container.log")); + let container: Container | undefined; + let agent: StepResult = skipped(); + let verifier: StepResult = skipped(); + let reward: number | null = null; + let usage: Usage | undefined; + let error: string | undefined; + try { + const image = await resolveImage(task, containerLog, signal); + container = await Container.start( + { + image, + name: `eve-bench-${task.name}-${input.attempt}-${Date.now().toString(36)}`, + job: input.job, + task: task.name, + cpus: task.environment.cpus, + memoryMb: task.environment.memoryMb, + network: task.environment.allowInternet, + }, + signal, + ); + const workdir = await resolveWorkdir(container); + const bundleArch = input.bundle.arch; + const arch = bundleArch ? await containerArch(container) : undefined; + await container.exec( + `mkdir -p ${INSTALL_DIR} ${LOGS_DIR}/agent ${LOGS_DIR}/verifier ${TESTS_DIR}${arch ? ` ${INSTALL_DIR}/arch` : ""}`, + { timeoutMs: 30_000, signal }, + ); + await container.upload(`${input.bundle.dir}/.`, INSTALL_DIR, signal); + if (arch && bundleArch) { + await container.upload(`${bundleArch[arch]}/.`, `${INSTALL_DIR}/arch`, signal); + } + for (const path of input.harness.stage?.({ taskDir: task.dir }) ?? []) { + await container.upload(path, INSTALL_DIR, signal); + } + const instructionPath = join(input.dir, "instruction.md"); + await writeFile(instructionPath, task.instruction); + await container.upload(instructionPath, `${INSTALL_DIR}/instruction.md`, signal); + + const runContext = { + model: input.model, + installDir: INSTALL_DIR, + instructionPath: `${INSTALL_DIR}/instruction.md`, + taskWorkdir: workdir, + logsDir: `${LOGS_DIR}/agent`, + }; + agent = await step(() => + withBoundedLog(join(input.dir, "agent.log"), (log) => + container!.exec(input.harness.command(runContext), { + cwd: workdir, + env: { ...input.forwardEnv, ...input.harness.env(runContext) }, + timeoutMs: task.agentTimeoutMs, + signal, + log, + }), + ), + ); + + await container.upload(`${join(task.dir, "tests")}/.`, TESTS_DIR, signal); + verifier = await step(() => + withBoundedLog(join(input.dir, "verifier.log"), (log) => + container!.exec(`bash ${TESTS_DIR}/test.sh`, { + cwd: workdir, + timeoutMs: task.verifierTimeoutMs, + signal, + log, + }), + ), + ); + + const missing: string[] = []; + for (const logs of ["agent", "verifier"]) { + if (!(await container.download(`${LOGS_DIR}/${logs}`, input.dir))) { + missing.push(`${LOGS_DIR}/${logs}`); + } + } + reward = await readReward(join(input.dir, "verifier", "reward.txt")); + usage = await input.harness.readUsage?.(join(input.dir, "agent")); + if (missing.length > 0) error = `could not download ${missing.join(", ")}`; + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } finally { + if (container) await container.remove(); + containerLog.end(); + await finished(containerLog); + } + const invalid = classifyTrial( + { agent, verifier, reward, usage, error }, + input.harness.modelFree === true, + ); + const recorded: { usage?: Usage; error?: string; invalid?: InvalidTrial } = {}; + if (usage) recorded.usage = usage; + if (error) recorded.error = error; + if (invalid) recorded.invalid = invalid; + const result: TrialResult = { + task: task.name, + attempt: input.attempt, + harness: input.harness.name, + model: input.model, + reward, + agent, + verifier, + ...recorded, + startedAt, + finishedAt: new Date().toISOString(), + }; + const target = join(input.dir, "trial.json"); + const temporary = join(input.dir, `trial.${process.pid}.tmp`); + try { + await writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`, { flag: "wx" }); + await rename(temporary, target); + } finally { + await rm(temporary, { force: true }); + } + return result; +} + +async function withBoundedLog(path: string, run: (log: Writable) => Promise): Promise { + const log = createBoundedLog(path); + try { + return await run(log); + } finally { + log.end(); + await finished(log); + } +} + +async function resolveImage(task: Task, log: Writable, signal: AbortSignal): Promise { + const options = { timeoutMs: task.buildTimeoutMs, signal, log }; + if (task.environment.dockerImage) { + await pullImage(task.environment.dockerImage, options); + return task.environment.dockerImage; + } + const dockerfile = await readFile(join(task.environment.dockerfileDir, "Dockerfile")); + const tag = `eve-bench/${task.name}:${createHash("sha256").update(dockerfile).digest("hex").slice(0, 12)}`; + await buildImage(tag, task.environment.dockerfileDir, options); + return tag; +} + +async function resolveWorkdir(container: Container): Promise { + const result = await container.exec("pwd", { timeoutMs: 10_000 }); + const workdir = result.stdout.trim(); + if (result.exitCode !== 0 || workdir.length === 0) { + throw new Error(`Could not resolve the task working directory: ${result.stderr.trim()}`); + } + return workdir; +} + +async function step(run: () => Promise): Promise { + const started = Date.now(); + const result = await run(); + return { + status: result.timedOut + ? "timeout" + : result.aborted || result.exitCode !== 0 + ? "failed" + : "completed", + exitCode: result.exitCode, + durationMs: Date.now() - started, + }; +} + +function skipped(): StepResult { + return { status: "skipped", exitCode: null, durationMs: 0 }; +} + +async function readReward(path: string): Promise { + try { + const value = Number((await readFile(path, "utf8")).trim()); + return Number.isFinite(value) ? value : null; + } catch { + return null; + } +} diff --git a/packages/eve-bench/src/core/usage.test.ts b/packages/eve-bench/src/core/usage.test.ts new file mode 100644 index 0000000000..668aaa43ca --- /dev/null +++ b/packages/eve-bench/src/core/usage.test.ts @@ -0,0 +1,123 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { CLI_USAGE, readCliUsage } from "../harnesses/cli/usage.ts"; +import { eveStepUsage, readEveUsage } from "../harnesses/eve/usage.ts"; +import { readJsonlUsage } from "./usage.ts"; + +// Shapes below are copied from recorded native logs (pi 0.85.1, opencode, eve 0.35). +const piAssistant = (input: number, cacheRead: number, output: number, total = 0) => ({ + type: "message_end", + message: { + role: "assistant", + usage: { + input, + output, + cacheRead, + cacheWrite: 0, + totalTokens: input + output, + cost: { total }, + }, + stopReason: "toolUse", + }, +}); + +test("pi usage sums assistant message_end records and adds cache reads to input", () => { + assert.equal( + CLI_USAGE.pi({ type: "message_end", message: { role: "user", usage: null } }), + undefined, + ); + assert.equal(CLI_USAGE.pi({ type: "agent_end", messages: [] }), undefined); + assert.deepEqual(CLI_USAGE.pi(piAssistant(100, 900, 50, 0.01)), { + input: 1000, + output: 50, + cached: 900, + cost: 0.01, + }); +}); + +test("opencode usage counts reasoning as output and cache reads as input", () => { + const stepFinish = { + type: "step_finish", + part: { + type: "step-finish", + reason: "tool-calls", + tokens: { + total: 8542, + input: 8154, + output: 33, + reasoning: 355, + cache: { write: 0, read: 12 }, + }, + cost: 0, + }, + }; + assert.deepEqual(CLI_USAGE.opencode(stepFinish), { + input: 8166, + output: 388, + cached: 12, + cost: 0, + }); + assert.equal(CLI_USAGE.opencode({ type: "text", part: {} }), undefined); +}); + +test("codex usage reads turn.completed totals", () => { + assert.deepEqual( + CLI_USAGE.codex({ + type: "turn.completed", + usage: { input_tokens: 24763, cached_input_tokens: 24448, output_tokens: 122 }, + }), + { input: 24763, output: 122, cached: 24448 }, + ); + assert.equal(CLI_USAGE.codex({ type: "turn.failed", error: { message: "401" } }), undefined); +}); + +test("eve usage reads step.completed usage", () => { + assert.deepEqual( + eveStepUsage({ + type: "step.completed", + data: { usage: { inputTokens: 10, outputTokens: 2, cacheReadTokens: 4, costUsd: 0.5 } }, + }), + { input: 10, output: 2, cached: 4, cost: 0.5 }, + ); + assert.equal(eveStepUsage({ type: "step.started", data: { modelId: "x" } }), undefined); +}); + +test("log readers skip diagnostics, report missing logs, and mark zero cost unknown", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "eve-bench-usage-")); + t.after(() => rm(dir, { recursive: true, force: true })); + + assert.equal(await readCliUsage("pi", dir), undefined); + + // A 401 still emits an assistant message_end, so usage exists but is zero. + await writeFile( + join(dir, "pi.jsonl"), + ["native warning: not json", JSON.stringify(piAssistant(0, 0, 0)), "{truncated", ""].join("\n"), + ); + assert.deepEqual(await readCliUsage("pi", dir), { + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + costUsd: null, + }); + + await writeFile( + join(dir, "events.ndjson"), + `${JSON.stringify({ type: "session.started" })}\n${JSON.stringify({ + type: "step.completed", + data: { usage: { inputTokens: 7, outputTokens: 3, costUsd: 0.25 } }, + })}\n${JSON.stringify({ type: "step.completed", data: { usage: { inputTokens: 5, outputTokens: 1 } } })}\n`, + ); + assert.deepEqual(await readEveUsage(dir), { + inputTokens: 12, + outputTokens: 4, + cachedTokens: 0, + costUsd: 0.25, + }); + + await writeFile(join(dir, "empty.jsonl"), `${JSON.stringify({ type: "session.started" })}\n`); + assert.equal(await readJsonlUsage(join(dir, "empty.jsonl"), eveStepUsage), undefined); +}); diff --git a/packages/eve-bench/src/core/usage.ts b/packages/eve-bench/src/core/usage.ts new file mode 100644 index 0000000000..90dbb58a5d --- /dev/null +++ b/packages/eve-bench/src/core/usage.ts @@ -0,0 +1,63 @@ +import { createReadStream } from "node:fs"; +import { createInterface } from "node:readline"; + +import type { Usage } from "./result.ts"; + +export interface UsageRecord { + readonly input: number; + readonly output: number; + readonly cached: number; + readonly cost?: number; +} + +/** + * Sums usage records from a JSON Lines log, skipping non-JSON diagnostics. + * Returns undefined when the log is missing or holds no usage records. A zero + * cost total is reported as unknown because harnesses without pricing data + * report zero. + */ +export async function readJsonlUsage( + path: string, + extract: (event: unknown) => UsageRecord | undefined, +): Promise { + let records = 0; + let inputTokens = 0; + let outputTokens = 0; + let cachedTokens = 0; + let cost = 0; + try { + const lines = createInterface({ input: createReadStream(path), crlfDelay: Infinity }); + for await (const line of lines) { + if (!line.startsWith("{")) continue; + let event: unknown; + try { + event = JSON.parse(line); + } catch { + continue; + } + const record = extract(event); + if (!record) continue; + records++; + inputTokens += record.input; + outputTokens += record.output; + cachedTokens += record.cached; + cost += record.cost ?? 0; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + if (records === 0) return undefined; + return { inputTokens, outputTokens, cachedTokens, costUsd: cost > 0 ? cost : null }; +} + +export function count(...values: unknown[]): number { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return 0; +} + +export function record(value: unknown): Record { + return value !== null && typeof value === "object" ? (value as Record) : {}; +} diff --git a/packages/eve-bench/src/harnesses/cli/index.ts b/packages/eve-bench/src/harnesses/cli/index.ts new file mode 100644 index 0000000000..03a74b7d9d --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/index.ts @@ -0,0 +1,50 @@ +import type { Harness } from "../../core/harness.ts"; +import { prepareCliBundle } from "./prepare.ts"; +import { readCliUsage } from "./usage.ts"; +import { + GATEWAY_BASE_URL, + shellQuote, + validateModel, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; + +export function createCliHarness( + name: "hermes" | "pi" | "opencode" | "codex", + options: { version: string; baseUrl?: string; reasoning?: string }, +): Harness { + const supported = validateOptions(name, options); + const pinned = { ...options }; + return { + name: `${name}@${pinned.version}`, + credentials: ["AI_GATEWAY_API_KEY"], + prepare: async (ctx) => { + validateModel(ctx.model); + const bundle = await prepareCliBundle(supported, pinned, ctx); + return { + ...bundle, + provenance: { + ...bundle.provenance, + model: ctx.model, + baseUrl: pinned.baseUrl ?? GATEWAY_BASE_URL, + reasoning: pinned.reasoning ?? null, + protocol: supported === "codex" ? "responses" : "chat-completions", + }, + }; + }, + command: (ctx) => + `exec ${shellQuote(`${ctx.installDir}/arch/node`)} ${shellQuote(`${ctx.installDir}/runner.ts`)}`, + readUsage: (agentLogsDir) => readCliUsage(supported, agentLogsDir), + env: (ctx) => { + validateModel(ctx.model); + return { + EVE_BENCH_CLI_NAME: name, + EVE_BENCH_CLI_OPTIONS: JSON.stringify(pinned), + EVE_BENCH_CLI_CONTEXT: JSON.stringify(ctx), + }; + }, + }; +} + +export type { CliName, CliOptions }; diff --git a/packages/eve-bench/src/harnesses/cli/output.ts b/packages/eve-bench/src/harnesses/cli/output.ts new file mode 100644 index 0000000000..7c5e8eb7f7 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/output.ts @@ -0,0 +1,53 @@ +import { StringDecoder } from "node:string_decoder"; + +/** Retain a suffix so even a credential split across pipe chunks is redacted. */ +export class SecretRedactor { + private readonly decoder = new StringDecoder("utf8"); + private pending = ""; + private readonly secrets: string[]; + private readonly keep: number; + + constructor(secret: string) { + this.secrets = [ + ...new Set([secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]), + ] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + this.keep = Math.max(1, ...this.secrets.map((value) => value.length)); + } + + push(chunk: Buffer): string { + this.pending += this.decoder.write(chunk); + return this.drain(false); + } + + finish(): string { + this.pending += this.decoder.end(); + return this.drain(true); + } + + private drain(final: boolean): string { + let output = ""; + while (this.pending.length > 0) { + const safe = final ? this.pending.length : Math.max(0, this.pending.length - this.keep + 1); + if (!safe) break; + let start = this.pending.length; + let matched = ""; + for (const secret of this.secrets) { + const index = this.pending.indexOf(secret); + if (index >= 0 && index < start) { + start = index; + matched = secret; + } + } + if (matched && start < safe) { + output += this.pending.slice(0, start) + "[REDACTED]"; + this.pending = this.pending.slice(start + matched.length); + } else { + output += this.pending.slice(0, safe); + this.pending = this.pending.slice(safe); + } + } + return output; + } +} diff --git a/packages/eve-bench/src/harnesses/cli/plan.test.ts b/packages/eve-bench/src/harnesses/cli/plan.test.ts new file mode 100644 index 0000000000..7be2b4233c --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createCliHarness } from "./index.ts"; +import { + executionPlan, + packageName, + validateModel, + validateOptions, + type SupportedCliName, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const ctx = { + model: "deepseek/deepseek-v4", + installDir: "/installed-agent", + instructionPath: "/instruction.md", + taskWorkdir: "/workspace", + logsDir: "/logs/agent", +}; + +for (const name of ["pi", "opencode", "codex"] as const) { + test(`${name} requires a pin and rejects credential-bearing endpoints`, () => { + for (const version of [ + "", + "latest", + "next", + "^1.2.3", + "1", + "1.2", + "*", + "1.2.3; echo unsafe", + "v1.2.3", + "01.2.3", + ]) { + assert.throws(() => createCliHarness(name, { version }), /explicit exact version/); + } + assert.equal(createCliHarness(name, { version: "1.2.3-rc.1" }).name, `${name}@1.2.3-rc.1`); + for (const baseUrl of [ + "https://user:key@example.com/v1", + "https://example.com/v1?key=secret", + "https://example.com/#secret", + "http://example.com", + "not a URL", + ]) { + assert.throws( + () => createCliHarness(name, { version: "1.2.3", baseUrl }), + /without credentials/, + ); + } + }); + + test(`${name} retains full provider-qualified model IDs`, () => { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6", "openai/acme/served-model"]) { + const plan = executionPlan( + name, + { version: "1.2.3", reasoning: "high" }, + { ...ctx, model }, + "/runtime", + "--a 'quote'\n$(touch /never)", + ); + assert.ok( + plan.args.includes(model) || plan.args.includes(`--model=eve-bench-gateway/${model}`), + ); + if (name === "pi") { + assert.equal(plan.stdin, "--a 'quote'\n$(touch /never)"); + assert.ok(!plan.args.includes("--")); + } else { + assert.equal(plan.args.at(-1), "--a 'quote'\n$(touch /never)"); + assert.equal(plan.args.at(-2), "--"); + } + assert.match(plan.config, /https:\/\/ai-gateway.vercel.sh\/v1/); + assert.ok( + !JSON.stringify( + createCliHarness(name, { version: "1.2.3" }).env({ ...ctx, model }), + ).includes("API_KEY"), + ); + } + }); +} + +test("pi custom models use runtime environment references and explicit thinking capability", () => { + const plan = executionPlan( + "pi", + { version: "0.74.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "chat-completions"); + assert.deepEqual(JSON.parse(plan.config).providers["eve-bench-gateway"], { + baseUrl: "https://ai-gateway.vercel.sh/v1", + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [{ id: ctx.model, reasoning: true }], + }); + assert.deepEqual(plan.args.slice(0, 4), ["--print", "--mode", "json", "--session-dir"]); + assert.ok(plan.args.includes("--thinking")); +}); + +test("OpenCode uses a custom compatible provider, not OpenAI's Responses loader", () => { + const plan = executionPlan( + "opencode", + { version: "1.2.3", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + const provider = JSON.parse(plan.config).provider["eve-bench-gateway"]; + assert.equal(provider.npm, "@ai-sdk/openai-compatible"); + assert.deepEqual(provider.env, ["OPENAI_API_KEY"]); + assert.equal(provider.options.baseURL, "https://ai-gateway.vercel.sh/v1"); + assert.deepEqual(provider.models[ctx.model].variants, { high: { reasoningEffort: "high" } }); + assert.ok(plan.args.includes("--dangerously-skip-permissions")); + assert.ok(plan.args.includes("--variant")); +}); + +test("Codex has env-key auth, a Responses wire API, and no auth.json", () => { + const plan = executionPlan( + "codex", + { version: "0.118.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "responses"); + assert.match(plan.config, /env_key = "AI_GATEWAY_API_KEY"/); + assert.match(plan.config, /requires_openai_auth = false/); + assert.match(plan.config, /wire_api = "responses"/); + assert.doesNotMatch(plan.config, /auth.json/); + assert.ok(plan.args.includes("--dangerously-bypass-approvals-and-sandbox")); + assert.ok(plan.args.includes("model_reasoning_effort=high")); +}); + +test("no reasoning option means no imposed reasoning setting", () => { + for (const name of ["pi", "opencode", "codex"] as const) { + const plan = executionPlan(name, { version: "1.2.3" }, ctx, "/runtime", "instruction"); + assert.doesNotMatch(plan.config, /reasoningEffort|model_reasoning_effort|"reasoning"/); + assert.ok(!plan.args.includes("--variant")); + } + assert.throws(() => validateOptions("pi", { version: "1.2.3", reasoning: "max" }), /reasoning/); + assert.throws( + () => validateOptions("codex", { version: "1.2.3", reasoning: "off" }), + /reasoning/, + ); +}); + +test("package names follow Harbor's pi rename boundary", () => { + assert.equal(packageName("pi", "0.73.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0-rc.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("pi", "1.0.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("codex", "0.118.0"), "@openai/codex"); + assert.equal(packageName("opencode", "1.2.3"), "opencode-ai"); +}); + +test("Hermes fails explicitly before installation, without a substitute agent", () => { + assert.throws( + () => createCliHarness("hermes", { version: "v2026.7.20" }), + /Hermes is not supported.*pinned, audited Python\/uv bundle/, + ); +}); + +test("model validation rejects unqualified IDs and command/control injection", () => { + for (const model of [ + "gpt-5.6", + "openai/", + "openai/gpt\nmodel", + "openai/$(cmd)", + "openai/gpt model", + ]) { + assert.throws(() => validateModel(model), /full provider-qualified/); + } +}); + +test("constructor snapshots options instead of accepting later mutation", () => { + const options = { version: "1.2.3" }; + const harness = createCliHarness("pi", options); + options.version = "latest"; + assert.match(harness.env(ctx).EVE_BENCH_CLI_OPTIONS!, /1.2.3/); +}); + +test("redaction covers every byte boundary including escaped and Unicode secrets", () => { + for (const secret of ["fake-key-abcdef", 'fake-"key', "fake-秘密"]) { + for (const form of [secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]) { + const input = Buffer.from(`prefix ${form} suffix ${form}!`); + for (let split = 0; split <= input.length; split++) { + const redactor = new SecretRedactor(secret); + const output = + redactor.push(input.subarray(0, split)) + + redactor.push(input.subarray(split)) + + redactor.finish(); + assert.equal(output, "prefix [REDACTED] suffix [REDACTED]!"); + } + } + } +}); + +// Keep the supported surface explicit in the pure tests, independently of parent wiring. +const supported: SupportedCliName[] = ["pi", "opencode", "codex"]; +test("all supported adapters implement the shared Harness contract", () => { + for (const name of supported) { + const harness = createCliHarness(name, { version: "1.2.3" }); + assert.equal(typeof harness.prepare, "function"); + assert.match(harness.command({ ...ctx, installDir: "/has space/and'quote" }), /^exec '/); + } +}); diff --git a/packages/eve-bench/src/harnesses/cli/plan.ts b/packages/eve-bench/src/harnesses/cli/plan.ts new file mode 100644 index 0000000000..d87ce508b0 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.ts @@ -0,0 +1,205 @@ +import type { HarnessRunContext } from "../../core/harness.ts"; + +export type CliName = "hermes" | "pi" | "opencode" | "codex"; +export type SupportedCliName = Exclude; +export interface CliOptions { + readonly version: string; + readonly baseUrl?: string; + readonly reasoning?: string; +} + +export const GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"; +const EXACT_VERSION = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/; + +export function validateOptions(name: CliName, options: CliOptions): SupportedCliName { + if (name === "hermes") { + throw new Error( + "Hermes is not supported by this adapter: Harbor installs a mutable main/scripts/install.sh and only establishes OPENAI_BASE_URL routing for openai/*, not Gateway DeepSeek. A pinned, audited Python/uv bundle and verified custom-provider routing are required; no fallback agent is run.", + ); + } + if (!["pi", "opencode", "codex"].includes(name)) throw new Error("Unsupported CLI harness"); + if (!EXACT_VERSION.test(options.version ?? "")) { + throw new Error( + "CLI harness requires an explicit exact version (for example 1.2.3); tags, ranges, and latest are not allowed.", + ); + } + let url: URL; + try { + url = new URL(options.baseUrl ?? GATEWAY_BASE_URL); + } catch { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (options.reasoning !== undefined) { + const allowed = + name === "pi" + ? ["off", "minimal", "low", "medium", "high", "xhigh"] + : name === "codex" + ? ["none", "minimal", "low", "medium", "high", "xhigh"] + : undefined; + if ( + allowed + ? !allowed.includes(options.reasoning) + : !/^[a-zA-Z0-9_-]{1,64}$/.test(options.reasoning) + ) { + throw new Error(`Unsupported ${name} reasoning value`); + } + } + return name; +} + +export function validateModel(model: string): void { + if ( + model.length > 256 || + !/^[a-zA-Z0-9][a-zA-Z0-9._-]*\/[a-zA-Z0-9][a-zA-Z0-9/._:-]*$/.test(model) + ) { + throw new Error( + "CLI model must be a full provider-qualified model ID, without whitespace or control characters.", + ); + } +} + +export function packageName(name: SupportedCliName, version: string): string { + if (name === "codex") return "@openai/codex"; + if (name === "opencode") return "opencode-ai"; + const [major, minor] = version.split(".").map(Number); + // Harbor pi.py changes package namespace at 0.74.0 (prereleases precede it). + return major === 0 && (minor! < 74 || (minor === 74 && version.startsWith("0.74.0-"))) + ? "@mariozechner/pi-coding-agent" + : "@earendil-works/pi-coding-agent"; +} + +export interface ExecutionPlan { + readonly args: string[]; + readonly stdin?: string; + readonly env: Record; + readonly configPath: string; + readonly config: string; + readonly protocol: "chat-completions" | "responses"; +} + +/** Native commands/config follow Harbor installed/{pi,opencode,codex}.py at + * 2fe1615503fed39ad82b7ce09b22497996b30f1f. Gateway model IDs deliberately stay intact. + */ +export function executionPlan( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessRunContext, + runtimeDir: string, + instruction: string, +): ExecutionPlan { + validateOptions(name, options); + validateModel(ctx.model); + const baseUrl = options.baseUrl ?? GATEWAY_BASE_URL; + const json = (value: unknown) => `${JSON.stringify(value, null, 2)}\n`; + if (name === "pi") { + const model: { id: string; reasoning?: boolean } = { id: ctx.model }; + if (options.reasoning) model.reasoning = options.reasoning !== "off"; + return { + protocol: "chat-completions", + args: [ + "--print", + "--mode", + "json", + "--session-dir", + `${runtimeDir}/pi/sessions`, + "--provider", + "eve-bench-gateway", + "--model", + ctx.model, + ...(options.reasoning ? ["--thinking", options.reasoning] : []), + ], + // pi 0.80.3's parseArgs treats -- as an unknown flag. main.readPipedStdin + // passes stdin to prepareInitialMessage without flag or @file interpretation. + stdin: instruction, + env: { PI_CODING_AGENT_DIR: `${runtimeDir}/pi` }, + configPath: `${runtimeDir}/pi/models.json`, + config: json({ + providers: { + "eve-bench-gateway": { + baseUrl, + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [model], + }, + }, + }), + }; + } + if (name === "opencode") { + return { + protocol: "chat-completions", + // Only the first component selects the CLI provider; the API ID stays intact. + args: [ + `--model=eve-bench-gateway/${ctx.model}`, + "run", + "--format=json", + ...(options.reasoning ? ["--variant", options.reasoning] : []), + "--thinking", + "--dangerously-skip-permissions", + "--", + instruction, + ], + env: { + OPENCODE_FAKE_VCS: "git", + OPENCODE_CONFIG: `${runtimeDir}/opencode/opencode.json`, + }, + configPath: `${runtimeDir}/opencode/opencode.json`, + // Harbor's mimo.py (OpenCode derivative) documents the compatible SDK override. + // OpenCode provider.ts resolves single-entry provider.env into the SDK API key. + config: json({ + provider: { + "eve-bench-gateway": { + npm: "@ai-sdk/openai-compatible", + env: ["OPENAI_API_KEY"], + options: { baseURL: baseUrl }, + models: { + [ctx.model]: options.reasoning + ? { + reasoning: true, + variants: { [options.reasoning]: { reasoningEffort: options.reasoning } }, + } + : {}, + }, + }, + }, + }), + }; + } + return { + protocol: "responses", + args: [ + "exec", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--model", + ctx.model, + "--json", + "--enable", + "unified_exec", + ...(options.reasoning ? ["-c", `model_reasoning_effort=${options.reasoning}`] : []), + "--", + instruction, + ], + env: { CODEX_HOME: `${runtimeDir}/codex`, OPENAI_BASE_URL: baseUrl }, + configPath: `${runtimeDir}/codex/config.toml`, + // Unlike Harbor's auth.json, Codex's native env_key needs no persisted credential. + // Source: openai/codex, codex-rs/model-provider-info/src/lib.rs, ModelProviderInfo. + config: + `model_provider = "eve-bench-gateway"\nopenai_base_url = ${JSON.stringify(baseUrl)}\n` + + `[model_providers.eve-bench-gateway]\nname = "eve-bench-gateway"\n` + + `base_url = ${JSON.stringify(baseUrl)}\nwire_api = "responses"\n` + + `env_key = "AI_GATEWAY_API_KEY"\nrequires_openai_auth = false\n`, + }; +} + +export function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} diff --git a/packages/eve-bench/src/harnesses/cli/prepare.ts b/packages/eve-bench/src/harnesses/cli/prepare.ts new file mode 100644 index 0000000000..9fd28ff377 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/prepare.ts @@ -0,0 +1,247 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; +import { packageName, type CliOptions, type SupportedCliName } from "./plan.ts"; + +const exec = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; +const SOURCES = ["plan.ts", "runner.ts", "output.ts", "prepare.ts", "index.ts"]; + +/** Installation is host-only, serial, and credential-free. Both Linux/glibc + * architectures are prepared because the Harness contract selects arch only after upload. + * Lifecycle scripts are disabled; native CLI launchers use their optional platform packages. + */ +export async function prepareCliBundle( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessPrepareContext, +): Promise { + const cache = resolve(ctx.cacheDir, "cli"); + await mkdir(cache, { recursive: true }); + const scratch = await mkdtemp(join(cache, ".prepare-")); + const cleanEnv = { PATH: process.env.PATH, HOME: join(scratch, "home"), CI: "true" }; + await mkdir(cleanEnv.HOME, { recursive: true }); + try { + let pnpmVersion: string; + try { + pnpmVersion = ( + await exec("pnpm", ["--version"], { env: cleanEnv, cwd: scratch }) + ).stdout.trim(); + } catch { + throw new Error( + "CLI bundles require pnpm 11+ on the host; pnpm could not run. No container installation was attempted.", + ); + } + if (!/^(?:1[1-9]|[2-9]\d)\./.test(pnpmVersion)) { + throw new Error( + "CLI bundles require pnpm 11+ for explicit Linux CPU/libc installs. No container installation was attempted.", + ); + } + const sources = await Promise.all(SOURCES.map((file) => readFile(join(here, file)))); + const sourceSha256 = sha256(Buffer.concat(sources)); + const pkg = packageName(name, options.version); + const key = sha256( + JSON.stringify({ pkg, version: options.version, pnpmVersion, NODE_VERSION, sourceSha256 }), + ); + const dir = join(cache, key); + if (await exists(join(dir, "provenance.json"))) return await bundleAt(dir); + + const started = Date.now(); + const building = join(scratch, "bundle"); + const common = join(building, "common"); + await mkdir(common, { recursive: true }); + for (const source of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(here, source), join(common, source)); + const lockSha256: Record = {}; + const nodeSha256: Record = {}; + for (const arch of ARCHES) { + const archDir = join(building, arch); + const app = join(archDir, "app"); + await mkdir(app, { recursive: true }); + await writeFile( + join(app, "package.json"), + JSON.stringify({ + name: "eve-bench-cli-bundle", + private: true, + type: "module", + dependencies: { [pkg]: options.version }, + }), + ); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + try { + await exec( + "pnpm", + [ + "install", + "--ignore-workspace", + "--ignore-scripts", + "--ignore-pnpmfile", + "--no-frozen-lockfile", + "--prod", + "--os=linux", + `--cpu=${arch}`, + "--libc=glibc", + "--package-import-method=copy", + "--network-concurrency=2", + "--child-concurrency=1", + "--store-dir", + join(cache, "store"), + "--reporter=silent", + ], + { cwd: app, env: cleanEnv, maxBuffer: 1024 * 1024 }, + ); + } catch { + // Do not embed package-manager stdout/errors: registry diagnostics may contain credentials. + throw new Error( + `Host pnpm install failed for ${pkg}@${options.version} (linux/${arch}, scripts disabled). No CLI setup was moved into the task timeout.`, + ); + } + const manifest = JSON.parse( + await readFile(join(app, "node_modules", pkg, "package.json"), "utf8"), + ) as { + version: string; + bin?: string | Record; + }; + if (manifest.version !== options.version) + throw new Error("Installed CLI version does not match the explicit pin"); + const bin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.[name]; + if (!bin || isAbsolute(bin)) + throw new Error("Pinned CLI package has no supported relative executable entry"); + const packageRoot = join(app, "node_modules", pkg); + let entry = resolve(packageRoot, bin); + let native = false; + if (name === "opencode") { + const platform = arch === "x64" ? "opencode-linux-x64-baseline" : "opencode-linux-arm64"; + entry = join( + app, + "node_modules/.pnpm", + `${platform}@${options.version}`, + "node_modules", + platform, + "bin/opencode", + ); + native = true; + } + if (!(await exists(entry))) { + throw new Error( + "Pinned CLI package executable is missing for the requested Linux platform", + ); + } + if (!native && relative(packageRoot, entry).startsWith("..")) { + throw new Error("Pinned CLI package executable escapes its package"); + } + await writeFile( + join(archDir, "entry.json"), + JSON.stringify({ entry: relative(archDir, entry), native }), + ); + lockSha256[arch] = sha256(await readFile(join(app, "pnpm-lock.yaml"))); + nodeSha256[arch] = await installNode(archDir, arch, scratch); + } + const provenance = { + key, + name, + version: options.version, + package: pkg, + sourceSha256, + installLocation: "host", + containerInstallMs: 0, + hostPrepareMs: Date.now() - started, + pnpmVersion, + nodeVersion: NODE_VERSION, + platforms: ["linux-x64-glibc", "linux-arm64-glibc"], + lifecycleScripts: false, + lockSha256, + nodeSha256, + adapterSources: { + harbor: + "https://github.com/harbor-framework/harbor/tree/2fe1615503fed39ad82b7ce09b22497996b30f1f/src/harbor/agents/installed", + codexEnvAuth: + "https://github.com/openai/codex/blob/1aaa453ce2868e14c10089c226ebec265657cb27/codex-rs/model-provider-info/src/lib.rs", + opencodeProviders: + "https://github.com/anomalyco/opencode/blob/ac1758c0e6b8e7368be133e87e8e27a5f60dec04/packages/opencode/src/provider/provider.ts", + }, + limitations: [ + "Native CLI/platform compatibility is not inferred from mock tests.", + "Host pnpm 11+ and Linux glibc containers are required; no musl/system-package bootstrap.", + "CLI-owned startup work is included in invocation wall time, not independently measured.", + "Custom model context/output limits use CLI defaults; reasoning settings are not normalized between agents.", + "Codex requires a Responses-compatible endpoint/model; no chat-completions fallback.", + "Native state stays ephemeral; only redacted stdout/stderr is exported. Usage is not normalized.", + ], + }; + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await cp(join(building, "provenance.json"), join(common, "provenance.json")); + try { + await rename(building, dir); + } catch (error) { + if (!(await exists(join(dir, "provenance.json")))) throw error; + } + return await bundleAt(dir); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function bundleAt(dir: string): Promise { + // Keep platform trees out of the common upload (which otherwise doubles every transfer). + return { + dir: join(dir, "common"), + arch: { x64: join(dir, "x64"), arm64: join(dir, "arm64") }, + provenance: JSON.parse(await readFile(join(dir, "provenance.json"), "utf8")) as Record< + string, + unknown + >, + }; +} + +async function installNode(archDir: string, arch: string, scratch: string): Promise { + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const base = `https://nodejs.org/dist/v${NODE_VERSION}`; + const sums = (await download(`${base}/SHASUMS256.txt`, 1024 * 1024)).toString("utf8"); + const expected = sums + .split("\n") + .find((line) => line.trim().endsWith(` ${name}.tar.xz`)) + ?.split(/\s+/)[0]; + if (!expected || !/^[a-f0-9]{64}$/.test(expected)) + throw new Error("Pinned Node checksum is missing"); + const archive = await download(`${base}/${name}.tar.xz`, 128 * 1024 * 1024); + if (sha256(archive) !== expected) throw new Error("Pinned Node archive checksum mismatch"); + const path = join(scratch, `${name}.tar.xz`); + await writeFile(path, archive); + await exec("tar", ["-xJf", path, "--strip-components=2", "-C", archDir, `${name}/bin/node`]); + await chmod(join(archDir, "node"), 0o755); + await rm(path); + return expected; +} + +async function download(url: string, limit: number): Promise { + const response = await fetch(url, { signal: AbortSignal.timeout(120_000), redirect: "error" }); + if (!response.ok || !response.body) + throw new Error(`Pinned Node download failed (HTTP ${response.status})`); + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > limit) throw new Error("Pinned Node download exceeds size limit"); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/eve-bench/src/harnesses/cli/runner.ts b/packages/eve-bench/src/harnesses/cli/runner.ts new file mode 100644 index 0000000000..d22645f000 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/runner.ts @@ -0,0 +1,237 @@ +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { finished } from "node:stream/promises"; + +import type { HarnessRunContext } from "../../core/harness.ts"; +import { + GATEWAY_BASE_URL, + executionPlan, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const OUTPUT_LIMIT = 64 * 1024 * 1024; +const INSTRUCTION_LIMIT = 1024 * 1024; + +async function main(): Promise { + const ctx = JSON.parse(process.env.EVE_BENCH_CLI_CONTEXT!) as HarnessRunContext; + const options = JSON.parse(process.env.EVE_BENCH_CLI_OPTIONS!) as CliOptions; + const name = validateOptions(process.env.EVE_BENCH_CLI_NAME as CliName, options); + const start = Date.now(); + let agentStart: number | undefined; + let exitCode = 1; + let failure: string | undefined; + let signal: string | null = null; + await mkdir(ctx.logsDir, { recursive: true }); + const runtimeDir = await mkdtemp(join(ctx.installDir, ".cli-runtime-")); + const secret = process.env.AI_GATEWAY_API_KEY; + try { + if (!secret?.trim()) + throw new Error("AI_GATEWAY_API_KEY must be forwarded to the CLI runtime environment."); + if (secret.length > 8192 || /[\r\n\0]/.test(secret)) + throw new Error("AI_GATEWAY_API_KEY has an invalid format."); + if ((await stat(ctx.instructionPath)).size > INSTRUCTION_LIMIT) + throw new Error("CLI instruction exceeds 1 MiB."); + const instruction = await readFile(ctx.instructionPath, "utf8"); + const plan = executionPlan(name, options, ctx, runtimeDir, instruction); + await mkdir(dirname(plan.configPath), { recursive: true }); + await writeFile(plan.configPath, plan.config, { mode: 0o600 }); + const { entry, native = false } = JSON.parse( + await readFile(join(ctx.installDir, "arch/entry.json"), "utf8"), + ) as { entry: string; native?: boolean }; + const executable = join(ctx.installDir, "arch", entry); + // The CLI and its tool subprocesses keep the task container's own environment + // (HOME, PATH, image ENV), as eve's tools do, so state they leave in the task + // survives into verification. Only harness-owned config paths are added. + const { + EVE_BENCH_CLI_CONTEXT: _context, + EVE_BENCH_CLI_OPTIONS: _options, + EVE_BENCH_CLI_NAME: _name, + AI_GATEWAY_API_KEY: _gatewayKey, + OPENAI_API_KEY: _openaiKey, + ...inherited + } = process.env; + const baseEnv = { ...(inherited as Record), ...plan.env }; + const version = await run( + executable, + ["--version"], + baseEnv, + ctx, + "cli-setup.log", + secret, + false, + undefined, + native, + ); + const escapedVersion = options.version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if ( + version.code !== 0 || + !new RegExp(`(?:^|\\s)${escapedVersion}(?:\\s|$)`).test(version.text) + ) { + throw new Error( + "Pinned CLI failed its --version check; native package/runtime compatibility must be checked before benchmarking.", + ); + } + agentStart = Date.now(); + const result = await run( + executable, + plan.args, + { ...baseEnv, AI_GATEWAY_API_KEY: secret, OPENAI_API_KEY: secret }, + ctx, + `${name}.jsonl`, + secret, + name !== "codex", + plan.stdin, + native, + ); + exitCode = result.code; + signal = result.signal; + if (result.errorEvent && exitCode === 0) { + exitCode = 1; + failure = `${name} emitted an error event despite exiting zero.`; + } + } catch (error) { + // Only our actionable messages are emitted; filesystem/spawn diagnostics can contain arbitrary data. + const message = error instanceof Error ? error.message : ""; + failure = + /^(AI_GATEWAY_API_KEY |CLI instruction |Pinned CLI |CLI model |Unsupported .* reasoning)/.test( + message, + ) + ? message + : "CLI setup or execution failed; inspect the redacted CLI logs and the pinned bundle."; + } finally { + const end = Date.now(); + await rm(runtimeDir, { recursive: true, force: true }); + const runtime: Record = { + name, + version: options.version, + model: ctx.model, + baseUrl: options.baseUrl ?? GATEWAY_BASE_URL, + reasoning: options.reasoning ?? null, + protocol: name === "codex" ? "responses" : "chat-completions", + installLocation: "host", + containerInstallMs: 0, + setupMs: (agentStart ?? end) - start, + agentMs: agentStart === undefined ? 0 : end - agentStart, + exitCode, + signal, + }; + if (failure) runtime.failure = failure; + await writeFile(join(ctx.logsDir, "cli-runtime.json"), `${JSON.stringify(runtime, null, 2)}\n`); + } + if (failure) process.stderr.write(`[eve-bench] ${failure}\n`); + return exitCode; +} + +async function run( + entry: string, + args: string[], + env: Record, + ctx: HarnessRunContext, + logName: string, + secret: string, + detectErrors = false, + input?: string, + native = false, +): Promise<{ code: number; signal: string | null; text: string; errorEvent: boolean }> { + const log = createWriteStream(join(ctx.logsDir, logName), { mode: 0o600 }); + let bytes = 0; + let text = ""; + let line = ""; + let skipLine = false; + let errorEvent = false; + let truncated = false; + const emit = (chunk: string, stdout: boolean) => { + if (stdout && detectErrors) { + for (const piece of chunk.split(/(?<=\n)/)) { + if (!skipLine) line += piece; + if (line.length > INSTRUCTION_LIMIT) { + line = ""; + skipLine = true; + } + if (piece.endsWith("\n")) { + if (!skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Native diagnostics need not be JSON. */ + } + } + line = ""; + skipLine = false; + } + } + } + if (text.length < 65536) text += chunk.slice(0, 65536 - text.length); + const buffer = Buffer.from(chunk); + const kept = buffer.subarray(0, Math.max(0, OUTPUT_LIMIT - bytes)); + bytes += kept.length; + if (kept.length > 0) { + log.write(kept); + process.stdout.write(kept); + } + if (!truncated && kept.length < buffer.length) { + truncated = true; + log.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + process.stdout.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + } + }; + const child = spawn(native ? entry : process.execPath, native ? args : [entry, ...args], { + cwd: ctx.taskWorkdir, + env, + stdio: ["pipe", "pipe", "pipe"], + }); + // A CLI may exit before consuming stdin; its exit status remains authoritative. + child.stdin.on("error", () => {}); + child.stdin.end(input); + const forwardTerm = () => child.kill("SIGTERM"); + const forwardInt = () => child.kill("SIGINT"); + process.on("SIGTERM", forwardTerm); + process.on("SIGINT", forwardInt); + const stdout = new SecretRedactor(secret); + const stderr = new SecretRedactor(secret); + child.stdout.on("data", (chunk: Buffer) => emit(stdout.push(chunk), true)); + child.stderr.on("data", (chunk: Buffer) => emit(stderr.push(chunk), false)); + let spawnFailed = false; + child.on("error", () => { + spawnFailed = true; + }); + const result = await new Promise<{ code: number; signal: string | null }>((resolve) => { + child.on("close", (code, signal) => resolve({ code: spawnFailed ? 1 : (code ?? 1), signal })); + }); + process.off("SIGTERM", forwardTerm); + process.off("SIGINT", forwardInt); + emit(stdout.finish(), true); + emit(stderr.finish(), false); + if (detectErrors && line && !skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Partial diagnostic. */ + } + } + log.end(); + await finished(log); + return { ...result, text, errorEvent }; +} + +function isErrorEvent(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const event = value as { type?: string; errorMessage?: unknown; message?: unknown }; + if (event.type === "error" || typeof event.errorMessage === "string") return true; + return event.message !== undefined && isErrorEvent(event.message); +} + +main().then( + (code) => { + process.exitCode = code; + }, + () => { + process.stderr.write("[eve-bench] Invalid CLI runtime context or unwritable logs directory.\n"); + process.exitCode = 1; + }, +); diff --git a/packages/eve-bench/src/harnesses/cli/usage.ts b/packages/eve-bench/src/harnesses/cli/usage.ts new file mode 100644 index 0000000000..3437d57822 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/usage.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import type { Usage } from "../../core/result.ts"; +import { count, readJsonlUsage, record, type UsageRecord } from "../../core/usage.ts"; +import type { SupportedCliName } from "./plan.ts"; + +/** Reads usage from the native JSON event stream the runner saves as `.jsonl`. */ +export function readCliUsage( + name: SupportedCliName, + agentLogsDir: string, +): Promise { + return readJsonlUsage(join(agentLogsDir, `${name}.jsonl`), CLI_USAGE[name]); +} + +export const CLI_USAGE: Record UsageRecord | undefined> = { + // pi --mode json: one assistant `message_end` per model call; `input` excludes cache reads. + pi(event) { + const value = record(event); + const message = record(value.message); + if (value.type !== "message_end" || message.role !== "assistant") return undefined; + const usage = record(message.usage); + const cached = count(usage.cacheRead); + return { + input: count(usage.input) + cached, + output: count(usage.output), + cached, + cost: count(record(usage.cost).total), + }; + }, + // opencode run --format=json: one `step_finish` per model call; reasoning is reported apart from output. + opencode(event) { + const value = record(event); + if (value.type !== "step_finish") return undefined; + const part = record(value.part); + const tokens = record(part.tokens); + const cached = count(record(tokens.cache).read); + return { + input: count(tokens.input) + cached, + output: count(tokens.output) + count(tokens.reasoning), + cached, + cost: count(part.cost), + }; + }, + // codex exec --json: one `turn.completed` per turn; input_tokens already include cached input. + codex(event) { + const value = record(event); + if (value.type !== "turn.completed") return undefined; + const usage = record(value.usage); + return { + input: count(usage.input_tokens), + output: count(usage.output_tokens), + cached: count(usage.cached_input_tokens), + }; + }, +}; diff --git a/packages/eve-bench/src/harnesses/eve/index.ts b/packages/eve-bench/src/harnesses/eve/index.ts new file mode 100644 index 0000000000..d01a6976d6 --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/index.ts @@ -0,0 +1,250 @@ +import { execFile } from "node:child_process"; +import { createHash, type Hash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { chmod, cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { + Harness, + HarnessBundle, + HarnessPrepareContext, + HarnessRunContext, +} from "../../core/harness.ts"; +import { readEveUsage } from "./usage.ts"; + +const execFileAsync = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const packageRoot = resolve(here, "../../.."); +const repoRoot = resolve(packageRoot, "../.."); +export const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; + +export type EveSource = + | { readonly kind: "local" } + | { readonly kind: "release"; readonly version: string }; + +/** + * Builds the `agent/` app against a local or released eve, and packages the + * Nitro output and runner with architecture-specific Linux Node binaries. + * Cached by content hash. + */ +export function createEveHarness(source: EveSource): Harness { + return { + name: source.kind === "local" ? "eve@local" : `eve@${source.version}`, + credentials: ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"], + prepare: (ctx) => prepare(source, ctx), + command: (ctx) => `sh ${ctx.installDir}/run.sh`, + readUsage: readEveUsage, + env: (ctx: HarnessRunContext) => ({ + EVE_PROJECT_DIR: `${ctx.installDir}/app`, + EVE_INSTRUCTION_PATH: ctx.instructionPath, + EVE_AGENT_LOGS_DIR: ctx.logsDir, + EVE_BENCH_MODEL: ctx.model, + EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, + }), + }; +} + +async function prepare(source: EveSource, ctx: HarnessPrepareContext): Promise { + const eve: ResolvedEve = + source.kind === "local" + ? await resolveLocalEve() + : { + dependency: source.version, + identity: source.version, + provenance: { kind: "release", version: source.version }, + }; + + const hash = createHash("sha256"); + hash.update(eve.identity).update(ctx.model).update(NODE_VERSION); + for (const file of await listFiles(join(packageRoot, "agent"))) + hash.update(file).update(await readFile(join(packageRoot, "agent", file))); + hash.update(await readFile(join(here, "runner.mjs"))).update(RUN_SH); + const key = hash.digest("hex").slice(0, 16); + const bundleDir = join(ctx.cacheDir, "bundles", key); + const provenance = { key, model: ctx.model, eve: eve.provenance }; + const nodeDir = join(ctx.cacheDir, `node-v${NODE_VERSION}`); + await Promise.all(ARCHES.map((arch) => ensureNode(nodeDir, arch))); + const arch = Object.fromEntries(ARCHES.map((name) => [name, join(nodeDir, name)])) as Record< + (typeof ARCHES)[number], + string + >; + if (await exists(join(bundleDir, "run.sh"))) return { dir: bundleDir, arch, provenance }; + + const building = `${bundleDir}.building`; + await rm(building, { recursive: true, force: true }); + const app = join(building, "app"); + await mkdir(app, { recursive: true }); + await cp(join(packageRoot, "agent"), join(app, "agent"), { recursive: true }); + await writeFile( + join(app, "package.json"), + `${JSON.stringify({ name: "eve-bench-app", private: true, type: "module", dependencies: { eve: eve.dependency, zod: "4.4.3" } }, null, 2)}\n`, + ); + await writeFile(join(app, "tsconfig.json"), APP_TSCONFIG); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + if (source.kind === "local") { + const artifacts = join(ctx.cacheDir, "artifacts"); + await mkdir(artifacts, { recursive: true }); + await cp(await packLocalEve(artifacts), join(app, "eve.tgz")); + } + const npm = (...args: string[]) => + execFileAsync("npm", [...args, "--ignore-scripts", "--userconfig", join(app, ".npmrc")], { + cwd: app, + maxBuffer: 64 * 1024 * 1024, + }); + await npm("install", "--package-lock-only"); + await npm("ci"); + await execFileAsync(join(app, "node_modules", ".bin", "eve"), ["build"], { + cwd: app, + env: { ...process.env, EVE_BENCH_MODEL: ctx.model, EVE_BENCH_TASK_WORKDIR: "/workspace" }, + maxBuffer: 64 * 1024 * 1024, + }); + await rm(join(app, "agent"), { recursive: true, force: true }); + await rm(join(app, "package-lock.json"), { force: true }); + await rm(join(app, ".npmrc"), { force: true }); + await rm(join(app, "eve.tgz"), { force: true }); + await rm(join(app, "tsconfig.json"), { force: true }); + await rm(join(app, ".eve"), { recursive: true, force: true }); + // Nitro's `.output` is self-contained and the runner speaks plain HTTP. + await rm(join(app, "node_modules"), { recursive: true, force: true }); + await cp(join(here, "runner.mjs"), join(app, "runner.mjs")); + await writeFile(join(building, "run.sh"), RUN_SH, { mode: 0o755 }); + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await rm(bundleDir, { recursive: true, force: true }); + await rename(building, bundleDir); + return { dir: bundleDir, arch, provenance }; +} + +interface ResolvedEve { + readonly dependency: string; + readonly identity: string; + readonly provenance: Record; +} + +async function resolveLocalEve(): Promise { + const evePackage = join(repoRoot, "packages/eve"); + const dist = join(evePackage, "dist"); + if (!(await isDirectory(dist))) { + throw new Error( + "Local eve dist is missing. Run `pnpm build` or `pnpm --filter eve build` first.", + ); + } + + const hash = createHash("sha256"); + hash.update("package.json\0").update(await readFile(join(evePackage, "package.json"))); + for (const file of await listFiles(dist)) { + hash.update("\0").update(file).update("\0"); + await hashFile(hash, join(dist, file)); + } + const distSha256 = hash.digest("hex"); + const [gitSha, status] = await Promise.all([ + execFileAsync("git", ["rev-parse", "HEAD"], { cwd: repoRoot }), + execFileAsync("git", ["status", "--porcelain", "packages/eve"], { cwd: repoRoot }), + ]); + return { + dependency: "file:eve.tgz", + identity: distSha256, + provenance: { + kind: "local", + gitSha: gitSha.stdout.trim(), + dirty: status.stdout.trim().length > 0, + distSha256, + }, + }; +} + +async function packLocalEve(artifacts: string): Promise { + const { stdout } = await execFileAsync( + "pnpm", + ["--filter", "eve", "pack", "--pack-destination", artifacts, "--json"], + { cwd: repoRoot, maxBuffer: 16 * 1024 * 1024 }, + ); + const jsonStart = stdout.indexOf("{"); + const packed = JSON.parse(stdout.slice(jsonStart)) as { filename: string }; + return resolve(artifacts, packed.filename); +} + +export async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { + const archDir = join(nodeDir, arch); + const target = join(archDir, "node"); + if (await exists(target)) return; + await mkdir(archDir, { recursive: true }); + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const url = `https://nodejs.org/dist/v${NODE_VERSION}/${name}.tar.xz`; + const response = await fetch(url); + if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status}`); + const archive = join(nodeDir, `${name}.tar.xz`); + await writeFile(archive, Buffer.from(await response.arrayBuffer())); + const extractDir = join(nodeDir, `${name}.extract`); + await mkdir(extractDir, { recursive: true }); + await execFileAsync("tar", [ + "-xJf", + archive, + "--strip-components=2", + "-C", + extractDir, + `${name}/bin/node`, + ]); + await rename(join(extractDir, "node"), target); + await chmod(target, 0o755); + await rm(extractDir, { recursive: true, force: true }); + await rm(archive, { force: true }); +} + +async function listFiles(dir: string, prefix = ""): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) files.push(...(await listFiles(join(dir, entry.name), rel))); + else files.push(rel); + } + return files; +} + +async function hashFile(hash: Hash, path: string): Promise { + for await (const chunk of createReadStream(path)) hash.update(chunk); +} + +async function isDirectory(path: string): Promise { + return stat(path).then( + (entry) => entry.isDirectory(), + () => false, + ); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} + +const RUN_SH = `#!/bin/sh +set -eu +dir=$(cd "$(dirname "$0")" && pwd) +exec "$dir/arch/node" "$dir/app/runner.mjs" +`; + +const APP_TSCONFIG = `${JSON.stringify( + { + compilerOptions: { + target: "ES2024", + lib: ["ES2024"], + module: "NodeNext", + moduleResolution: "NodeNext", + moduleDetection: "force", + verbatimModuleSyntax: true, + noEmit: true, + strict: true, + isolatedModules: true, + skipLibCheck: true, + types: ["node"], + }, + include: ["agent/**/*.ts"], + }, + null, + 2, +)}\n`; diff --git a/packages/eve-bench/src/harnesses/eve/runner.mjs b/packages/eve-bench/src/harnesses/eve/runner.mjs new file mode 100644 index 0000000000..6a16082445 --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/runner.mjs @@ -0,0 +1,226 @@ +// Runs inside the task container: boots the built eve server, sends the task +// instruction as one session, follows the event stream to its boundary, and +// writes /logs/agent/{events.ndjson,result.json,server.log}. Events are +// appended as they arrive so the host can read usage after an agent timeout. +// Talks to the server over plain HTTP so the bundle ships no node_modules. +import { spawn } from "node:child_process"; +import { appendFileSync, createWriteStream, writeFileSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const projectDir = requireEnv("EVE_PROJECT_DIR"); +const instructionPath = requireEnv("EVE_INSTRUCTION_PATH"); +const logsDir = process.env.EVE_AGENT_LOGS_DIR ?? "/logs/agent"; +const host = process.env.EVE_HOST ?? "127.0.0.1"; +const port = Number(process.env.EVE_PORT ?? "3024"); +const serverUrl = `http://${host}:${port}`; +const startupTimeoutMs = Number(process.env.EVE_STARTUP_TIMEOUT_MS ?? "60000"); + +await mkdir(logsDir, { recursive: true }); +const eventsPath = join(logsDir, "events.ndjson"); +const resultPath = join(logsDir, "result.json"); +const serverLogPath = join(logsDir, "server.log"); + +let server; +let wroteResult = false; +try { + server = startServer(); + await waitForHealth(server); + + const instruction = await readFile(instructionPath, "utf8"); + const sessionId = await createSession(instruction); + writeFileSync(eventsPath, ""); + const events = await followStream(sessionId); + + const status = statusFromEvents(events); + const summary = { + status, + sessionId, + message: finalMessage(events), + inputRequests: events + .filter((event) => event.type === "input.requested") + .flatMap((event) => event.data?.requests ?? []), + eventCount: events.length, + failure: failureFromEvents(events) ?? failureFromStatus(status), + }; + await writeFile(resultPath, `${JSON.stringify(summary, null, 2)}\n`); + wroteResult = true; + if (summary.failure) throw new Error(summary.failure.message); +} catch (error) { + if (!wroteResult) { + await writeFile( + resultPath, + `${JSON.stringify( + { + status: "failed", + inputRequests: [], + eventCount: 0, + failure: { type: "runner.error", message: error?.message ?? String(error) }, + }, + null, + 2, + )}\n`, + ); + } + throw error; +} finally { + if (server) await stopServer(server); +} + +function requireEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} must be set`); + return value; +} + +function startServer() { + const log = createWriteStream(serverLogPath, { flags: "a" }); + const entry = join(projectDir, ".output", "server", "index.mjs"); + const serverEnv = { + EVE_DEV: "1", + HOST: host, + NITRO_HOST: host, + NITRO_PORT: String(port), + PORT: String(port), + }; + // Agent tools restore these so task commands see the container's own values. + const taskEnv = Object.fromEntries( + Object.keys(serverEnv).map((key) => [key, process.env[key] ?? null]), + ); + const child = spawn(process.execPath, [entry], { + cwd: projectDir, + env: { + ...process.env, + ...serverEnv, + EVE_BENCH_TASK_ENV_RESTORE: JSON.stringify(taskEnv), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + child.once("exit", (code, signal) => { + log.write(`\n[eve-server exited code=${code ?? ""} signal=${signal ?? ""}]\n`); + }); + return child; +} + +async function waitForHealth(child) { + const deadline = Date.now() + startupTimeoutMs; + let lastError; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`eve server exited before becoming healthy (code ${child.exitCode})`); + } + try { + const response = await fetch(`${serverUrl}/eve/v1/health`); + if (response.ok) return; + lastError = new Error(`health returned ${response.status}`); + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for eve health: ${lastError?.message ?? "unknown"}`); +} + +async function createSession(message) { + const response = await fetch(`${serverUrl}/eve/v1/session`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message }), + }); + if (!response.ok) + throw new Error(`session create failed: ${response.status} ${await response.text()}`); + const payload = await response.json(); + const sessionId = payload?.sessionId ?? response.headers.get("x-eve-session-id"); + if (!sessionId) throw new Error("session create returned no session id"); + return sessionId; +} + +/** Reads the durable NDJSON stream until a session boundary, reconnecting from the cursor if the transport ends first. */ +async function followStream(sessionId) { + const events = []; + for (;;) { + const url = new URL(`${serverUrl}/eve/v1/session/${encodeURIComponent(sessionId)}/stream`); + if (events.length > 0) url.searchParams.set("startIndex", String(events.length)); + const response = await fetch(url); + if (!response.ok || !response.body) + throw new Error(`stream failed: ${response.status} ${await response.text()}`); + for await (const event of ndjson(response.body)) { + events.push(event); + appendFileSync(eventsPath, `${JSON.stringify(event)}\n`); + if (isBoundary(event)) return events; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } +} + +async function* ndjson(body) { + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of body) { + buffer += decoder.decode(chunk, { stream: true }); + let newline; + while ((newline = buffer.indexOf("\n")) !== -1) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) yield JSON.parse(line); + } + } + if (buffer.trim()) yield JSON.parse(buffer); +} + +function isBoundary(event) { + return ( + event.type === "session.completed" || + event.type === "session.failed" || + event.type === "session.waiting" + ); +} + +function statusFromEvents(events) { + const boundary = events.findLast(isBoundary); + if (boundary?.type === "session.completed") return "completed"; + if (boundary?.type === "session.waiting") return "waiting"; + if (boundary?.type === "session.failed") return "failed"; + return "unknown"; +} + +function finalMessage(events) { + return events.findLast( + (event) => event.type === "message.completed" && event.data?.finishReason !== "tool-calls", + )?.data?.message; +} + +function failureFromEvents(events) { + for (const event of events) { + if (event.type === "input.requested") { + return { + type: event.type, + message: "eve requested human input during a non-interactive benchmark run", + data: event.data, + }; + } + if (event.type === "turn.failed" || event.type === "session.failed") { + const message = + event.data?.message ?? event.data?.error?.message ?? `${event.type} emitted during eve run`; + return { type: event.type, message, data: event.data }; + } + } + return null; +} + +function failureFromStatus(status) { + if (status === "waiting" || status === "completed") return null; + return { type: "session.unsettled", message: `eve run ended with status: ${status}` }; +} + +async function stopServer(child) { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", resolve)), + new Promise((resolve) => setTimeout(resolve, 5000)), + ]); + if (child.exitCode === null) child.kill("SIGKILL"); +} diff --git a/packages/eve-bench/src/harnesses/eve/usage.ts b/packages/eve-bench/src/harnesses/eve/usage.ts new file mode 100644 index 0000000000..51d3bfef86 --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/usage.ts @@ -0,0 +1,22 @@ +import { join } from "node:path"; + +import type { Usage } from "../../core/result.ts"; +import { count, readJsonlUsage, record, type UsageRecord } from "../../core/usage.ts"; + +/** Sums `step.completed` usage from the runner's incrementally written event log. */ +export function readEveUsage(agentLogsDir: string): Promise { + return readJsonlUsage(join(agentLogsDir, "events.ndjson"), eveStepUsage); +} + +export function eveStepUsage(event: unknown): UsageRecord | undefined { + const value = record(event); + if (value.type !== "step.completed") return undefined; + const data = record(value.data); + const usage = record(data.usage ?? data); + return { + input: count(usage.inputTokens, usage.input_tokens), + output: count(usage.outputTokens, usage.output_tokens), + cached: count(usage.cacheReadTokens, usage.cachedTokens, usage.cached_tokens), + cost: count(usage.costUsd, usage.cost_usd), + }; +} diff --git a/packages/eve-bench/src/harnesses/oracle.ts b/packages/eve-bench/src/harnesses/oracle.ts new file mode 100644 index 0000000000..c8e764f2c7 --- /dev/null +++ b/packages/eve-bench/src/harnesses/oracle.ts @@ -0,0 +1,24 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import type { Harness } from "../core/harness.ts"; + +/** + * Runs each task's reference `solution/solve.sh`. Validates the runner and + * verifier end to end without a model, and gives the reward ceiling a real + * harness is measured against. + */ +export function createOracleHarness(): Harness { + return { + name: "oracle", + modelFree: true, + async prepare(ctx) { + const dir = join(ctx.cacheDir, "bundles", "oracle"); + await mkdir(dir, { recursive: true }); + return { dir }; + }, + stage: (ctx) => [join(ctx.taskDir, "solution")], + command: (ctx) => `bash ${ctx.installDir}/solution/solve.sh`, + env: () => ({}), + }; +} diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts new file mode 100644 index 0000000000..5fe4f97c25 --- /dev/null +++ b/packages/eve-bench/src/options.ts @@ -0,0 +1,125 @@ +import { readdir } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts"; +import type { Harness } from "./core/harness.ts"; +import { createEveHarness } from "./harnesses/eve/index.ts"; +import { createOracleHarness } from "./harnesses/oracle.ts"; +import { createCliHarness } from "./harnesses/cli/index.ts"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +export const paths = { + packageRoot, + datasetsRoot: join(packageRoot, "datasets"), + generatedRoot: join(packageRoot, ".generated"), +} as const; + +export interface TaskSelection { + readonly lock: DatasetLock; + readonly tasks: readonly string[]; + readonly taskDirs: readonly string[]; +} + +export async function selectTasks(input: { + cohort?: string; + dataset?: string; + task?: readonly string[]; + taskDir?: readonly string[]; +}): Promise { + const localTaskDirs = input.taskDir?.map((dir) => resolve(dir)) ?? []; + if (localTaskDirs.length > 0 && !input.cohort && !input.dataset && !input.task?.length) { + return { + lock: { name: "local", version: "0", commit: "", gitUrl: "", tasks: [] }, + tasks: [], + taskDirs: localTaskDirs, + }; + } + + if (input.cohort) { + const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); + const entry = await lock(cohort.dataset); + const tasks = input.task?.length + ? cohort.tasks.filter((task) => input.task!.includes(task)) + : cohort.tasks; + return { lock: entry, tasks, taskDirs: localTaskDirs }; + } + + const entry = await lock(input.dataset ?? (await defaultDataset())); + const tasks = input.task?.length + ? entry.tasks.filter((task) => input.task!.includes(task)) + : entry.tasks; + if (input.task?.length && tasks.length !== input.task.length) { + const missing = input.task.filter((task) => !tasks.includes(task)); + throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); + } + return { lock: entry, tasks, taskDirs: localTaskDirs }; +} + +export function jobDir(nameOrPath: string): string { + return nameOrPath.includes("/") + ? resolve(nameOrPath) + : join(paths.generatedRoot, "jobs", nameOrPath); +} + +export async function lock(name: string): Promise { + return readDatasetLock(join(paths.datasetsRoot, `${name}.json`)); +} + +export async function allLocks(): Promise { + const files = (await readdir(paths.datasetsRoot)).filter((file) => file.endsWith(".json")); + return Promise.all(files.map((file) => readDatasetLock(join(paths.datasetsRoot, file)))); +} + +export function defaultJobName(label: string): string { + const stamp = new Date().toISOString().replaceAll(/[:.]/gu, "-").slice(0, 19); + return `${stamp}-${label.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}`; +} + +export function selectHarness( + name: string, + eve: string, + options: { version?: string; baseUrl?: string; reasoning?: string } = {}, +): Harness { + if (name === "pi" || name === "opencode" || name === "codex" || name === "hermes") { + return createCliHarness(name, { + version: options.version ?? "", + baseUrl: options.baseUrl, + reasoning: options.reasoning, + }); + } + if (options.version || options.baseUrl || options.reasoning) + throw new Error("--version, --base-url and --reasoning require a supporting harness"); + if (name === "oracle") return createOracleHarness(); + if (name === "eve") { + return createEveHarness( + eve === "local" ? { kind: "local" } : { kind: "release", version: eve }, + ); + } + throw new Error(`unknown harness: ${name}`); +} + +/** Host credentials the harness may receive; fails before any trial when none are exported. */ +export function forwardedEnv(harness: Harness): Record { + const names = harness.credentials ?? []; + const env: Record = {}; + for (const key of names) { + const value = process.env[key]; + if (value) env[key] = value; + } + if (names.length > 0 && Object.keys(env).length === 0) { + throw new Error( + `No model credentials exported for ${harness.name}. Set one of ${names.join(", ")} (AI_GATEWAY_API_KEY for gateway models); use prepare for a model-free build.`, + ); + } + return env; +} + +async function defaultDataset(): Promise { + const locks = await allLocks(); + if (locks.length !== 1) { + throw new Error("pass --dataset or --cohort; multiple datasets are available"); + } + return `${locks[0]!.name}-${locks[0]!.version}`; +} diff --git a/packages/eve-bench/src/tools/diff_bench.ts b/packages/eve-bench/src/tools/diff_bench.ts new file mode 100644 index 0000000000..ff59f82988 --- /dev/null +++ b/packages/eve-bench/src/tools/diff_bench.ts @@ -0,0 +1,25 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { diffJobs } from "../core/report.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ + base: z.string().min(1), + candidate: z.string().min(1), +}); + +export default defineTool({ + description: "Compare task rewards between two eve benchmark jobs.", + inputSchema, + approval: never(), + async execute(input) { + const [base, candidate] = await Promise.all([ + readJobResult(jobDir(input.base)), + readJobResult(jobDir(input.candidate)), + ]); + return diffJobs(base, candidate); + }, +}); diff --git a/packages/eve-bench/src/tools/get_bench_result.ts b/packages/eve-bench/src/tools/get_bench_result.ts new file mode 100644 index 0000000000..a3ff347ffc --- /dev/null +++ b/packages/eve-bench/src/tools/get_bench_result.ts @@ -0,0 +1,24 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ job: z.string().min(1) }); + +export default defineTool({ + description: "Read a completed eve benchmark job result.", + inputSchema, + approval: never(), + async execute(input) { + try { + return await readJobResult(jobDir(input.job)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`benchmark result not found for job "${input.job}"`); + } + throw error; + } + }, +}); diff --git a/packages/eve-bench/src/tools/list_bench_tasks.ts b/packages/eve-bench/src/tools/list_bench_tasks.ts new file mode 100644 index 0000000000..ad2460a546 --- /dev/null +++ b/packages/eve-bench/src/tools/list_bench_tasks.ts @@ -0,0 +1,19 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { selectTasks } from "../options.ts"; + +export const inputSchema = z.object({ + cohort: z.string().min(1).optional(), + dataset: z.string().min(1).optional(), +}); + +export default defineTool({ + description: "List task names in an eve benchmark dataset or cohort.", + inputSchema, + approval: never(), + async execute(input) { + return (await selectTasks(input)).tasks; + }, +}); diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts new file mode 100644 index 0000000000..8b63a2c7c4 --- /dev/null +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -0,0 +1,62 @@ +import { join } from "node:path"; + +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { syncDataset, taskDirs } from "../core/dataset.ts"; +import { runJob } from "../core/job.ts"; +import { loadTask } from "../core/task.ts"; +import { defaultJobName, forwardedEnv, paths, selectHarness, selectTasks } from "../options.ts"; + +export const inputSchema = z + .object({ + cohort: z.string().min(1).optional(), + tasks: z.array(z.string().min(1)).min(1).optional(), + model: z.string().min(1).optional(), + harness: z.enum(["eve", "oracle", "pi", "opencode", "codex", "hermes"]).default("eve"), + version: z.string().min(1).optional(), + baseUrl: z.string().url().optional(), + reasoning: z.string().min(1).optional(), + eve: z.string().min(1).default("local"), + attempts: z.number().int().positive().default(1), + concurrency: z.number().int().positive().default(4), + job: z.string().min(1).optional(), + }) + .superRefine((input, ctx) => { + if (input.harness !== "oracle" && !input.model) { + ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); + } + if (["pi", "opencode", "codex", "hermes"].includes(input.harness) && !input.version) { + ctx.addIssue({ code: "custom", path: ["version"], message: "version is required" }); + } + }); + +export default defineTool({ + description: "Run an eve benchmark job and return its result.", + inputSchema, + async execute(input, ctx) { + const harness = selectHarness(input.harness, input.eve, input); + const forwardEnv = forwardedEnv(harness); + const model = input.model ?? "none"; + const selection = await selectTasks({ cohort: input.cohort, task: input.tasks }); + const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const tasks = await Promise.all( + (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + ); + const name = input.job ?? defaultJobName(`${harness.name}-${model}`); + + return runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks, + harness, + model, + attempts: input.attempts, + concurrency: input.concurrency, + forwardEnv, + signal: ctx.abortSignal, + }); + }, +}); diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts new file mode 100644 index 0000000000..7def3079d6 --- /dev/null +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { inputSchema as diffBenchInput } from "./diff_bench.ts"; +import { inputSchema as getBenchResultInput } from "./get_bench_result.ts"; +import { inputSchema as listBenchTasksInput } from "./list_bench_tasks.ts"; +import { inputSchema as runBenchInput } from "./run_bench.ts"; + +describe("run_bench input", () => { + it("accepts valid eve and oracle inputs", () => { + assert.equal( + runBenchInput.safeParse({ + cohort: "smoke", + tasks: ["fix-git"], + model: "zai/glm-5.2", + harness: "eve", + eve: "local", + attempts: 2, + concurrency: 1, + job: "t1", + }).success, + true, + ); + assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); + assert.equal( + runBenchInput.safeParse({ + harness: "pi", + version: "0.80.3", + model: "deepseek/deepseek-v4-pro", + }).success, + true, + ); + }); + + it("rejects invalid inputs", () => { + assert.equal(runBenchInput.safeParse({}).success, false); + assert.equal(runBenchInput.safeParse({ harness: "other", model: "model" }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", attempts: 0 }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", tasks: "fix-git" }).success, false); + assert.equal( + runBenchInput.safeParse({ harness: "e0", model: "openai/gpt-5.6-terra" }).success, + false, + "e0 is not a supported harness", + ); + assert.equal( + runBenchInput.safeParse({ harness: "codex", model: "openai/gpt-5.6-terra" }).success, + false, + ); + }); +}); + +describe("get_bench_result input", () => { + it("accepts a job name", () => { + assert.equal(getBenchResultInput.safeParse({ job: "smoke-run" }).success, true); + }); + + it("rejects a missing or empty job name", () => { + assert.equal(getBenchResultInput.safeParse({}).success, false); + assert.equal(getBenchResultInput.safeParse({ job: "" }).success, false); + }); +}); + +describe("diff_bench input", () => { + it("accepts base and candidate jobs", () => { + assert.equal( + diffBenchInput.safeParse({ base: "base-run", candidate: "candidate-run" }).success, + true, + ); + }); + + it("rejects incomplete input", () => { + assert.equal(diffBenchInput.safeParse({ base: "base-run" }).success, false); + assert.equal(diffBenchInput.safeParse({ base: 1, candidate: "candidate-run" }).success, false); + }); +}); + +describe("list_bench_tasks input", () => { + it("accepts empty, cohort, and dataset selections", () => { + assert.equal(listBenchTasksInput.safeParse({}).success, true); + assert.equal(listBenchTasksInput.safeParse({ cohort: "smoke" }).success, true); + assert.equal(listBenchTasksInput.safeParse({ dataset: "terminal-bench-2.0" }).success, true); + }); + + it("rejects invalid selections", () => { + assert.equal(listBenchTasksInput.safeParse({ cohort: "" }).success, false); + assert.equal(listBenchTasksInput.safeParse({ dataset: ["terminal-bench-2.0"] }).success, false); + }); +}); diff --git a/packages/eve-bench/test/posix-only.ts b/packages/eve-bench/test/posix-only.ts new file mode 100644 index 0000000000..a8bc93183c --- /dev/null +++ b/packages/eve-bench/test/posix-only.ts @@ -0,0 +1,12 @@ +// eve-bench drives Linux task containers, and its integration tests stub POSIX +// tools (sh, shebang scripts, process groups), so they cannot run on Windows hosts. +import { spawnSync } from "node:child_process"; + +if (process.platform === "win32") { + console.log("Skipping eve-bench integration tests: they require a POSIX host."); + process.exit(0); +} +const { status } = spawnSync(process.execPath, ["--test", ...process.argv.slice(2)], { + stdio: "inherit", +}); +process.exit(status ?? 1); diff --git a/packages/eve-bench/test/scenarios/cli-adapters.test.ts b/packages/eve-bench/test/scenarios/cli-adapters.test.ts new file mode 100644 index 0000000000..3d10c8af26 --- /dev/null +++ b/packages/eve-bench/test/scenarios/cli-adapters.test.ts @@ -0,0 +1,320 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { createCliHarness } from "../../src/harnesses/cli/index.ts"; +import { shellQuote, type SupportedCliName } from "../../src/harnesses/cli/plan.ts"; +import type { Harness, HarnessBundle, HarnessRunContext } from "../../src/core/harness.ts"; + +const exec = promisify(execFile); +const sourceDir = fileURLToPath(new URL("../../src/harnesses/cli/", import.meta.url)); +const FAKE_KEY = 'model-free-fake-key-"abcdef'; +const TASK_HOME = "/task-home"; + +async function temp(t: TestContext): Promise { + const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-cli-"))); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} + +const mockCli = `import { readFile } from 'node:fs/promises'; +const args = process.argv.slice(2); +if (args[0] === '--version') { + if (process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY) process.exit(92); + console.log('mock-cli 1.2.3'); +} else { + const name = JSON.parse(await readFile(new URL('./identity.json', import.meta.url), 'utf8')).name; + const path = name === 'pi' ? process.env.PI_CODING_AGENT_DIR + '/models.json' + : name === 'codex' ? process.env.CODEX_HOME + '/config.toml' + : process.env.OPENCODE_CONFIG; + const config = await readFile(path, 'utf8'); + let input = ''; + for await (const chunk of process.stdin) input += chunk.toString(); + const prompt = name === 'pi' ? input : args.at(-1); + if (name === 'pi' && args.includes('--')) throw new Error('pi does not support a -- separator'); + console.log(JSON.stringify({ type: 'mock', args, prompt, cwd: process.cwd(), config, + env: { PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR, CODEX_HOME: process.env.CODEX_HOME, + OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, OPENCODE_FAKE_VCS: process.env.OPENCODE_FAKE_VCS, + HOME: process.env.HOME, PATH: process.env.PATH, TASK_IMAGE_ENV: process.env.TASK_IMAGE_ENV, + XDG_DATA_HOME: process.env.XDG_DATA_HOME, runnerContext: process.env.EVE_BENCH_CLI_CONTEXT }, + credentialMapped: process.env.AI_GATEWAY_API_KEY === ${JSON.stringify(FAKE_KEY)} && process.env.OPENAI_API_KEY === process.env.AI_GATEWAY_API_KEY })); + const key = process.env.AI_GATEWAY_API_KEY; + process.stdout.write(key.slice(0, 12)); + await new Promise(r => setTimeout(r, 10)); + process.stdout.write(key.slice(12) + '\\n'); + process.stderr.write(JSON.stringify(key) + '\\n'); + if (prompt === '__opencode_error__') process.stdout.write(JSON.stringify({ type: 'error', error: 'mock failure' })); + if (prompt === '__exit_7__') process.exitCode = 7; + if (prompt === '__signal__') process.kill(process.pid, 'SIGTERM'); +} +`; + +async function mockInstalled(t: TestContext, name: SupportedCliName) { + const root = await temp(t); + const installDir = join(root, "install with 'quote"); + await mkdir(join(installDir, "arch"), { recursive: true }); + for (const file of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(sourceDir, file), join(installDir, file)); + await writeFile( + join(installDir, "arch/node"), + `#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`, + { mode: 0o755 }, + ); + await writeFile(join(installDir, "arch/mock.mjs"), mockCli); + await writeFile(join(installDir, "arch/identity.json"), JSON.stringify({ name })); + await writeFile(join(installDir, "arch/entry.json"), JSON.stringify({ entry: "mock.mjs" })); + const ctx = { + model: "deepseek/deepseek-v4", + installDir, + instructionPath: join(root, "instruction with 'quote.md"), + taskWorkdir: join(root, "task with space"), + logsDir: join(root, "logs"), + }; + await mkdir(ctx.taskWorkdir); + return { root, ctx, harness: createCliHarness(name, { version: "1.2.3", reasoning: "high" }) }; +} + +async function invoke( + harness: Harness, + ctx: HarnessRunContext, + instruction: string, + key: string | undefined = FAKE_KEY, +) { + await writeFile(ctx.instructionPath, instruction); + try { + const result = await exec("sh", ["-c", harness.command(ctx)], { + env: { + PATH: process.env.PATH, + HOME: TASK_HOME, + TASK_IMAGE_ENV: "from-image", + AI_GATEWAY_API_KEY: key, + ...harness.env(ctx), + }, + maxBuffer: 4 * 1024 * 1024, + }); + return { code: 0, ...result }; + } catch (error) { + const failed = error as Error & { code: number; stdout: string; stderr: string }; + return { code: failed.code, stdout: failed.stdout, stderr: failed.stderr }; + } +} + +for (const name of ["pi", "opencode", "codex"] as const) { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6"]) { + test(`${name} mock binary receives native flags, full ${model}, safe config and runtime credentials`, async (t) => { + const { ctx, harness } = await mockInstalled(t, name); + ctx.model = model; + const prompt = "--untrusted 'quote'\n$(touch SHOULD_NOT_EXIST)"; + const result = await invoke(harness, ctx, prompt); + assert.equal(result.code, 0, result.stderr); + const log = await readFile(join(ctx.logsDir, `${name}.jsonl`), "utf8"); + const record = JSON.parse(log.split("\n")[0]!); + assert.equal(record.cwd, ctx.taskWorkdir); + assert.equal(record.prompt, prompt); + assert.equal(record.credentialMapped, true); + // Task-side state must stay where the verifier will look for it. + assert.equal(record.env.HOME, TASK_HOME); + assert.equal(record.env.PATH, process.env.PATH); + assert.equal(record.env.TASK_IMAGE_ENV, "from-image"); + assert.equal(record.env.XDG_DATA_HOME, undefined); + assert.equal(record.env.runnerContext, undefined); + assert.ok( + record.args.includes(model) || record.args.includes(`--model=eve-bench-gateway/${model}`), + ); + assert.match(record.config, /ai-gateway.vercel.sh/); + if (name === "pi") { + assert.equal(JSON.parse(record.config).providers["eve-bench-gateway"].models[0].id, model); + assert.match(record.config, /\$AI_GATEWAY_API_KEY/); + } else if (name === "opencode") { + assert.ok(JSON.parse(record.config).provider["eve-bench-gateway"].models[model]); + assert.equal(record.env.OPENCODE_FAKE_VCS, "git"); + } else { + assert.match(record.config, /wire_api = "responses"/); + assert.match(record.config, /env_key = "AI_GATEWAY_API_KEY"/); + } + for (const file of await readdir(ctx.logsDir)) { + const content = await readFile(join(ctx.logsDir, file), "utf8"); + assert.ok(!content.includes(FAKE_KEY)); + assert.ok(!content.includes(JSON.stringify(FAKE_KEY).slice(1, -1))); + } + assert.ok(!result.stdout.includes(FAKE_KEY)); + assert.ok(!result.stderr.includes(FAKE_KEY)); + assert.match(log, /\[REDACTED\]/); + assert.deepEqual(await readdir(ctx.taskWorkdir), []); + assert.ok(!(await readdir(ctx.installDir)).some((file) => file.startsWith(".cli-runtime-"))); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.installLocation, "host"); + assert.equal(metrics.containerInstallMs, 0); + assert.ok(metrics.setupMs >= 0); + assert.ok(metrics.agentMs > 0); + assert.equal(metrics.exitCode, 0); + }); + } +} + +test("native nonzero exits, signals, and OpenCode zero-exit error events fail the harness", async (t) => { + for (const [name, instruction, expected] of [ + ["pi", "__exit_7__", 7], + ["codex", "__signal__", 1], + ["opencode", "__opencode_error__", 1], + ] as const) { + const { ctx, harness } = await mockInstalled(t, name); + const result = await invoke(harness, ctx, instruction); + assert.equal(result.code, expected); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.exitCode, expected); + if (instruction === "__signal__") assert.equal(metrics.signal, "SIGTERM"); + if (instruction === "__opencode_error__") assert.match(metrics.failure, /error event/); + } +}); + +test("missing key, oversized prompt, and version mismatch fail before the agent starts", async (t) => { + const { ctx, harness } = await mockInstalled(t, "pi"); + const missing = await invoke(harness, ctx, "do not run", ""); + assert.equal(missing.code, 1); + assert.match(missing.stderr, /AI_GATEWAY_API_KEY must be forwarded/); + assert.deepEqual(await readdir(ctx.logsDir), ["cli-runtime.json"]); + const oversized = await invoke(harness, ctx, "x".repeat(1024 * 1024 + 1)); + assert.equal(oversized.code, 1); + assert.match(oversized.stderr, /exceeds 1 MiB/); + const wrong = await invoke(createCliHarness("pi", { version: "1.2.30" }), ctx, "do not run"); + assert.equal(wrong.code, 1); + assert.match(wrong.stderr, /failed its --version check/); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.agentMs, 0); +}); + +test("host prepare uses pinned pnpm installs, verifies archives, caches across models, and relocates", async (t) => { + const root = await temp(t); + const bin = join(root, "bin"); + await mkdir(bin); + const calls = join(root, "pnpm-calls.jsonl"); + const shebang = `#!${process.execPath}\n`; + await writeFile( + join(bin, "pnpm"), + shebang + + ` +import { readFile, writeFile, mkdir, appendFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +if (process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY || process.env.UNRELATED_TEST_API_KEY) process.exit(91); +if (args[0] === '--version') { console.log('11.21.0'); } +else { + await appendFile(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); + const manifest = JSON.parse(await readFile('package.json', 'utf8')); + const [pkg, version] = Object.entries(manifest.dependencies)[0]; + const dir = join(process.cwd(), 'node_modules', pkg); + const name = pkg.includes('pi-coding-agent') ? 'pi' : pkg === 'opencode-ai' ? 'opencode' : 'codex'; + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'package.json'), JSON.stringify({ version, bin: { [name]: 'cli.mjs' } })); + await writeFile(join(dir, 'cli.mjs'), ${JSON.stringify(mockCli)}); + await writeFile(join(dir, 'identity.json'), JSON.stringify({ name })); + await writeFile('pnpm-lock.yaml', 'lockfileVersion: 9.0\\n'); +} +`, + { mode: 0o755 }, + ); + await writeFile( + join(bin, "tar"), + shebang + + ` +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +await writeFile(join(args[args.indexOf('-C') + 1], 'node'), ${JSON.stringify(`#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`)}); +`, + { mode: 0o755 }, + ); + const originalPath = process.env.PATH; + const originalKey = process.env.AI_GATEWAY_API_KEY; + process.env.PATH = `${bin}:${originalPath}`; + process.env.AI_GATEWAY_API_KEY = FAKE_KEY; + t.after(() => { + process.env.PATH = originalPath; + if (originalKey === undefined) delete process.env.AI_GATEWAY_API_KEY; + else process.env.AI_GATEWAY_API_KEY = originalKey; + }); + const archive = Buffer.from("mock node archive"); + const hash = createHash("sha256").update(archive).digest("hex"); + let fetches = 0; + t.mock.method(globalThis, "fetch", async (url: string) => { + fetches++; + assert.match(url, /^https:\/\/nodejs.org\/dist\/v24\.19\.0\//); + return new Response( + url.endsWith("SHASUMS256.txt") + ? `${hash} node-v24.19.0-linux-x64.tar.xz\n${hash} node-v24.19.0-linux-arm64.tar.xz\n` + : archive, + ); + }); + const harness = createCliHarness("pi", { version: "1.2.3" }); + const cacheDir = join(root, "cache"); + const bundle = await harness.prepare({ model: "deepseek/deepseek-v4", cacheDir }); + assert.equal(fetches, 4); + const sameBundle = await harness.prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(sameBundle.dir, bundle.dir); + assert.equal(bundle.provenance?.model, "deepseek/deepseek-v4"); + assert.equal(sameBundle.provenance?.model, "openai/gpt-5.6"); + const differentConfig = await createCliHarness("pi", { + version: "1.2.3", + reasoning: "high", + baseUrl: "https://gateway.example.test/v1", + }).prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(differentConfig.dir, bundle.dir); + assert.equal(differentConfig.provenance?.reasoning, "high"); + assert.equal(differentConfig.provenance?.baseUrl, "https://gateway.example.test/v1"); + assert.equal(fetches, 4); + const installs = (await readFile(calls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + assert.equal(installs.length, 2); + for (const [i, args] of installs.entries()) { + assert.ok(args.includes("--ignore-scripts")); + assert.ok(args.includes("--ignore-workspace")); + assert.ok(args.includes("--os=linux")); + assert.ok(args.includes(`--cpu=${i === 0 ? "x64" : "arm64"}`)); + assert.ok(!args.some((arg) => arg.includes("latest"))); + } + assert.equal(bundle.provenance?.installLocation, "host"); + assert.equal(bundle.provenance?.version, "1.2.3"); + assert.equal(bundle.provenance?.containerInstallMs, 0); + assert.match(JSON.stringify(bundle.provenance?.lockSha256), /[a-f0-9]{64}/); + assert.ok(!JSON.stringify(bundle.provenance).includes(FAKE_KEY)); + for (const arch of ["x64", "arm64"] as const) await assertRelocated(bundle, harness, root, arch); + + // A failed build must not publish a cache entry or retain partial build trees. + t.mock.method(globalThis, "fetch", async () => new Response("bad checksum")); + await assert.rejects( + createCliHarness("codex", { version: "1.2.3" }).prepare({ model: "openai/gpt-5.6", cacheDir }), + /checksum is missing/, + ); + assert.ok(!(await readdir(join(cacheDir, "cli"))).some((name) => name.startsWith(".prepare-"))); +}); + +async function assertRelocated( + bundle: HarnessBundle, + harness: Harness, + root: string, + arch: "x64" | "arm64", +) { + const installDir = join(root, `relocated ${arch}`); + await cp(bundle.dir, installDir, { recursive: true }); + await cp(bundle.arch![arch], join(installDir, "arch"), { recursive: true }); + const ctx = { + model: "openai/gpt-5.6", + installDir, + taskWorkdir: root, + instructionPath: join(root, `${arch}-instruction.md`), + logsDir: join(root, `${arch}-logs`), + }; + const result = await invoke(harness, ctx, "mock relocation"); + assert.equal(result.code, 0, result.stderr); + assert.match(await readFile(join(ctx.logsDir, "pi.jsonl"), "utf8"), /mock relocation/); + assert.equal(dirname(bundle.dir), dirname(bundle.arch![arch])); +} diff --git a/packages/eve-bench/test/scenarios/cli-report.test.ts b/packages/eve-bench/test/scenarios/cli-report.test.ts new file mode 100644 index 0000000000..27db41d22e --- /dev/null +++ b/packages/eve-bench/test/scenarios/cli-report.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { test, type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { summarize, type JobResult, type TrialResult } from "../../src/core/result.ts"; + +const exec = promisify(execFile); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +function trial(attempt: number, extra: Partial = {}): TrialResult { + return { + task: "a", + attempt, + harness: "pi@0.85.1", + model: "test/model", + reward: 0, + agent: { status: "completed", exitCode: 0, durationMs: 1 }, + verifier: { status: "completed", exitCode: 0, durationMs: 1 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + ...extra, + }; +} + +async function report(t: TestContext, trials: TrialResult[], ...args: string[]) { + const job = `cli-report-test-${randomUUID().slice(0, 8)}`; + const dir = join(packageRoot, ".generated", "jobs", job); + await mkdir(dir, { recursive: true }); + t.after(() => rm(dir, { recursive: true, force: true })); + const result: JobResult = { + job, + dataset: { name: "local", version: "0", commit: "" }, + harness: "pi@0.85.1", + model: "test/model", + trials, + summary: summarize(trials), + }; + await writeFile(join(dir, "result.json"), JSON.stringify(result)); + return exec(process.execPath, ["src/cli.ts", "report", job, ...args], { cwd: packageRoot }).then( + ({ stdout, stderr }) => ({ code: 0, stdout, stderr }), + (error: { code: number; stdout: string; stderr: string }) => error, + ); +} + +test("report --fail-on-invalid exits 1 and names each invalid attempt", async (t) => { + const result = await report( + t, + [ + trial(1), + trial(2, { invalid: { phase: "harness", reason: "harness reported zero token usage" } }), + ], + "--fail-on-invalid", + ); + assert.equal(result.code, 1); + assert.match(result.stdout, /a #2 \[harness\] harness reported zero token usage/); + assert.match(result.stderr, /1 of 2 attempt\(s\) are invalid/); +}); + +test("report --fail-on-invalid passes when every attempt was scored, including failures", async (t) => { + const result = await report(t, [trial(1), trial(2, { reward: 1 })], "--fail-on-invalid"); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /scored 2 {2}resolved 1/); +}); diff --git a/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts b/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts new file mode 100644 index 0000000000..b7abe10b5f --- /dev/null +++ b/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import bash from "../../agent/tools/bash.ts"; +import grep from "../../agent/tools/grep.ts"; + +interface CommandResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +type Executable = { + execute: (input: never, ctx: { abortSignal: AbortSignal }) => Promise; +}; + +function run( + tool: unknown, + input: Record, + signal = new AbortController().signal, +): Promise { + return (tool as Executable).execute(input as never, { + abortSignal: signal, + }) as Promise; +} + +/** Points the tools at a fresh task directory and restores the process env afterwards. */ +async function taskEnvironment(t: TestContext, env: Record = {}): Promise { + const workdir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-tools-"))); + const saved = { ...process.env }; + Object.assign(process.env, { EVE_BENCH_TASK_WORKDIR: workdir, ...env }); + t.after(async () => { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]; + Object.assign(process.env, saved); + await rm(workdir, { recursive: true, force: true }); + }); + return workdir; +} + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test("bash runs in the task directory and passes exit codes and streams through", async (t) => { + const workdir = await taskEnvironment(t); + assert.deepEqual(await run(bash, { command: "pwd; echo err >&2; exit 7" }), { + exitCode: 7, + stdout: `${workdir}\n`, + stderr: "err\n", + }); +}); + +test("bash commands see the task's own PORT and HOST, not the eve server's", async (t) => { + // The runner overrides these for the server and records the container's values. + await taskEnvironment(t, { + PORT: "3024", + HOST: "127.0.0.1", + NITRO_PORT: "3024", + EVE_BENCH_TASK_ENV_RESTORE: JSON.stringify({ PORT: null, HOST: "task-host", NITRO_PORT: null }), + }); + const result = await run(bash, { + command: + 'echo "PORT=${PORT-unset} HOST=$HOST NITRO_PORT=${NITRO_PORT-unset} RESTORE=${EVE_BENCH_TASK_ENV_RESTORE-unset}"', + }); + assert.equal(result.stdout, "PORT=unset HOST=task-host NITRO_PORT=unset RESTORE=unset\n"); +}); + +test("bash timeout returns 124 with partial output and kills the whole process group", async (t) => { + await taskEnvironment(t); + const started = Date.now(); + const result = await run(bash, { + command: "sleep 30 & echo child=$!; sleep 30", + timeoutSeconds: 1, + }); + assert.ok(Date.now() - started < 5_000, `took ${Date.now() - started}ms`); + assert.equal(result.exitCode, 124); + assert.match(result.stderr, /stopped after 1 seconds/); + const child = Number(/child=(\d+)/.exec(result.stdout)?.[1]); + assert.ok(child > 0, result.stdout); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal(alive(child), false, "background child survived the timeout"); +}); + +test("bash returns when the shell exits even if a background job holds the output pipe", async (t) => { + await taskEnvironment(t); + const started = Date.now(); + const result = await run(bash, { command: "sleep 5 & echo started" }); + assert.ok(Date.now() - started < 2_000, `took ${Date.now() - started}ms`); + assert.deepEqual(result, { exitCode: 0, stdout: "started\n", stderr: "" }); +}); + +test("bash abort stops the command", async (t) => { + await taskEnvironment(t); + const controller = new AbortController(); + const started = Date.now(); + const pending = run(bash, { command: "sleep 30" }, controller.signal); + setTimeout(() => controller.abort(), 100); + const result = await pending; + assert.ok(Date.now() - started < 5_000, `took ${Date.now() - started}ms`); + assert.notEqual(result.exitCode, 0); +}); + +test("grep reports a missing ripgrep instead of returning no matches", async (t) => { + await taskEnvironment(t, { PATH: "/nonexistent" }); + await assert.rejects(run(grep, { pattern: "needle" }), /ripgrep \(rg\) is not installed/); +}); + +let hasRipgrep = true; +try { + execFileSync("rg", ["--version"], { stdio: "ignore" }); +} catch { + hasRipgrep = false; +} + +test( + "grep distinguishes matches from no matches", + { skip: !hasRipgrep && "rg not installed" }, + async (t) => { + const workdir = await taskEnvironment(t); + await writeFile(join(workdir, "notes.txt"), "alpha\nneedle here\n"); + const found = await run(grep, { pattern: "needle" }); + assert.equal(found.exitCode, 0); + assert.match(found.stdout, /notes\.txt:2:needle here/); + assert.deepEqual(await run(grep, { pattern: "absent" }), { + exitCode: 1, + stderr: "", + stdout: "", + }); + }, +); diff --git a/packages/eve-bench/test/scenarios/job-resume.test.ts b/packages/eve-bench/test/scenarios/job-resume.test.ts new file mode 100644 index 0000000000..bfefcba339 --- /dev/null +++ b/packages/eve-bench/test/scenarios/job-resume.test.ts @@ -0,0 +1,364 @@ +import assert from "node:assert/strict"; +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + symlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import { runJob, type JobInput } from "../../src/core/job.ts"; +import { hashTask, identityHash } from "../../src/core/job-identity.ts"; +import { loadTask } from "../../src/core/task.ts"; +import type { TrialResult } from "../../src/core/result.ts"; + +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), "eve-bench-resume-")); + const bin = join(root, "bin"); + const calls = join(root, "docker-calls.jsonl"); + const containers = join(root, "containers.json"); + const taskDir = join(root, "tasks", "a"); + await mkdir(bin); + for (const dir of ["environment", "tests", "solution"]) + await mkdir(join(taskDir, dir), { recursive: true }); + await writeFile(join(taskDir, "instruction.md"), "Do the task.\n"); + await writeFile(join(taskDir, "task.toml"), '[task]\nname = "a"\n'); + await writeFile(join(taskDir, "environment", "Dockerfile"), "FROM scratch\n"); + await writeFile(join(taskDir, "environment", "input.bin"), Buffer.from([0, 255, 1])); + await writeFile(join(taskDir, "tests", "test.sh"), "true\n"); + await writeFile(join(taskDir, "solution", "solve.sh"), "true\n"); + await writeFile(calls, ""); + await writeFile(containers, "[]"); + // No daemon or model is used: every Docker invocation terminates in this local stub. + await writeFile( + join(bin, "docker"), + `#!${process.execPath} +import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; +const args = process.argv.slice(2); +appendFileSync(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); +if (args[0] === 'version') console.log('fake-docker'); +else if (args[0] === 'ps') for (const row of JSON.parse(readFileSync(${JSON.stringify(containers)}, 'utf8'))) console.log(JSON.stringify(row)); +else if (args[0] === 'rm') writeFileSync(${JSON.stringify(containers)}, '[]'); +else if (args[0] === 'exec') console.log('/workspace'); +`, + { mode: 0o755 }, + ); + const previousPath = process.env.PATH; + process.env.PATH = `${bin}:${previousPath}`; + t.after(async () => { + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + await rm(root, { recursive: true, force: true }); + }); + const input: JobInput = { + name: basename(root), + dir: join(root, "job"), + cacheDir: join(root, "cache"), + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a"], + }, + tasks: [await loadTask(taskDir)], + harness: { + name: "fake", + prepare: async () => ({ + dir: bin, + provenance: { key: "v1", source: { sha: "abc", dirty: false } }, + }), + command: () => "true", + env: () => ({}), + }, + model: "no-model", + attempts: 1, + concurrency: 1, + forwardEnv: {}, + signal: new AbortController().signal, + }; + const trial: TrialResult = { + task: "a", + attempt: 1, + harness: "fake", + model: "no-model", + reward: 1, + agent: { status: "completed", exitCode: 0, durationMs: 1 }, + verifier: { status: "completed", exitCode: 0, durationMs: 1 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + }; + async function initialize() { + await runJob({ ...input, signal: AbortSignal.abort() }); + await mkdir(join(input.dir, "a", "1"), { recursive: true }); + await writeFile(join(input.dir, "a", "1", "trial.json"), JSON.stringify(trial)); + await writeFile(calls, ""); + } + async function leftovers() { + await writeFile( + containers, + JSON.stringify([ + { + ID: "leftover", + Names: "leftover", + Labels: `eve-bench=1,eve-bench.job=${input.name}`, + CreatedAt: "yesterday", + }, + ]), + ); + } + return { root, input, taskDir, calls, containers, trial, initialize, leftovers }; +} + +async function snapshot(dir: string) { + return Promise.all( + ["job.json", "result.json", "a/1/trial.json"].map((file) => readFile(join(dir, file), "utf8")), + ); +} + +test("every resume identity mismatch rejects before metadata/results overwrite or Docker cleanup", async (t) => { + const f = await fixture(t); + await f.initialize(); + await f.leftovers(); + const before = await snapshot(f.input.dir); + const changes: Partial[] = [ + { model: "different" }, + { attempts: 2 }, + { harness: { ...f.input.harness, name: "other" } }, + { + harness: { + ...f.input.harness, + prepare: async () => ({ dir: f.root, provenance: { key: "v2" } }), + }, + }, + { tasks: [{ ...f.input.tasks[0]!, name: "renamed" }] }, + { tasks: [] }, + { tasks: [...f.input.tasks, { ...f.input.tasks[0]!, name: "added" }] }, + { tasks: [{ ...f.input.tasks[0]!, instruction: "Loaded instruction override" }] }, + { tasks: [{ ...f.input.tasks[0]!, agentTimeoutMs: 123 }] }, + ...(["name", "version", "gitUrl", "commit"] as const).map((key) => ({ + dataset: { ...f.input.dataset, [key]: "changed" }, + })), + ]; + for (const change of changes) { + await assert.rejects( + runJob({ ...f.input, ...change }), + /resume identity differs.*new job name/, + ); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + } + for (const path of [ + "instruction.md", + "task.toml", + "tests/test.sh", + "environment/Dockerfile", + "environment/input.bin", + "solution/solve.sh", + ]) { + const file = join(f.taskDir, path); + const content = await readFile(file); + await writeFile(file, Buffer.concat([content, Buffer.from("changed")])); + await assert.rejects(runJob(f.input), /differs \(tasks\).*new job name/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + await writeFile(file, content); + } +}); + +test("matching resume keeps original metadata and cached trials, and then cleans leftovers", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + await f.leftovers(); + const seen: boolean[] = []; + const result = await runJob({ + ...f.input, + concurrency: 4, + forwardEnv: { FAKE_KEY: "not-identity-or-metadata" }, + harness: { + ...f.input.harness, + prepare: async () => ({ + dir: f.root, + provenance: { source: { dirty: false, sha: "abc" }, key: "v1" }, + }), + }, + onTrial: (_trial, resumed) => seen.push(resumed), + }); + assert.deepEqual(result.trials, [f.trial]); + assert.deepEqual(seen, [true]); + const after = await snapshot(f.input.dir); + assert.equal(after[0], before[0]); + assert.equal(after[2], before[2]); + assert.match(await readFile(f.calls, "utf8"), /\["rm","--force","--volumes","leftover"\]/); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["run"/); + assert.equal(await readFile(f.containers, "utf8"), "[]"); +}); + +test("legacy, malformed, unsupported and orphaned jobs refuse without preparation or Docker calls", async (t) => { + const f = await fixture(t); + await mkdir(f.input.dir); + let prepared = false; + const input = { + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + prepared = true; + throw new Error("must not prepare"); + }, + }, + }; + for (const text of [ + JSON.stringify({ job: f.input.name, harness: "fake" }), + "{invalid", + "null", + JSON.stringify({ identity: { version: 2 } }), + ]) { + await writeFile(join(input.dir, "job.json"), text); + await assert.rejects(runJob(input), /new job name/); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), text); + } + await rm(join(input.dir, "job.json")); + await writeFile(join(input.dir, "result.json"), "orphaned"); + await assert.rejects(runJob(input), /no job.json.*new job name/); + assert.equal(await readFile(join(input.dir, "result.json"), "utf8"), "orphaned"); + assert.equal(prepared, false); + assert.equal(await readFile(f.calls, "utf8"), ""); +}); + +test("task hashes are relocatable and ignore unselected trees and mtimes, but include paths and executable bits", async (t) => { + const f = await fixture(t); + const original = await hashTask(f.input.tasks[0]!); + const relocated = join(f.root, "relocated"); + await cp(f.taskDir, relocated, { recursive: true }); + const task = await loadTask(relocated); + assert.equal(await hashTask(task), original); + await utimes(join(relocated, "instruction.md"), new Date(0), new Date(0)); + await writeFile(join(f.root, "tasks", "unselected"), "does not affect a"); + assert.equal(await hashTask(task), original); + await rename( + join(relocated, "environment", "input.bin"), + join(relocated, "environment", "renamed.bin"), + ); + assert.notEqual(await hashTask(task), original); + await rename( + join(relocated, "environment", "renamed.bin"), + join(relocated, "environment", "input.bin"), + ); + await chmod(join(relocated, "solution", "solve.sh"), 0o755); + assert.notEqual(await hashTask(task), original); + await chmod(join(relocated, "solution", "solve.sh"), 0o644); + assert.equal(await hashTask(task), original); + await rm(join(relocated, "environment", "input.bin")); + assert.notEqual(await hashTask(task), original); +}); + +test("task hashing refuses symlinks and environments outside the selected tree", async (t) => { + const f = await fixture(t); + await symlink(f.calls, join(f.taskDir, "external")); + await assert.rejects(hashTask(f.input.tasks[0]!), /regular files\/directories/); + await rm(join(f.taskDir, "external")); + const task = f.input.tasks[0]!; + await assert.rejects( + hashTask({ ...task, environment: { ...task.environment, dockerfileDir: f.root } }), + /inside the selected task tree/, + ); +}); + +test("an aborted job retains its identity, releases the lock, and resumes the same remaining work", async (t) => { + const f = await fixture(t); + const controller = new AbortController(); + const input = { ...f.input, attempts: 2 }; + const partial = await runJob({ + ...input, + signal: controller.signal, + onTrial: () => controller.abort(), + }); + assert.equal(partial.trials.length, 1); + const metadata = await readFile(join(input.dir, "job.json"), "utf8"); + const seen: boolean[] = []; + const result = await runJob({ ...input, onTrial: (_trial, resumed) => seen.push(resumed) }); + assert.deepEqual(seen, [true, false]); + assert.equal(result.trials.length, 2); + assert.deepEqual(result.trials[0], partial.trials[0]); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), metadata); +}); + +test("same-job lock rejects concurrent runs even in another output directory, and releases on failure", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + let enter!: () => void; + let release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + const active = runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + enter(); + await released; + return f.input.harness.prepare({ model: f.input.model, cacheDir: f.input.cacheDir }); + }, + }, + }); + try { + await entered; + await assert.rejects(runJob(f.input), /is locked.*active run/); + const otherDir = join(f.root, "other-job-dir"); + await assert.rejects(runJob({ ...f.input, dir: otherDir }), /is locked/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + assert.ok(!(await readdir(f.root)).includes("other-job-dir")); + } finally { + release(); + await active; + } + await assert.rejects( + runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + throw new Error("prepare failed"); + }, + }, + }), + /prepare failed/, + ); + await runJob(f.input); +}); + +test("stale locks fail closed with explicit recovery, and unknown containers are not removed", async (t) => { + const f = await fixture(t); + const lock = join(tmpdir(), `eve-bench-job-${identityHash(f.input.name)}.lock`); + await writeFile(lock, "crashed owner", { flag: "wx" }); + try { + await assert.rejects(runJob(f.input), /confirm it has stopped before removing/); + assert.equal(await readFile(lock, "utf8"), "crashed owner"); + assert.equal(await readFile(f.calls, "utf8"), ""); + } finally { + await rm(lock); + } + await f.leftovers(); + const containers = await readFile(f.containers, "utf8"); + await assert.rejects(runJob(f.input), /containers but no resume identity.*new job name/); + assert.equal(await readFile(f.containers, "utf8"), containers); + assert.deepEqual(await readdir(f.input.dir), []); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["rm"/); +}); diff --git a/packages/eve-bench/test/scenarios/trial-validity.test.ts b/packages/eve-bench/test/scenarios/trial-validity.test.ts new file mode 100644 index 0000000000..553478b69c --- /dev/null +++ b/packages/eve-bench/test/scenarios/trial-validity.test.ts @@ -0,0 +1,208 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import type { Harness } from "../../src/core/harness.ts"; +import { runJob } from "../../src/core/job.ts"; +import type { JobResult, TrialResult } from "../../src/core/result.ts"; +import { loadTask } from "../../src/core/task.ts"; +import { readCliUsage } from "../../src/harnesses/cli/usage.ts"; + +/** Files the harness and verifier leave in the container; null leaves the directory absent. */ +interface ContainerLogs { + readonly agent: Readonly> | null; + readonly verifier: Readonly> | null; +} + +interface Scenario { + readonly logs: ContainerLogs; + readonly buildFails?: boolean; + readonly agentTimeoutSec?: number; + readonly agentRunsMs?: number; + readonly modelFree?: boolean; +} + +// pi --mode json records one assistant message_end per model call; a rejected key records zero usage. +const piCall = (input: number, output: number) => + `${JSON.stringify({ + type: "message_end", + message: { role: "assistant", usage: { input, output, cacheRead: 0, cost: { total: 0 } } }, + })}\n`; +const reward = (value: string) => ({ "reward.txt": `${value}\n` }); + +async function runScenario(t: TestContext, scenario: Scenario) { + const root = await mkdtemp(join(tmpdir(), "eve-bench-validity-")); + const bin = join(root, "bin"); + const fs = join(root, "container"); + const taskDir = join(root, "tasks", "a"); + await mkdir(bin); + for (const dir of ["environment", "tests"]) await mkdir(join(taskDir, dir), { recursive: true }); + await writeFile(join(taskDir, "instruction.md"), "Do the task.\n"); + await writeFile( + join(taskDir, "task.toml"), + `[task]\nname = "a"\n${scenario.agentTimeoutSec ? `[agent]\ntimeout_sec = ${scenario.agentTimeoutSec}\n` : ""}`, + ); + await writeFile(join(taskDir, "environment", "Dockerfile"), "FROM scratch\n"); + await writeFile(join(taskDir, "tests", "test.sh"), "true\n"); + for (const [dir, files] of Object.entries(scenario.logs)) { + if (files === null) continue; + await mkdir(join(fs, "logs", dir), { recursive: true }); + for (const [name, content] of Object.entries(files)) + await writeFile(join(fs, "logs", dir, name), content); + } + // No daemon or model: copying out of the container serves the scripted log tree. + await writeFile( + join(bin, "docker"), + `#!${process.execPath} +import { cpSync, existsSync } from 'node:fs'; +import { basename, join } from 'node:path'; +const args = process.argv.slice(2); +const fail = (message) => { process.stderr.write(message + '\\n'); process.exit(1); }; +switch (args[0]) { + case 'version': console.log('fake-docker'); break; + case 'image': process.exit(1); + case 'build': if (${scenario.buildFails === true}) fail('fake build failed'); break; + case 'exec': { + const command = args.at(-1); + if (command === 'pwd') console.log('/workspace'); + if (command === 'run-agent') await new Promise((r) => setTimeout(r, ${scenario.agentRunsMs ?? 0})); + break; + } + case 'cp': { + const [source, target] = args.slice(1); + const match = /^[^/][^:]*:(\\/.*)$/.exec(source); + if (!match) break; + const from = join(${JSON.stringify(fs)}, match[1]); + if (!existsSync(from)) fail('No such container path: ' + match[1]); + cpSync(from, join(target, basename(from)), { recursive: true }); + break; + } +} +`, + { mode: 0o755 }, + ); + const previousPath = process.env.PATH; + process.env.PATH = `${bin}:${previousPath}`; + t.after(async () => { + process.env.PATH = previousPath; + await rm(root, { recursive: true, force: true }); + }); + const harness: Harness = { + name: "fake", + modelFree: scenario.modelFree === true, + prepare: async () => ({ dir: bin, provenance: { key: "v1" } }), + command: () => "run-agent", + env: () => ({}), + readUsage: (dir) => readCliUsage("pi", dir), + }; + const dir = join(root, "job"); + const result: JobResult = await runJob({ + name: basename(root), + dir, + cacheDir: join(root, "cache"), + dataset: { name: "local", version: "0", gitUrl: "", commit: "", tasks: ["a"] }, + tasks: [await loadTask(taskDir)], + harness, + model: "test/model", + attempts: 1, + concurrency: 1, + forwardEnv: {}, + signal: new AbortController().signal, + }); + const saved = JSON.parse( + await readFile(join(dir, "a", "1", "trial.json"), "utf8"), + ) as TrialResult; + assert.deepEqual(saved, result.trials[0], "trial.json must record the classification"); + return { trial: saved, summary: result.summary }; +} + +test("a trial with a completed model call and a reward is scored", async (t) => { + const { trial, summary } = await runScenario(t, { + logs: { agent: { "pi.jsonl": piCall(100, 20) + piCall(50, 5) }, verifier: reward("1") }, + }); + assert.equal(trial.invalid, undefined); + assert.deepEqual(trial.usage, { + inputTokens: 150, + outputTokens: 25, + cachedTokens: 0, + costUsd: null, + }); + assert.deepEqual(summary, { + tasks: 1, + attempts: 1, + scored: 1, + invalid: { environment: 0, harness: 0, verifier: 0 }, + resolved: 1, + meanReward: 1, + }); +}); + +test("a harness whose model calls all failed is invalid, not a task failure", async (t) => { + const { trial, summary } = await runScenario(t, { + logs: { agent: { "pi.jsonl": `not json\n${piCall(0, 0)}` }, verifier: reward("0") }, + }); + assert.equal(trial.invalid?.phase, "harness"); + assert.match(trial.invalid!.reason, /zero token usage/); + assert.deepEqual(summary.invalid, { environment: 0, harness: 1, verifier: 0 }); + assert.equal(summary.scored, 0); + assert.equal(summary.meanReward, null); +}); + +test("a harness that logged no usage is invalid even when the verifier passes", async (t) => { + const { trial } = await runScenario(t, { logs: { agent: {}, verifier: reward("1") } }); + assert.equal(trial.reward, 1); + assert.equal(trial.invalid?.phase, "harness"); + assert.match(trial.invalid!.reason, /no model usage/); +}); + +test("a model-free harness needs no usage to be scored", async (t) => { + const { trial } = await runScenario(t, { + modelFree: true, + logs: { agent: {}, verifier: reward("1") }, + }); + assert.equal(trial.invalid, undefined); +}); + +test("an agent timeout after model calls is scored against the harness", async (t) => { + const { trial, summary } = await runScenario(t, { + agentTimeoutSec: 0.3, + agentRunsMs: 5_000, + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: reward("0") }, + }); + assert.equal(trial.agent.status, "timeout"); + assert.equal(trial.invalid, undefined); + assert.equal(trial.usage?.inputTokens, 10); + assert.equal(summary.scored, 1); + assert.equal(summary.meanReward, 0); +}); + +test("a missing reward is a verifier failure", async (t) => { + const { trial } = await runScenario(t, { + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: {} }, + }); + assert.equal(trial.reward, null); + assert.deepEqual(trial.invalid, { phase: "verifier", reason: "verifier wrote no reward" }); +}); + +test("undownloadable agent logs are an environment failure", async (t) => { + const { trial } = await runScenario(t, { logs: { agent: null, verifier: reward("1") } }); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.invalid?.phase, "environment"); + assert.match( + trial.invalid!.reason, + /after the harness started: could not download \/logs\/agent/, + ); +}); + +test("an image build failure is an environment failure before the harness starts", async (t) => { + const { trial, summary } = await runScenario(t, { + buildFails: true, + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: reward("1") }, + }); + assert.equal(trial.agent.status, "skipped"); + assert.equal(trial.invalid?.phase, "environment"); + assert.match(trial.invalid!.reason, /before the harness started: .*fake build failed/); + assert.deepEqual(summary.invalid, { environment: 1, harness: 0, verifier: 0 }); +}); diff --git a/packages/eve-bench/test/stress/stress.test.ts b/packages/eve-bench/test/stress/stress.test.ts new file mode 100644 index 0000000000..e18b3b48af --- /dev/null +++ b/packages/eve-bench/test/stress/stress.test.ts @@ -0,0 +1,276 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFile, rm } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const tasksRoot = join(packageRoot, "test", "stress", "tasks"); +const jobsRoot = join(packageRoot, ".generated", "jobs"); +const timeout = 300_000; + +interface StepResult { + status: "completed" | "timeout" | "failed" | "skipped"; +} + +interface TrialResult { + task: string; + reward: number | null; + agent: StepResult; + verifier: StepResult; + error?: string; + invalid?: { phase: "environment" | "harness" | "verifier"; reason: string }; +} + +interface JobResult { + dataset: { name: string; version: string; commit: string }; + trials: TrialResult[]; + summary: { scored: number; invalid: Record; resolved: number }; +} + +interface CliResult { + code: number | null; + stdout: string; + stderr: string; + durationMs: number; +} + +function taskDir(name: string): string { + return join(tasksRoot, name); +} + +function cliArgs( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], +): string[] { + return [ + "src/cli.ts", + "run", + "--harness", + "oracle", + "--job", + job, + "--json", + ...taskDirs.flatMap((dir) => ["--task-dir", dir]), + ...extra, + ]; +} + +async function runCli( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], + fresh = true, +): Promise { + if (fresh) await rm(join(jobsRoot, job), { recursive: true, force: true }); + const started = Date.now(); + const child = spawn(process.execPath, cliArgs(job, taskDirs, extra), { + cwd: packageRoot, + stdio: ["ignore", "pipe", "pipe"], + }); + return collect(child, started); +} + +function collect(child: ReturnType, started = Date.now()): Promise { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout?.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk)); + return new Promise((resolvePromise, reject) => { + child.once("error", reject); + child.once("close", (code) => { + resolvePromise({ + code, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + durationMs: Date.now() - started, + }); + }); + }); +} + +async function readResult(job: string): Promise { + return JSON.parse(await readFile(join(jobsRoot, job, "result.json"), "utf8")) as JobResult; +} + +async function readTrial(job: string, task: string): Promise { + return JSON.parse( + await readFile(join(jobsRoot, job, task, "1", "trial.json"), "utf8"), + ) as TrialResult; +} + +async function dockerNames(filter: string): Promise { + const result = await runCommand("docker", [ + "ps", + "-a", + "--filter", + `name=eve-bench-${filter}`, + "--format", + "{{.Names}}", + ]); + assert.equal(result.code, 0, result.stderr); + return result.stdout.trim() ? result.stdout.trim().split("\n") : []; +} + +async function cleanupContainers(filter: string): Promise { + const names = await dockerNames(filter); + if (names.length === 0) return; + const result = await runCommand("docker", ["rm", "--force", ...names]); + assert.equal(result.code, 0, result.stderr); +} + +async function runCommand(command: string, args: readonly string[]): Promise { + const started = Date.now(); + return collect( + spawn(command, args, { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }), + started, + ); +} + +async function waitFor( + condition: () => Promise, + waitMs: number, + description: string, +): Promise { + const deadline = Date.now() + waitMs; + while (Date.now() < deadline) { + if (await condition()) return; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 250)); + } + assert.fail(`timed out waiting for ${description}`); +} + +function assertPassed(result: CliResult): void { + assert.equal(result.code, 0, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`); +} + +test("happy: records completed steps and reward 1", { timeout }, async () => { + const job = "stress-happy"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("happy")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-happy"); + assert.equal(trial.reward, 1); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "completed"); + assert.equal(trial.invalid, undefined); + const saved = await readResult(job); + assert.deepEqual(saved.dataset, { name: "local", version: "0", commit: "" }); + assert.equal(saved.summary.scored, 1); + assert.equal(saved.summary.resolved, 1); +}); + +test("agent-timeout: verifier still runs and the container is removed", { timeout }, async () => { + const job = "stress-agent-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("agent-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-agent-timeout"); + assert.equal(trial.agent.status, "timeout"); + assert.equal(trial.verifier.status, "completed"); + assert.equal(trial.reward, 0); + // A timeout is the harness's outcome, so it stays scored. + assert.equal(trial.invalid, undefined); + assert.deepEqual(await dockerNames(job), []); +}); + +test("verifier-timeout: records timeout and null reward", { timeout }, async () => { + const job = "stress-verifier-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("verifier-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-verifier-timeout"); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "timeout"); + assert.equal(trial.reward, null); + assert.equal(trial.invalid?.phase, "verifier"); + assert.equal((await readResult(job)).summary.scored, 0); +}); + +test( + "bad-image: records the setup error, skips both steps, and leaves no container", + { timeout }, + async () => { + const job = "stress-bad-image"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("bad-image")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-bad-image"); + assert.ok(trial.error); + assert.equal(trial.agent.status, "skipped"); + assert.equal(trial.verifier.status, "skipped"); + assert.equal(trial.invalid?.phase, "environment"); + assert.equal((await readResult(job)).summary.invalid.environment, 1); + assert.deepEqual(await dockerNames(job), []); + }, +); + +test( + "abort: exits 130, cleans up, and rejects a different resume identity", + { timeout }, + async () => { + const job = "stress-abort"; + await cleanupContainers(job); + await rm(join(jobsRoot, job), { recursive: true, force: true }); + const child = spawn( + process.execPath, + cliArgs(job, [taskDir("abort-a"), taskDir("abort-b")], ["--concurrency", "2"]), + { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }, + ); + const completed = collect(child); + + try { + await waitFor( + async () => (await dockerNames(job)).length === 2, + 120_000, + "both abort containers", + ); + child.kill("SIGINT"); + const aborted = await completed; + assert.equal(aborted.code, 130, `stdout:\n${aborted.stdout}\nstderr:\n${aborted.stderr}`); + + const partial = await readResult(job); + assert.ok(partial.trials.length <= 2); + await waitFor(async () => (await dockerNames(job)).length === 0, 15_000, "abort cleanup"); + + const resumed = await runCli(job, [taskDir("abort-resume")], [], false); + assert.equal(resumed.code, 1); + assert.match(resumed.stderr, /resume identity differs/); + assert.deepEqual(await dockerNames(job), []); + } finally { + if (child.exitCode === null) child.kill("SIGKILL"); + await cleanupContainers(job); + } + }, +); + +test( + "concurrency: six trials finish substantially faster than serial execution", + { timeout }, + async () => { + const dirs = Array.from({ length: 6 }, (_, index) => taskDir(`concurrency-${index + 1}`)); + await cleanupContainers("stress-concurrency"); + + const single = await runCli("stress-concurrency-single", [dirs[0]!], ["--concurrency", "1"]); + assertPassed(single); + const concurrent = await runCli("stress-concurrency", dirs, ["--concurrency", "6"]); + assertPassed(concurrent); + + const result = await readResult("stress-concurrency"); + assert.equal(result.trials.length, 6); + assert.deepEqual( + result.trials.map((trial) => trial.reward), + [1, 1, 1, 1, 1, 1], + ); + assert.ok( + concurrent.durationMs < single.durationMs * 3, + `single=${single.durationMs}ms concurrent=${concurrent.durationMs}ms`, + ); + }, +); diff --git a/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-a/instruction.md b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-a/task.toml b/packages/eve-bench/test/stress/tasks/abort-a/task.toml new file mode 100644 index 0000000000..d0a9a638a5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-a" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-b/instruction.md b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-b/task.toml b/packages/eve-bench/test/stress/tasks/abort-b/task.toml new file mode 100644 index 0000000000..cf271a1f1a --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-b" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/task.toml b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml new file mode 100644 index 0000000000..cb0de4aea3 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-resume" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh new file mode 100644 index 0000000000..eee73eb76e --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 60; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml new file mode 100644 index 0000000000..58831360d4 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-agent-timeout" + +[agent] +timeout_sec = 3 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/bad-image/instruction.md b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/bad-image/task.toml b/packages/eve-bench/test/stress/tasks/bad-image/task.toml new file mode 100644 index 0000000000..d74d09f84c --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/task.toml @@ -0,0 +1,13 @@ +[task] +name = "stress-bad-image" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false +docker_image = "eve-bench/does-not-exist:1" diff --git a/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh new file mode 100644 index 0000000000..169b2c15e2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +echo 1 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml new file mode 100644 index 0000000000..257a3f2044 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-1" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml new file mode 100644 index 0000000000..21e6f19aeb --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-2" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml new file mode 100644 index 0000000000..c0faeb0b80 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-3" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml new file mode 100644 index 0000000000..0b2d2ba227 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-4" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml new file mode 100644 index 0000000000..52baba26c5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-5" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml new file mode 100644 index 0000000000..15ae72e375 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-6" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/happy/instruction.md b/packages/eve-bench/test/stress/tasks/happy/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/happy/task.toml b/packages/eve-bench/test/stress/tasks/happy/task.toml new file mode 100644 index 0000000000..eaf9709812 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-happy" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/happy/tests/test.sh b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml new file mode 100644 index 0000000000..7672b068e9 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-verifier-timeout" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 3 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh new file mode 100644 index 0000000000..fb0ab7d284 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +sleep 60; echo 1 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/tsconfig.json b/packages/eve-bench/tsconfig.json new file mode 100644 index 0000000000..807ccbdd2d --- /dev/null +++ b/packages/eve-bench/tsconfig.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": ["ES2024"], + "rootDir": ".", + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": false + }, + "include": ["agent/**/*.ts", "src/**/*.ts"], + "exclude": [] +} diff --git a/packages/eve-bench/turbo.json b/packages/eve-bench/turbo.json new file mode 100644 index 0000000000..d77ff0acd1 --- /dev/null +++ b/packages/eve-bench/turbo.json @@ -0,0 +1,11 @@ +{ + "extends": ["//"], + "tasks": { + "test:integration": { + "dependsOn": ["eve#build"] + }, + "test:scenario": { + "dependsOn": ["^build"] + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1dff9f2f9..1518d07849 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1864,6 +1864,22 @@ importers: specifier: 5.0.0 version: 5.0.0(zod@4.5.4) + packages/eve-bench: + dependencies: + eve: + specifier: workspace:* + version: link:../eve + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + typescript: + specifier: 'catalog:' + version: 7.0.2 + packages/eve-buzz-acp-adapter: dependencies: eve: diff --git a/research/eve-bench-runner.md b/research/eve-bench-runner.md new file mode 100644 index 0000000000..e290609aae --- /dev/null +++ b/research/eve-bench-runner.md @@ -0,0 +1,238 @@ +--- +issue: TODO +status: approved +last_updated: "2026-09-09" +--- + +# eve-bench: a zero-dependency benchmark runner + +`packages/eve-bench` currently benchmarks the eve harness on Terminal-Bench +through Harbor: a Python adapter subclasses Harbor's private `Eve` agent, a +Node runner drives the built app, and a materialization script produces a +self-contained app directory (two Node binaries, a rolldown bundle, `npm ci`) +because Harbor copies the agent directory into every task container. + +That stack needs three runtimes (Python, Harbor at a pinned revision, Node), +leaks Harbor-specific env (`HARBOR_MODEL`, `EVE_HARBOR_TASK_WORKDIR`) into the +harness under test, breaks on any upstream refactor of Harbor's private +methods, and has no path to a CLI, an agent tool, or a GitHub Action that +does not drag Python along. + +Per trial, Harbor does five things: resolve a task directory, run a Docker +container, exec the agent under a timeout, exec `tests/test.sh` under a +timeout and read `/logs/verifier/reward.txt`, and write a result record. A +job is that loop over tasks × attempts with concurrency and resume. All of it +fits in a small amount of Node with no third-party dependencies. + +## Proposal + +Replace Harbor with an eve-owned runner. Keep the Terminal-Bench task format +and reward contract unchanged so datasets stay interchangeable and results +remain comparable with published Harbor runs. + +``` +packages/eve-bench/ + src/core/ zero-dependency library; does not import eve + task.ts load task.toml + instruction.md + docker.ts container lifecycle over the docker CLI + trial.ts run one attempt: start → agent → verify → record + job.ts matrix, concurrency, attempts, resume + result.ts trial/job JSON schema + report.ts console, json, junit, diff + src/harnesses/eve/ the eve harness: build once, exec in container + src/harnesses/ oracle (reference solutions) and future harnesses + src/cli.ts eve-bench run | tasks | report | diff + src/tools/ eve tools wrapping core for agent consumers (follow-up) + datasets/ pinned dataset locks and cohorts (JSON) + agent/ the harness under test (unchanged) +``` + +The runner is plain `.ts` executed by Node 24 with type stripping; the root +tsconfig already enforces `erasableSyntaxOnly`. No build step, no bundler. + +### Datasets + +A dataset lock pins a git remote, commit, and the task paths within it: + +```json +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": ["adaptive-rejection-sampler", "fix-git", "..."] +} +``` + +`eve-bench tasks sync ` clones the pinned commit (shallow, sparse) into +`.generated/datasets/-/`. Cohorts are named task subsets in +`datasets/cohorts/*.json` and reference a lock by name; membership changes +are reviewed like code. + +### Tasks + +A task is a Terminal-Bench directory: `task.toml`, `instruction.md`, +`environment/Dockerfile` (optionally with a prebuilt `docker_image`), +`tests/test.sh`, `solution/`. The loader reads only what the runner needs: + +- `task.name` +- `agent.timeout_sec`, `verifier.timeout_sec`, `environment.build_timeout_sec` +- `environment.docker_image`, `cpus`, `memory_mb`, `allow_internet` + +`task.toml` is parsed by a minimal TOML reader covering the subset the +dataset uses (tables, strings, numbers, booleans, arrays). Anything outside +that subset is a load error naming the line. + +### Environment + +One container per trial, always via the `docker` CLI. Prebuilt +`environment.docker_image` is preferred; otherwise the Dockerfile is built +once per task and cached by image tag. Every operation (`build`, `run`, +`exec`, `cp`, `rm`) accepts an `AbortSignal`; SIGINT to the runner aborts +in-flight trials and removes their containers before exiting. + +### Harness contract + +A harness is something that can be placed in a container and told to solve +an instruction. It is a spawned shell, nothing more: + +```ts +interface Harness { + readonly name: string; + /** Runs once per job on the host; returns a directory to place in the container. */ + prepare(ctx: { model: string; cacheDir: string }): Promise<{ dir: string; provenance?: object }>; + /** Optional per-task host paths uploaded into the install directory. */ + stage?(ctx: { taskDir: string }): readonly string[]; + /** Shell command executed inside the container in the task working directory. */ + command(ctx: { + installDir: string; + instructionPath: string; + taskWorkdir: string; + logsDir: string; + }): string; + /** Env visible to the command. Provider credentials are forwarded by the runner. */ + env(ctx: { model: string }): Record; +} +``` + +The bundle directory is uploaded to `/installed-agent`. The eve harness +bundles the Nitro output built from `agent/`, a runner, and Linux Node +binaries, cached by content hash of `agent/`, the eve source, and the model. +The runner drives the server over plain HTTP (`POST /eve/v1/session`, then +`GET /eve/v1/session/:id/stream`), so the bundle carries no `node_modules` +and no bundler is involved. It writes `/logs/agent/events.ndjson` and +`/logs/agent/result.json`. The env the harness reads is `EVE_BENCH_MODEL` and +`EVE_BENCH_TASK_WORKDIR`. + +An `oracle` harness runs each task's `solution/solve.sh` via `stage`. It +validates the runner without a model and is the reward ceiling a real harness +is measured against. A harness wrapping another CLI agent is the same +interface and enables side-by-side comparison. + +### Trial lifecycle + +``` +resolve task ─▶ start container ─▶ upload harness ─▶ exec harness (agent timeout) + │ + record ◀── read reward.txt ◀── exec tests/test.sh ◀─────┘ + (verifier timeout) +``` + +The verifier always runs, including after an agent timeout or failure, so a +trial's reward reflects the filesystem the agent left behind. Container +removal is unconditional. Each trial writes to its own directory: + +``` +.generated/jobs//// + trial.json the result record below + agent/ copied /logs/agent + verifier/ copied /logs/verifier + container.log docker output for the trial +``` + +Resume skips trial directories that already contain a `trial.json`. + +### Result schema + +```ts +interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; // null when the verifier produced none + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: { + inputTokens: number; + outputTokens: number; + cachedTokens: number; + costUsd: number; + }; + readonly error?: string; + readonly startedAt: string; + readonly finishedAt: string; +} + +interface StepResult { + readonly status: "completed" | "timeout" | "failed" | "skipped"; + readonly exitCode: number | null; + readonly durationMs: number; +} + +interface JobResult { + readonly job: string; + readonly dataset: { name: string; version: string; commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: { tasks: number; attempts: number; resolved: number; meanReward: number }; +} +``` + +This schema is the API. The CLI, the GitHub Action, and agent tools all read +and write it; nothing else is stable. + +### CLI + +``` +eve-bench tasks sync fetch a pinned dataset +eve-bench tasks list --cohort smoke print task names +eve-bench run --cohort smoke --model zai/glm-5.2 [--attempts 3] [--concurrency 8] [--task fix-git ...] + [--harness eve|oracle] [--eve local|] [--job ] # same --job resumes +eve-bench report [--format console|json|junit] +eve-bench diff per-task reward deltas +``` + +Every command accepts `--json` and never prompts. Provider credentials come +from the environment (`AI_GATEWAY_API_KEY`) and are forwarded into the +container only for the harness exec, never written to disk. + +### Consumers + +- CI: a workflow runs `eve-bench run --cohort smoke --json`, uploads the job + directory as an artifact, and posts `eve-bench diff` against the base + branch's artifact. +- Humans: the same CLI with the console reporter. +- Agents: `src/tools/` exposes `run_trial`, `get_result`, and `diff` as eve + tools over `core`; they return the schema above. + +## Invariants + +- `src/core` has no dependency on `eve` or any third-party package; only the + eve harness imports eve. +- Task format, verifier contract, and reward semantics are Terminal-Bench's; + eve-bench adds nothing to a task directory. +- A harness never sees runner internals beyond the contract above. +- Trial directories are append-only; resume never rewrites a `trial.json`. + +## Out of scope + +- Non-Docker environments (Vercel Sandbox, remote runners). The `docker.ts` + surface is small enough to add a second implementation later; it is not + abstracted ahead of need. +- Reusing eve's `SandboxBackend` for the environment. The benchmark must not + depend on the surface it measures. +- Uploading results to a hosted dashboard. +- `src/tools/` (eve tools over `core`) and the CI workflow. Both consume the + result schema above and follow once the CLI has run a full dataset.