From d449b72146b67a3db7f5c0fe1016667e69f97f57 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Wed, 9 Sep 2026 14:11:33 -0400 Subject: [PATCH 1/3] feat(eve-bench): replace Harbor with a zero-dependency benchmark runner Terminal-Bench evaluation of the eve harness no longer goes through Harbor and a Python adapter. packages/eve-bench now owns a small Node runner (task loader, docker lifecycle, trial/job scheduler with resume, reporters), an eve harness whose in-container runner speaks plain HTTP to the built server, an oracle harness for reference solutions, pinned dataset locks and cohorts, eve tools over the same core, and a CI workflow. Design in research/eve-bench-runner.md. Signed-off-by: Rui Conti --- .github/workflows/bench.yml | 157 +++++++++++ .gitignore | 3 + packages/eve-bench/README.md | 105 ++++++++ packages/eve-bench/agent/agent.ts | 10 + packages/eve-bench/agent/instructions.md | 11 + packages/eve-bench/agent/task-path.ts | 13 + .../eve-bench/agent/tools/ask_question.ts | 3 + packages/eve-bench/agent/tools/bash.ts | 37 +++ packages/eve-bench/agent/tools/glob.ts | 22 ++ packages/eve-bench/agent/tools/grep.ts | 45 ++++ packages/eve-bench/agent/tools/read_file.ts | 27 ++ packages/eve-bench/agent/tools/todo.ts | 3 + packages/eve-bench/agent/tools/web_fetch.ts | 3 + packages/eve-bench/agent/tools/web_search.ts | 3 + packages/eve-bench/agent/tools/write_file.ts | 24 ++ .../eve-bench/datasets/cohorts/smoke.json | 11 + .../datasets/terminal-bench-2.0.json | 97 +++++++ packages/eve-bench/package.json | 26 ++ packages/eve-bench/src/cli.ts | 175 +++++++++++++ packages/eve-bench/src/core/dataset.test.ts | 103 ++++++++ packages/eve-bench/src/core/dataset.ts | 77 ++++++ packages/eve-bench/src/core/docker.ts | 177 +++++++++++++ packages/eve-bench/src/core/harness.ts | 34 +++ packages/eve-bench/src/core/job.ts | 108 ++++++++ packages/eve-bench/src/core/report.test.ts | 63 +++++ packages/eve-bench/src/core/report.ts | 118 +++++++++ packages/eve-bench/src/core/result.ts | 52 ++++ packages/eve-bench/src/core/task.test.ts | 156 +++++++++++ packages/eve-bench/src/core/task.ts | 82 ++++++ packages/eve-bench/src/core/toml.test.ts | 68 +++++ packages/eve-bench/src/core/toml.ts | 159 +++++++++++ packages/eve-bench/src/core/trial.ts | 188 +++++++++++++ packages/eve-bench/src/harnesses/eve/index.ts | 247 ++++++++++++++++++ .../eve-bench/src/harnesses/eve/runner.mjs | 242 +++++++++++++++++ packages/eve-bench/src/harnesses/oracle.ts | 23 ++ packages/eve-bench/src/options.ts | 98 +++++++ packages/eve-bench/src/tools/diff_bench.ts | 25 ++ .../eve-bench/src/tools/get_bench_result.ts | 24 ++ .../eve-bench/src/tools/list_bench_tasks.ts | 19 ++ packages/eve-bench/src/tools/run_bench.ts | 55 ++++ packages/eve-bench/src/tools/tools.test.ts | 71 +++++ packages/eve-bench/tsconfig.json | 12 + pnpm-lock.yaml | 16 ++ research/eve-bench-runner.md | 238 +++++++++++++++++ 44 files changed, 3230 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 packages/eve-bench/README.md create mode 100644 packages/eve-bench/agent/agent.ts create mode 100644 packages/eve-bench/agent/instructions.md create mode 100644 packages/eve-bench/agent/task-path.ts create mode 100644 packages/eve-bench/agent/tools/ask_question.ts create mode 100644 packages/eve-bench/agent/tools/bash.ts create mode 100644 packages/eve-bench/agent/tools/glob.ts create mode 100644 packages/eve-bench/agent/tools/grep.ts create mode 100644 packages/eve-bench/agent/tools/read_file.ts create mode 100644 packages/eve-bench/agent/tools/todo.ts create mode 100644 packages/eve-bench/agent/tools/web_fetch.ts create mode 100644 packages/eve-bench/agent/tools/web_search.ts create mode 100644 packages/eve-bench/agent/tools/write_file.ts create mode 100644 packages/eve-bench/datasets/cohorts/smoke.json create mode 100644 packages/eve-bench/datasets/terminal-bench-2.0.json create mode 100644 packages/eve-bench/package.json create mode 100644 packages/eve-bench/src/cli.ts create mode 100644 packages/eve-bench/src/core/dataset.test.ts create mode 100644 packages/eve-bench/src/core/dataset.ts create mode 100644 packages/eve-bench/src/core/docker.ts create mode 100644 packages/eve-bench/src/core/harness.ts create mode 100644 packages/eve-bench/src/core/job.ts create mode 100644 packages/eve-bench/src/core/report.test.ts create mode 100644 packages/eve-bench/src/core/report.ts create mode 100644 packages/eve-bench/src/core/result.ts create mode 100644 packages/eve-bench/src/core/task.test.ts create mode 100644 packages/eve-bench/src/core/task.ts create mode 100644 packages/eve-bench/src/core/toml.test.ts create mode 100644 packages/eve-bench/src/core/toml.ts create mode 100644 packages/eve-bench/src/core/trial.ts create mode 100644 packages/eve-bench/src/harnesses/eve/index.ts create mode 100644 packages/eve-bench/src/harnesses/eve/runner.mjs create mode 100644 packages/eve-bench/src/harnesses/oracle.ts create mode 100644 packages/eve-bench/src/options.ts create mode 100644 packages/eve-bench/src/tools/diff_bench.ts create mode 100644 packages/eve-bench/src/tools/get_bench_result.ts create mode 100644 packages/eve-bench/src/tools/list_bench_tasks.ts create mode 100644 packages/eve-bench/src/tools/run_bench.ts create mode 100644 packages/eve-bench/src/tools/tools.test.ts create mode 100644 packages/eve-bench/tsconfig.json create mode 100644 research/eve-bench-runner.md diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..a759cdf859 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,157 @@ +name: eve Benchmarks + +on: + pull_request: + paths: + - "packages/eve-bench/**" + - research/eve-bench-runner.md + workflow_dispatch: + inputs: + model: + description: Model to benchmark + required: false + default: zai/glm-5.2 + type: string + cohort: + description: Task cohort to run + required: false + default: smoke + type: string + base_job_artifact: + description: Optional artifact containing a base job to compare + required: false + type: string + +concurrency: + group: eve-bench-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + oracle-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + - name: Run tests + run: pnpm run test + + - name: Sync benchmark tasks + run: pnpm bench tasks sync --json + + - name: Run oracle smoke benchmark + run: pnpm bench run --harness oracle --cohort smoke --concurrency 5 --job oracle-smoke --format junit > junit.xml + + - name: Print oracle smoke report + run: pnpm bench report oracle-smoke + + - name: Upload oracle smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-oracle-smoke + path: | + packages/eve-bench/.generated/jobs/oracle-smoke + packages/eve-bench/junit.xml + include-hidden-files: true + if-no-files-found: warn + + - name: Require every oracle attempt to resolve + run: | + node -e ' + const result = require("./.generated/jobs/oracle-smoke/result.json"); + const { attempts, resolved } = result.summary; + if (resolved < attempts) { + console.error(`Oracle resolved ${resolved} of ${attempts} attempts`); + process.exit(1); + } + ' + + eve-smoke: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + - name: Build eve package + run: pnpm --filter eve run build + working-directory: ${{ github.workspace }} + + - name: Run eve smoke benchmark + env: + MODEL: ${{ inputs.model }} + COHORT: ${{ inputs.cohort }} + run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke + + - name: Download base job artifact + if: ${{ inputs.base_job_artifact != '' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.base_job_artifact }} + path: packages/eve-bench/.generated/base-job + + - name: Compare with base job + if: ${{ inputs.base_job_artifact != '' }} + run: | + pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt + { + echo "## Benchmark diff" + echo '```text' + cat diff.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload eve smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-eve-smoke + path: packages/eve-bench/.generated/jobs/eve-smoke + include-hidden-files: true + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 8b2830ffc5..adebbfbd91 100644 --- a/.gitignore +++ b/.gitignore @@ -13,12 +13,14 @@ coverage/ .pnpm-store/ .bun/ .npm/ +__pycache__/ # Test artifacts .vitest/ .extension-contracts-cache/ packages/eve/.workflow-vitest/ packages/eve/.generated/ +packages/eve-bench/.generated/ playwright-report/ test-results/ @@ -31,6 +33,7 @@ packages/eve-buzz-acp-adapter/NOTICE # Logs *.log +/jobs/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md new file mode 100644 index 0000000000..149054b23b --- /dev/null +++ b/packages/eve-bench/README.md @@ -0,0 +1,105 @@ +# @eve/bench + +Terminal-Bench harness benchmarks for eve, run by a zero-dependency Node runner. + +The package owns the harness under evaluation (`agent/`), pinned dataset locks +and task cohorts (`datasets/`), and the runner (`src/`). Tasks keep the +Terminal-Bench layout and reward contract, so results stay comparable with +published runs. See `research/eve-bench-runner.md` for the design. + +## Prerequisites + +- Node.js 24 and pnpm +- Docker +- model-provider credentials in the environment, such as `AI_GATEWAY_API_KEY` +- a built local eve package when using `--eve local`; run `pnpm build` from the + repository root first + +Everything below runs from this directory with `pnpm bench` (alias for +`node src/cli.ts`). + +## Run + +```bash +pnpm bench tasks sync # fetch pinned datasets into .generated/datasets +pnpm bench tasks list --cohort smoke # print a cohort's task names + +AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8 +pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model +pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release + +pnpm bench report --format junit --out junit.xml +pnpm bench diff +``` + +Every command accepts `--json` and never prompts. Jobs live in +`.generated/jobs//`; re-running with the same `--job` name resumes it, +skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight +trials and removes their containers. + +## How a trial runs + +1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). +2. Start one container, upload the harness bundle to `/installed-agent`, and + run the harness command under the task's agent timeout. +3. Upload `tests/` and run `tests/test.sh` under the verifier timeout. The + verifier always runs, so the reward reflects what the agent left behind. +4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` + to the trial directory, remove the container, and write `trial.json`. + +## Harnesses + +- `eve` (default): builds `agent/` against the local checkout (`--eve local`) + or an exact published version, then packages the Nitro output and a small + runner that drives the server over HTTP. Build eve from the repository root + with `pnpm build` before using `--eve local`. For cache lookup, `prepare()` + derives the local eve identity from `packages/eve/dist` instead of packing + first. The cache lives in `.generated/cache/`. Each trial uploads the bundle + and only the Linux Node binary matching the task container's architecture, + for about 128 MB total. + + The agent's tools operate on the task working directory through + `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass + eve's nested sandbox on purpose. + +- `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner + and as the reward ceiling. + +Adding a harness means implementing `src/core/harness.ts` (prepare a +directory, return a command and env) and registering it in `src/cli.ts`. + +## Agent tools + +`src/tools/` exposes `run_bench`, `get_bench_result`, `diff_bench`, and +`list_bench_tasks` as eve `defineTool` definitions over the same core used by +the CLI. Their results use the same `JobResult`, `TaskDelta`, and task-list JSON +as the CLI. Add a file to an agent's `tools/` directory that re-exports a +definition by relative path, or copy the definition there. + +## Datasets and cohorts + +`datasets/-.json` pins a git URL, commit, and the task paths in +it. `datasets/cohorts/.json` names a reviewed task subset and the reason +it exists. Compare base and candidate runs with the same cohort, model, +attempt count, and eve source. + +## CI + +`.github/workflows/bench.yml` runs the oracle smoke cohort on pull requests that +touch this package or its design document and requires every task attempt to +resolve. It uploads the job directory and `junit.xml` as artifacts. + +The `eve-smoke` job runs only through `workflow_dispatch`, accepts `model` and +`cohort` inputs, and requires `AI_GATEWAY_API_KEY`. It uploads its job directory +and can optionally diff the result against a downloaded base-job artifact. + +## Layout + +- `agent/`: harness behavior under evaluation (model binding, instructions, tools) +- `src/core/`: runner library with no dependency on eve or third parties +- `src/harnesses/`: `eve` and `oracle` +- `src/tools/`: eve tool definitions over the runner core +- `src/options.ts`: run options shared by the CLI and tools +- `src/cli.ts`: `eve-bench` CLI +- `datasets/`: locks and cohorts +- `.generated/`: datasets, bundle cache, and job output; never committed diff --git a/packages/eve-bench/agent/agent.ts b/packages/eve-bench/agent/agent.ts new file mode 100644 index 0000000000..595843cf27 --- /dev/null +++ b/packages/eve-bench/agent/agent.ts @@ -0,0 +1,10 @@ +import { defineAgent } from "eve"; + +const model = process.env.EVE_BENCH_MODEL; +if (model === undefined || model.length === 0) { + throw new Error("EVE_BENCH_MODEL must be set by the eve-bench harness."); +} + +export default defineAgent({ + model, +}); diff --git a/packages/eve-bench/agent/instructions.md b/packages/eve-bench/agent/instructions.md new file mode 100644 index 0000000000..75ea47d355 --- /dev/null +++ b/packages/eve-bench/agent/instructions.md @@ -0,0 +1,11 @@ +You are an autonomous terminal task agent. + +Work until the requested task is complete or conclusively cannot be completed. + +- Inspect relevant files before modifying them. +- Use the available terminal and file tools to work directly in the task environment. +- Make the smallest correct changes that satisfy the task. +- Run relevant validation after making changes. +- Do not ask the user questions. +- Do not use network access unless the task explicitly requires it. +- Finish with a concise description of the completed work and validation. diff --git a/packages/eve-bench/agent/task-path.ts b/packages/eve-bench/agent/task-path.ts new file mode 100644 index 0000000000..8303fe355f --- /dev/null +++ b/packages/eve-bench/agent/task-path.ts @@ -0,0 +1,13 @@ +import { isAbsolute, resolve } from "node:path"; + +export function taskRoot(): string { + const root = process.env.EVE_BENCH_TASK_WORKDIR; + if (root === undefined || root.length === 0) { + throw new Error("EVE_BENCH_TASK_WORKDIR must be set by the eve-bench harness."); + } + return root; +} + +export function resolveTaskPath(path = "."): string { + return isAbsolute(path) ? path : resolve(taskRoot(), path); +} diff --git a/packages/eve-bench/agent/tools/ask_question.ts b/packages/eve-bench/agent/tools/ask_question.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/ask_question.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/bash.ts b/packages/eve-bench/agent/tools/bash.ts new file mode 100644 index 0000000000..2d8121c64d --- /dev/null +++ b/packages/eve-bench/agent/tools/bash.ts @@ -0,0 +1,37 @@ +import { exec } from "node:child_process"; +import { promisify } from "node:util"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { taskRoot } from "../task-path.js"; + +const execAsync = promisify(exec); + +export default defineTool({ + approval: never(), + description: + "Execute a shell command in the task environment. Commands start in the task working directory.", + inputSchema: z.object({ + command: z.string().describe("The shell command to execute."), + }), + async execute({ command }, ctx) { + try { + const { stdout, stderr } = await execAsync(command, { + cwd: taskRoot(), + env: process.env, + maxBuffer: 10 * 1024 * 1024, + signal: ctx.abortSignal, + }); + return { exitCode: 0, stderr, stdout }; + } catch (error) { + const result = error as Error & { code?: number; stderr?: string; stdout?: string }; + return { + exitCode: typeof result.code === "number" ? result.code : 1, + stderr: result.stderr ?? result.message, + stdout: result.stdout ?? "", + }; + } + }, +}); diff --git a/packages/eve-bench/agent/tools/glob.ts b/packages/eve-bench/agent/tools/glob.ts new file mode 100644 index 0000000000..764ac99ab9 --- /dev/null +++ b/packages/eve-bench/agent/tools/glob.ts @@ -0,0 +1,22 @@ +import { glob } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskRoot } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: "Find task-environment files whose paths match a glob pattern.", + inputSchema: z.object({ + pattern: z.string(), + path: z.string().optional().describe("Search root relative to the task working directory."), + }), + async execute({ pattern, path }) { + const cwd = path === undefined ? taskRoot() : resolveTaskPath(path); + const matches: string[] = []; + for await (const match of glob(pattern, { cwd })) matches.push(match); + return { matches: matches.sort() }; + }, +}); diff --git a/packages/eve-bench/agent/tools/grep.ts b/packages/eve-bench/agent/tools/grep.ts new file mode 100644 index 0000000000..5ce048f7de --- /dev/null +++ b/packages/eve-bench/agent/tools/grep.ts @@ -0,0 +1,45 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskRoot } from "../task-path.js"; + +const execFileAsync = promisify(execFile); + +export default defineTool({ + approval: never(), + description: + "Search task-environment text files with ripgrep and return matching lines with file names and line numbers.", + inputSchema: z.object({ + pattern: z.string().describe("Regular expression accepted by ripgrep."), + path: z + .string() + .optional() + .describe("File or directory relative to the task working directory."), + include: z.string().optional().describe("Optional glob restricting searched files."), + }), + async execute({ pattern, path, include }, ctx) { + const args = ["--line-number", "--no-heading", "--color=never"]; + if (include !== undefined) args.push("--glob", include); + args.push("--", pattern, path === undefined ? "." : resolveTaskPath(path)); + try { + const { stdout, stderr } = await execFileAsync("rg", args, { + cwd: taskRoot(), + maxBuffer: 10 * 1024 * 1024, + signal: ctx.abortSignal, + }); + return { exitCode: 0, stderr, stdout }; + } catch (error) { + const result = error as Error & { code?: number; stderr?: string; stdout?: string }; + const exitCode = typeof result.code === "number" ? result.code : 1; + return { + exitCode, + stderr: result.stderr ?? (exitCode === 1 ? "" : result.message), + stdout: result.stdout ?? "", + }; + } + }, +}); diff --git a/packages/eve-bench/agent/tools/read_file.ts b/packages/eve-bench/agent/tools/read_file.ts new file mode 100644 index 0000000000..c64d4fe2da --- /dev/null +++ b/packages/eve-bench/agent/tools/read_file.ts @@ -0,0 +1,27 @@ +import { readFile } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: + "Read a UTF-8 text file from the task environment, optionally selecting an inclusive 1-based line range.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + startLine: z.number().int().positive().optional(), + endLine: z.number().int().positive().optional(), + }), + async execute({ path, startLine, endLine }) { + if (startLine !== undefined && endLine !== undefined && startLine > endLine) { + throw new Error("startLine must not exceed endLine."); + } + const content = await readFile(resolveTaskPath(path), "utf8"); + if (startLine === undefined && endLine === undefined) return { content }; + const lines = content.split(/(?<=\n)/u); + return { content: lines.slice((startLine ?? 1) - 1, endLine).join("") }; + }, +}); diff --git a/packages/eve-bench/agent/tools/todo.ts b/packages/eve-bench/agent/tools/todo.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/todo.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_fetch.ts b/packages/eve-bench/agent/tools/web_fetch.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_fetch.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_search.ts b/packages/eve-bench/agent/tools/web_search.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_search.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/write_file.ts b/packages/eve-bench/agent/tools/write_file.ts new file mode 100644 index 0000000000..65d547a02d --- /dev/null +++ b/packages/eve-bench/agent/tools/write_file.ts @@ -0,0 +1,24 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: + "Create or replace a UTF-8 text file in the task environment. Parent directories are created automatically.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + content: z.string(), + }), + async execute({ path, content }) { + const resolved = resolveTaskPath(path); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile(resolved, content, "utf8"); + return { bytesWritten: Buffer.byteLength(content), path: resolved }; + }, +}); diff --git a/packages/eve-bench/datasets/cohorts/smoke.json b/packages/eve-bench/datasets/cohorts/smoke.json new file mode 100644 index 0000000000..fe2d0a37f9 --- /dev/null +++ b/packages/eve-bench/datasets/cohorts/smoke.json @@ -0,0 +1,11 @@ +{ + "dataset": "terminal-bench-2.0", + "description": "Short, deterministic tasks with small images that exercise shell, file editing, and git tooling. Use for harness changes before running the full dataset.", + "tasks": [ + "fix-git", + "openssl-selfsigned-cert", + "regex-log", + "sqlite-db-truncate", + "log-summary-date-ranges" + ] +} diff --git a/packages/eve-bench/datasets/terminal-bench-2.0.json b/packages/eve-bench/datasets/terminal-bench-2.0.json new file mode 100644 index 0000000000..c13b5eaf35 --- /dev/null +++ b/packages/eve-bench/datasets/terminal-bench-2.0.json @@ -0,0 +1,97 @@ +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": [ + "adaptive-rejection-sampler", + "bn-fit-modify", + "break-filter-js-from-html", + "build-cython-ext", + "build-pmars", + "build-pov-ray", + "caffe-cifar-10", + "cancel-async-tasks", + "chess-best-move", + "circuit-fibsqrt", + "cobol-modernization", + "code-from-image", + "compile-compcert", + "configure-git-webserver", + "constraints-scheduling", + "count-dataset-tokens", + "crack-7z-hash", + "custom-memory-heap-crash", + "db-wal-recovery", + "distribution-search", + "dna-assembly", + "dna-insert", + "extract-elf", + "extract-moves-from-video", + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis", + "filter-js-from-html", + "financial-document-processor", + "fix-code-vulnerability", + "fix-git", + "fix-ocaml-gc", + "gcode-to-text", + "git-leak-recovery", + "git-multibranch", + "gpt2-codegolf", + "headless-terminal", + "hf-model-inference", + "install-windows-3.11", + "kv-store-grpc", + "large-scale-text-editing", + "largest-eigenval", + "llm-inference-batching-scheduler", + "log-summary-date-ranges", + "mailman", + "make-doom-for-mips", + "make-mips-interpreter", + "mcmc-sampling-stan", + "merge-diff-arc-agi-task", + "model-extraction-relu-logits", + "modernize-scientific-stack", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "nginx-request-logging", + "openssl-selfsigned-cert", + "overfull-hbox", + "password-recovery", + "path-tracing", + "path-tracing-reverse", + "polyglot-c-py", + "polyglot-rust-c", + "portfolio-optimization", + "protein-assembly", + "prove-plus-comm", + "pypi-server", + "pytorch-model-cli", + "pytorch-model-recovery", + "qemu-alpine-ssh", + "qemu-startup", + "query-optimize", + "raman-fitting", + "regex-chess", + "regex-log", + "reshard-c4-data", + "rstan-to-pystan", + "sam-cell-seg", + "sanitize-git-repo", + "schemelike-metacircular-eval", + "sparql-university", + "sqlite-db-truncate", + "sqlite-with-gcov", + "torch-pipeline-parallelism", + "torch-tensor-parallelism", + "train-fasttext", + "tune-mjcf", + "video-processing", + "vulnerable-secret", + "winning-avg-corewars", + "write-compressor" + ] +} diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json new file mode 100644 index 0000000000..c06941682a --- /dev/null +++ b/packages/eve-bench/package.json @@ -0,0 +1,26 @@ +{ + "name": "@eve/bench", + "version": "0.0.0", + "private": true, + "description": "Terminal-Bench harness benchmarks for eve.", + "bin": { + "eve-bench": "./src/cli.ts" + }, + "type": "module", + "scripts": { + "bench": "node src/cli.ts", + "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "eve": "workspace:*", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + }, + "engines": { + "node": ">=24" + } +} diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts new file mode 100644 index 0000000000..4c63c3d549 --- /dev/null +++ b/packages/eve-bench/src/cli.ts @@ -0,0 +1,175 @@ +#!/usr/bin/env node +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; + +import { syncDataset, taskDirs } from "./core/dataset.ts"; +import { readJobResult, runJob } from "./core/job.ts"; +import { diffJobs, formatDiff, formatReport, type ReportFormat } from "./core/report.ts"; +import { loadTask } from "./core/task.ts"; +import { + allLocks, + defaultJobName, + forwardedEnv, + jobDir, + lock, + paths, + selectHarness, + selectTasks, +} from "./options.ts"; + +const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve + + eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets + eve-bench tasks list [--cohort ] print task names + eve-bench run --model [--cohort ] [--task ...] [--attempts N] [--concurrency N] + [--harness eve|oracle] [--eve local|] [--job ] [--format console|json|junit] + eve-bench report [--format console|json|junit] + eve-bench diff [--json] + +Jobs live in .generated/jobs/; re-running with the same --job resumes it. +Provider credentials are read from the environment and forwarded into the task container. +`; + +const [command, subcommand, ...rest] = process.argv.slice(2); +try { + switch (command) { + case "tasks": + await tasks(subcommand, rest); + break; + case "run": + await run(subcommand === undefined ? [] : [subcommand, ...rest]); + break; + case "report": + await report(subcommand, rest); + break; + case "diff": + await diff(subcommand, rest); + break; + default: + process.stdout.write(USAGE); + process.exitCode = command === undefined || command === "--help" || command === "-h" ? 0 : 2; + } +} catch (error) { + process.stderr.write(`eve-bench: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} + +async function tasks(sub: string | undefined, args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { dataset: { type: "string" }, cohort: { type: "string" }, json: { type: "boolean" } }, + }); + if (sub === "sync") { + const locks = values.dataset ? [await lock(values.dataset)] : await allLocks(); + for (const entry of locks) { + const dir = await syncDataset(join(paths.generatedRoot, "datasets"), entry); + process.stdout.write( + values.json + ? `${JSON.stringify({ dataset: entry.name, version: entry.version, dir })}\n` + : `${entry.name}@${entry.version} -> ${dir}\n`, + ); + } + return; + } + if (sub === "list") { + const selection = await selectTasks(values); + process.stdout.write( + values.json ? `${JSON.stringify(selection.tasks)}\n` : `${selection.tasks.join("\n")}\n`, + ); + return; + } + throw new Error(`unknown tasks subcommand: ${sub ?? "(none)"}\n${USAGE}`); +} + +async function run(args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { + model: { type: "string" }, + cohort: { type: "string" }, + dataset: { type: "string" }, + task: { type: "string", multiple: true }, + attempts: { type: "string", default: "1" }, + concurrency: { type: "string", default: "4" }, + eve: { type: "string", default: "local" }, + harness: { type: "string", default: "eve" }, + job: { type: "string" }, + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + }, + }); + const harness = selectHarness(values.harness, values.eve); + const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); + if (!model) throw new Error("--model is required"); + const selection = await selectTasks(values); + const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const loaded = await Promise.all( + (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + ); + const name = values.job ?? defaultJobName(`${harness.name}-${model}`); + const controller = new AbortController(); + const stop = () => { + process.stderr.write("\neve-bench: aborting, cleaning up containers...\n"); + controller.abort(); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + const format = reportFormat(values.json ? "json" : values.format); + const result = await runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks: loaded, + harness, + model, + attempts: Number(values.attempts), + concurrency: Number(values.concurrency), + forwardEnv: forwardedEnv(), + signal: controller.signal, + onTrial: (trial, resumed) => { + if (format !== "console") return; + const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); + process.stderr.write( + `${resumed ? "resume" : "trial "} ${trial.task}#${trial.attempt} reward=${reward} agent=${trial.agent.status} verifier=${trial.verifier.status}${trial.error ? ` error=${trial.error}` : ""}\n`, + ); + }, + }); + process.stdout.write(formatReport(result, format)); + if (controller.signal.aborted) process.exitCode = 130; +} + +async function report(job: string | undefined, args: string[]): Promise { + if (!job) throw new Error("report requires a job name"); + const { values } = parseArgs({ + args, + options: { + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + out: { type: "string" }, + }, + }); + const output = formatReport( + await readJobResult(join(paths.generatedRoot, "jobs", job)), + reportFormat(values.json ? "json" : values.format), + ); + if (values.out) await writeFile(values.out, output); + else process.stdout.write(output); +} + +async function diff(base: string | undefined, args: string[]): Promise { + const [candidate, ...restArgs] = args; + if (!base || !candidate) throw new Error("diff requires "); + const { values } = parseArgs({ args: restArgs, options: { json: { type: "boolean" } } }); + const deltas = diffJobs( + await readJobResult(jobDir(base)), + await readJobResult(jobDir(candidate)), + ); + process.stdout.write(values.json ? `${JSON.stringify(deltas, null, 2)}\n` : formatDiff(deltas)); +} + +function reportFormat(value: string | undefined): ReportFormat { + if (value === "console" || value === "json" || value === "junit") return value; + throw new Error(`unknown format: ${value}`); +} diff --git a/packages/eve-bench/src/core/dataset.test.ts b/packages/eve-bench/src/core/dataset.test.ts new file mode 100644 index 0000000000..c199c11519 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.test.ts @@ -0,0 +1,103 @@ +// syncDataset is intentionally untested because it performs network and git operations. + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { datasetDir, readCohort, readDatasetLock, taskDirs } from "./dataset.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-dataset-")); +after(() => rm(root, { recursive: true, force: true })); + +const VALID_LOCK = { + name: "terminal-bench", + version: "2.0", + gitUrl: "https://github.com/laude-institute/terminal-bench-2.git", + commit: "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + tasks: ["adaptive-rejection-sampler", "fix-git"], +}; + +test("parses a valid dataset lock", async () => { + const path = join(root, "valid-lock.json"); + await writeFile(path, JSON.stringify(VALID_LOCK)); + + const lock = await readDatasetLock(path); + + assert.deepEqual(lock, VALID_LOCK); +}); + +for (const field of ["name", "version", "gitUrl", "commit"] as const) { + test(`rejects a dataset lock missing ${field}`, async () => { + const path = join(root, `missing-${field}.json`); + const lock: Record = { ...VALID_LOCK }; + delete lock[field]; + await writeFile(path, JSON.stringify(lock)); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); + + test(`rejects a dataset lock with an empty ${field}`, async () => { + const path = join(root, `empty-${field}.json`); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, [field]: "" })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); +} + +test("rejects a dataset lock with an empty tasks array", async () => { + const path = join(root, "empty-lock-tasks.json"); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, tasks: [] })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("rejects a cohort missing dataset", async () => { + const path = join(root, "missing-cohort-dataset.json"); + await writeFile(path, JSON.stringify({ tasks: ["fix-git"] })); + + await assert.rejects(readCohort(path), { + message: `${path}: missing "dataset"`, + }); +}); + +test("rejects a cohort with an empty tasks array", async () => { + const path = join(root, "empty-cohort-tasks.json"); + await writeFile(path, JSON.stringify({ dataset: "terminal-bench-2.0", tasks: [] })); + + await assert.rejects(readCohort(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("builds the versioned dataset directory path", () => { + assert.equal(datasetDir(root, VALID_LOCK), join(root, "terminal-bench-2.0")); +}); + +test("resolves task directories containing task.toml", async () => { + const dir = join(root, "tasks-valid"); + await mkdir(join(dir, "task-a"), { recursive: true }); + await mkdir(join(dir, "nested", "task-b"), { recursive: true }); + await writeFile(join(dir, "task-a", "task.toml"), ""); + await writeFile(join(dir, "nested", "task-b", "task.toml"), ""); + + const dirs = await taskDirs(dir, ["task-a", "nested/task-b"]); + + assert.deepEqual(dirs, [join(dir, "task-a"), join(dir, "nested", "task-b")]); +}); + +test("rejects when any task directory lacks task.toml", async () => { + const dir = join(root, "tasks-missing-toml"); + await mkdir(join(dir, "present"), { recursive: true }); + await mkdir(join(dir, "missing"), { recursive: true }); + await writeFile(join(dir, "present", "task.toml"), ""); + + await assert.rejects(taskDirs(dir, ["present", "missing"]), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/dataset.ts b/packages/eve-bench/src/core/dataset.ts new file mode 100644 index 0000000000..beaa588dd5 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.ts @@ -0,0 +1,77 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface DatasetLock { + readonly name: string; + readonly version: string; + readonly gitUrl: string; + readonly commit: string; + readonly tasks: readonly string[]; +} + +export interface Cohort { + readonly dataset: string; + readonly description?: string; + readonly tasks: readonly string[]; +} + +export async function readDatasetLock(path: string): Promise { + const lock = JSON.parse(await readFile(path, "utf8")) as DatasetLock; + for (const key of ["name", "version", "gitUrl", "commit"] as const) { + if (typeof lock[key] !== "string" || lock[key].length === 0) { + throw new Error(`${path}: missing "${key}"`); + } + } + if (!Array.isArray(lock.tasks) || lock.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return lock; +} + +export async function readCohort(path: string): Promise { + const cohort = JSON.parse(await readFile(path, "utf8")) as Cohort; + if (typeof cohort.dataset !== "string") throw new Error(`${path}: missing "dataset"`); + if (!Array.isArray(cohort.tasks) || cohort.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return cohort; +} + +export function datasetDir(root: string, lock: DatasetLock): string { + return join(root, `${lock.name}-${lock.version}`); +} + +/** Fetches the pinned commit with a sparse, blobless checkout of the locked tasks. Idempotent. */ +export async function syncDataset(root: string, lock: DatasetLock): Promise { + const dir = datasetDir(root, lock); + if (await isSynced(dir, lock)) return dir; + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + const git = (...args: string[]) => + execFileAsync("git", ["-C", dir, ...args], { maxBuffer: 64 * 1024 * 1024 }); + await git("init", "--quiet"); + await git("remote", "add", "origin", lock.gitUrl); + await git("sparse-checkout", "set", "--cone", ...lock.tasks); + await git("fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", lock.commit); + await git("checkout", "--quiet", lock.commit); + return dir; +} + +async function isSynced(dir: string, lock: DatasetLock): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", dir, "rev-parse", "HEAD"]); + if (stdout.trim() !== lock.commit) return false; + const present = new Set(await readdir(dir)); + return lock.tasks.every((task) => present.has(task)); + } catch { + return false; + } +} + +export async function taskDirs(dir: string, tasks: readonly string[]): Promise { + const dirs = tasks.map((task) => join(dir, task)); + await Promise.all(dirs.map((task) => access(join(task, "task.toml")))); + return dirs; +} diff --git a/packages/eve-bench/src/core/docker.ts b/packages/eve-bench/src/core/docker.ts new file mode 100644 index 0000000000..039ce37451 --- /dev/null +++ b/packages/eve-bench/src/core/docker.ts @@ -0,0 +1,177 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import type { Writable } from "node:stream"; + +export interface ProcessResult { + readonly exitCode: number | null; + readonly timedOut: boolean; + readonly aborted: boolean; + readonly stdout: string; + readonly stderr: string; +} + +export interface ExecOptions { + readonly cwd?: string; + readonly env?: Readonly>; + readonly timeoutMs?: number; + readonly signal?: AbortSignal; + readonly log?: Writable; +} + +export interface ContainerOptions { + readonly image: string; + readonly name: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly network: boolean; +} + +export class Container { + readonly name: string; + + private constructor(name: string) { + this.name = name; + } + + static async start(options: ContainerOptions, signal?: AbortSignal): Promise { + const args = ["run", "--detach", "--name", options.name, "--entrypoint", "sh"]; + if (options.cpus !== undefined) args.push("--cpus", String(options.cpus)); + if (options.memoryMb !== undefined) args.push("--memory", `${options.memoryMb}m`); + if (!options.network) args.push("--network", "none"); + args.push(options.image, "-c", "sleep infinity"); + await dockerOk(args, { signal }); + return new Container(options.name); + } + + /** + * Runs `command` through `sh -c` inside the container. A `docker exec` + * process is its own session and process-group leader, so recording its + * pid lets timeouts and aborts kill the whole group in-container; killing + * the host-side client alone would leave the workload running. + */ + async exec(command: string, options: ExecOptions = {}): Promise { + const pidFile = `/tmp/.eve-bench-${randomUUID()}.pid`; + const wrapped = `echo $$ > ${pidFile}; exec sh -c "$0"`; + const args = ["exec"]; + if (options.cwd) args.push("--workdir", options.cwd); + for (const [key, value] of Object.entries(options.env ?? {})) + args.push("--env", `${key}=${value}`); + args.push(this.name, "sh", "-c", wrapped, command); + const result = await docker(args, options); + if (result.timedOut || result.aborted) { + await docker( + [ + "exec", + this.name, + "sh", + "-c", + `pid=$(cat ${pidFile} 2>/dev/null); [ -n "$pid" ] && { kill -KILL -- -$pid 2>/dev/null; kill -KILL $pid 2>/dev/null; }; true`, + ], + { timeoutMs: 10_000 }, + ); + } + return result; + } + + async upload(hostPath: string, containerPath: string, signal?: AbortSignal): Promise { + await dockerOk(["cp", hostPath, `${this.name}:${containerPath}`], { signal }); + } + + async download(containerPath: string, hostPath: string, signal?: AbortSignal): Promise { + const result = await docker(["cp", `${this.name}:${containerPath}`, hostPath], { signal }); + return result.exitCode === 0; + } + + async remove(): Promise { + await docker(["rm", "--force", "--volumes", this.name], { timeoutMs: 60_000 }); + } +} + +export async function imageExists(tag: string): Promise { + const result = await docker(["image", "inspect", tag], { timeoutMs: 30_000 }); + return result.exitCode === 0; +} + +export async function pullImage(image: string, options: ExecOptions = {}): Promise { + if (await imageExists(image)) return; + await dockerOk(["pull", image], options); +} + +export async function buildImage( + tag: string, + contextDir: string, + options: ExecOptions = {}, +): Promise { + if (await imageExists(tag)) return; + await dockerOk(["build", "--tag", tag, contextDir], options); +} + +export async function containerArch(container: Container): Promise<"x64" | "arm64"> { + const result = await container.exec("uname -m", { timeoutMs: 10_000 }); + const machine = result.stdout.trim(); + if (machine === "x86_64") return "x64"; + if (machine === "aarch64") return "arm64"; + throw new Error(`Unsupported container architecture: ${machine || result.stderr.trim()}`); +} + +async function dockerOk(args: readonly string[], options: ExecOptions): Promise { + const result = await docker(args, options); + if (result.exitCode !== 0) { + const reason = result.timedOut + ? "timed out" + : result.aborted + ? "aborted" + : `exit ${result.exitCode}`; + throw new Error(`docker ${args[0]} ${reason}: ${result.stderr.trim() || result.stdout.trim()}`); + } + return result; +} + +function docker(args: readonly string[], options: ExecOptions): Promise { + return new Promise((resolve, reject) => { + const child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + let aborted = false; + const timer = + options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.timeoutMs); + const onAbort = () => { + aborted = true; + child.kill("SIGKILL"); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + child.stdout.on("data", (chunk: Buffer) => { + stdout.push(chunk); + options.log?.write(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr.push(chunk); + options.log?.write(chunk); + }); + child.once("error", (error) => { + cleanup(); + reject(error); + }); + child.once("close", (code) => { + cleanup(); + resolve({ + exitCode: code, + timedOut, + aborted, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + function cleanup() { + if (timer) clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + } + }); +} diff --git a/packages/eve-bench/src/core/harness.ts b/packages/eve-bench/src/core/harness.ts new file mode 100644 index 0000000000..797d481d27 --- /dev/null +++ b/packages/eve-bench/src/core/harness.ts @@ -0,0 +1,34 @@ +/** + * A harness is anything that can be placed in a task container and told to + * solve an instruction. The runner uploads `bundleDir` to `/installed-agent`, + * then runs `command` through `sh -c` in the task working directory. + */ +export interface Harness { + readonly name: string; + prepare(ctx: HarnessPrepareContext): Promise; + /** Optional per-task host paths uploaded into the install directory before `command` runs. */ + stage?(ctx: { readonly taskDir: string }): readonly string[]; + command(ctx: HarnessRunContext): string; + env(ctx: HarnessRunContext): Record; +} + +export interface HarnessPrepareContext { + readonly model: string; + readonly cacheDir: string; +} + +export interface HarnessBundle { + readonly dir: string; + /** Architecture-specific host directories uploaded to `${installDir}/arch`. */ + readonly arch?: Readonly>; + /** Recorded in results so a run can be traced back to the exact harness build. */ + readonly provenance?: Record; +} + +export interface HarnessRunContext { + readonly model: string; + readonly installDir: string; + readonly instructionPath: string; + readonly taskWorkdir: string; + readonly logsDir: string; +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts new file mode 100644 index 0000000000..d16feead4e --- /dev/null +++ b/packages/eve-bench/src/core/job.ts @@ -0,0 +1,108 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import type { Harness } from "./harness.ts"; +import { summarize, type JobResult, type TrialResult } from "./result.ts"; +import type { Task } from "./task.ts"; +import { runTrial } from "./trial.ts"; + +export interface JobInput { + readonly name: string; + readonly dir: string; + readonly cacheDir: string; + readonly dataset: DatasetLock; + readonly tasks: readonly Task[]; + readonly harness: Harness; + readonly model: string; + readonly attempts: number; + readonly concurrency: number; + readonly forwardEnv: Readonly>; + readonly signal: AbortSignal; + readonly onTrial?: (trial: TrialResult, resumed: boolean) => void; +} + +export async function runJob(input: JobInput): Promise { + await mkdir(input.dir, { recursive: true }); + const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); + await writeFile( + join(input.dir, "job.json"), + `${JSON.stringify( + { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + attempts: input.attempts, + tasks: input.tasks.map((task) => task.name), + bundle: bundle.provenance ?? {}, + startedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + ); + + const queue = input.tasks.flatMap((task) => + Array.from({ length: input.attempts }, (_, index) => ({ task, attempt: index + 1 })), + ); + const trials: TrialResult[] = []; + const workers = Array.from({ length: Math.max(1, input.concurrency) }, async () => { + for (;;) { + const next = queue.shift(); + if (!next || input.signal.aborted) return; + const dir = join(input.dir, next.task.name, String(next.attempt)); + const existing = await readTrial(dir); + if (existing) { + trials.push(existing); + input.onTrial?.(existing, true); + continue; + } + const trial = await runTrial({ + task: next.task, + attempt: next.attempt, + harness: input.harness, + bundle, + model: input.model, + forwardEnv: input.forwardEnv, + dir, + signal: input.signal, + }); + trials.push(trial); + input.onTrial?.(trial, false); + } + }); + await Promise.all(workers); + + trials.sort((a, b) => a.task.localeCompare(b.task) || a.attempt - b.attempt); + const result: JobResult = { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + trials, + summary: summarize(trials), + }; + await writeFile(join(input.dir, "result.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +export async function readJobResult(dir: string): Promise { + return JSON.parse(await readFile(join(dir, "result.json"), "utf8")) as JobResult; +} + +async function readTrial(dir: string): Promise { + try { + return JSON.parse(await readFile(join(dir, "trial.json"), "utf8")) as TrialResult; + } catch { + return undefined; + } +} diff --git a/packages/eve-bench/src/core/report.test.ts b/packages/eve-bench/src/core/report.test.ts new file mode 100644 index 0000000000..70ff0e7390 --- /dev/null +++ b/packages/eve-bench/src/core/report.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { diffJobs, formatDiff, formatReport } from "./report.ts"; +import { summarize, type JobResult, type TrialResult } from "./result.ts"; + +function trial( + task: string, + attempt: number, + reward: number | null, + extra: Partial = {}, +): TrialResult { + return { + task, + attempt, + harness: "eve@local", + model: "test/model", + reward, + agent: { status: "completed", exitCode: 0, durationMs: 1000 }, + verifier: { status: "completed", exitCode: 0, durationMs: 500 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.500Z", + ...extra, + }; +} + +function job(name: string, trials: TrialResult[]): JobResult { + return { + job: name, + dataset: { name: "terminal-bench", version: "2.0", commit: "abc" }, + harness: "eve@local", + model: "test/model", + trials, + summary: summarize(trials), + }; +} + +test("summarize counts resolved attempts and mean reward", () => { + const summary = summarize([trial("a", 1, 1), trial("a", 2, 0), trial("b", 1, null)]); + assert.deepEqual(summary, { tasks: 2, attempts: 3, resolved: 1, meanReward: 1 / 3 }); +}); + +test("junit report marks unresolved trials as failures", () => { + const output = formatReport( + job("j", [trial("a", 1, 1), trial("b", 1, 0, { error: "boom " })]), + "junit", + ); + assert.match(output, /tests="2" failures="1"/u); + assert.match(output, //u); + assert.match(output, //u); +}); + +test("diff reports per-task mean reward deltas", () => { + const base = job("base", [trial("a", 1, 0), trial("a", 2, 1), trial("b", 1, 1)]); + const candidate = job("cand", [trial("a", 1, 1), trial("a", 2, 1), trial("c", 1, 0)]); + const deltas = diffJobs(base, candidate); + assert.deepEqual(deltas, [ + { task: "a", base: 0.5, candidate: 1, delta: 0.5 }, + { task: "b", base: 1, candidate: null, delta: null }, + { task: "c", base: null, candidate: 0, delta: null }, + ]); + assert.match(formatDiff(deltas), /1 task\(s\) changed, net \+0.500/u); +}); diff --git a/packages/eve-bench/src/core/report.ts b/packages/eve-bench/src/core/report.ts new file mode 100644 index 0000000000..5a931878da --- /dev/null +++ b/packages/eve-bench/src/core/report.ts @@ -0,0 +1,118 @@ +import type { JobResult, TrialResult } from "./result.ts"; + +export type ReportFormat = "console" | "json" | "junit"; + +export function formatReport(result: JobResult, format: ReportFormat): string { + switch (format) { + case "json": + return `${JSON.stringify(result, null, 2)}\n`; + case "junit": + return junit(result); + case "console": + return console_(result); + } +} + +function console_(result: JobResult): string { + const lines = [ + `${result.job} ${result.harness} / ${result.model} ${result.dataset.name}@${result.dataset.version}`, + "", + ]; + const byTask = groupByTask(result.trials); + for (const [task, trials] of byTask) { + const rewards = trials.map((trial) => (trial.reward === null ? "-" : trial.reward.toFixed(2))); + const errors = trials.filter((trial) => trial.error || trial.agent.status === "timeout").length; + lines.push( + `${task.padEnd(44)} ${rewards.join(" ").padEnd(16)} ${errors > 0 ? `${errors} error(s)` : ""}`.trimEnd(), + ); + } + const { summary } = result; + lines.push( + "", + `tasks ${summary.tasks} attempts ${summary.attempts} resolved ${summary.resolved} mean reward ${summary.meanReward.toFixed(3)}`, + ); + return `${lines.join("\n")}\n`; +} + +function junit(result: JobResult): string { + const failures = result.trials.filter((trial) => (trial.reward ?? 0) < 1).length; + const cases = result.trials.map((trial) => { + const time = ((trial.agent.durationMs + trial.verifier.durationMs) / 1000).toFixed(3); + const name = escape(`${trial.task} #${trial.attempt}`); + if ((trial.reward ?? 0) >= 1) return ` `; + const message = escape( + trial.error ?? + `reward ${trial.reward ?? "none"}; agent ${trial.agent.status}; verifier ${trial.verifier.status}`, + ); + return ` \n \n `; + }); + return [ + ``, + ``, + ` `, + ...cases, + ` `, + ``, + "", + ].join("\n"); +} + +export interface TaskDelta { + readonly task: string; + readonly base: number | null; + readonly candidate: number | null; + readonly delta: number | null; +} + +export function diffJobs(base: JobResult, candidate: JobResult): TaskDelta[] { + const baseMeans = meanByTask(base.trials); + const candidateMeans = meanByTask(candidate.trials); + const tasks = [...new Set([...baseMeans.keys(), ...candidateMeans.keys()])].sort(); + return tasks.map((task) => { + const b = baseMeans.get(task) ?? null; + const c = candidateMeans.get(task) ?? null; + return { task, base: b, candidate: c, delta: b === null || c === null ? null : c - b }; + }); +} + +export function formatDiff(deltas: readonly TaskDelta[]): string { + const lines = deltas.map((row) => { + const fmt = (value: number | null) => + value === null ? " - " : value.toFixed(3).padStart(6); + const delta = + row.delta === null + ? "" + : row.delta === 0 + ? "" + : row.delta > 0 + ? `+${row.delta.toFixed(3)}` + : row.delta.toFixed(3); + return `${row.task.padEnd(44)} ${fmt(row.base)} -> ${fmt(row.candidate)} ${delta}`.trimEnd(); + }); + const changed = deltas.filter((row) => row.delta !== null && row.delta !== 0); + const net = changed.reduce((sum, row) => sum + (row.delta ?? 0), 0); + lines.push("", `${changed.length} task(s) changed, net ${net >= 0 ? "+" : ""}${net.toFixed(3)}`); + return `${lines.join("\n")}\n`; +} + +function groupByTask(trials: readonly TrialResult[]): Map { + const groups = new Map(); + for (const trial of trials) { + const list = groups.get(trial.task) ?? []; + list.push(trial); + groups.set(trial.task, list); + } + return groups; +} + +function meanByTask(trials: readonly TrialResult[]): Map { + const means = new Map(); + for (const [task, group] of groupByTask(trials)) { + means.set(task, group.reduce((sum, trial) => sum + (trial.reward ?? 0), 0) / group.length); + } + return means; +} + +function escape(text: string): string { + return text.replaceAll("&", "&").replaceAll("<", "<").replaceAll('"', """); +} diff --git a/packages/eve-bench/src/core/result.ts b/packages/eve-bench/src/core/result.ts new file mode 100644 index 0000000000..d42e58a798 --- /dev/null +++ b/packages/eve-bench/src/core/result.ts @@ -0,0 +1,52 @@ +export type StepStatus = "completed" | "timeout" | "failed" | "skipped"; + +export interface StepResult { + readonly status: StepStatus; + readonly exitCode: number | null; + readonly durationMs: number; +} + +export interface Usage { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cachedTokens: number; + readonly costUsd: number; +} + +export interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: Usage; + readonly error?: string; + readonly startedAt: string; + readonly finishedAt: string; +} + +export interface JobResult { + readonly job: string; + readonly dataset: { readonly name: string; readonly version: string; readonly commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: JobSummary; +} + +export interface JobSummary { + readonly tasks: number; + readonly attempts: number; + readonly resolved: number; + readonly meanReward: number; +} + +export function summarize(trials: readonly TrialResult[]): JobSummary { + const tasks = new Set(trials.map((trial) => trial.task)).size; + const rewards = trials.map((trial) => trial.reward ?? 0); + const resolved = rewards.filter((reward) => reward >= 1).length; + const meanReward = rewards.length === 0 ? 0 : rewards.reduce((a, b) => a + b, 0) / rewards.length; + return { tasks, attempts: trials.length, resolved, meanReward }; +} diff --git a/packages/eve-bench/src/core/task.test.ts b/packages/eve-bench/src/core/task.test.ts new file mode 100644 index 0000000000..7767b05c41 --- /dev/null +++ b/packages/eve-bench/src/core/task.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { loadTask } from "./task.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-task-")); +after(() => rm(root, { recursive: true, force: true })); + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[agent] +timeout_sec = 1.234 + +[verifier] +timeout_sec = 2.789 + +[environment] +build_timeout_sec = 3.456 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1.5 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +async function writeTask(dir: string, toml = TASK_TOML): Promise { + await mkdir(join(dir, "tests"), { recursive: true }); + await writeFile(join(dir, "task.toml"), toml); + await writeFile(join(dir, "instruction.md"), "Solve the task.\n"); + await writeFile(join(dir, "tests", "test.sh"), "#!/bin/sh\n"); +} + +test("loads Terminal-Bench task fields and converts timeout seconds to milliseconds", async () => { + const dir = join(root, "complete"); + await writeTask(dir); + + const task = await loadTask(dir); + + assert.equal(task.name, "adaptive-rejection-sampler"); + assert.equal(task.dir, dir); + assert.equal(task.instruction, "Solve the task.\n"); + assert.equal(task.agentTimeoutMs, 1234); + assert.equal(task.verifierTimeoutMs, 2789); + assert.equal(task.buildTimeoutMs, 3456); + assert.equal(task.environment.dockerImage, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(task.environment.cpus, 1.5); + assert.equal(task.environment.memoryMb, 2048); + assert.equal(task.environment.allowInternet, true); +}); + +test('converts older-schema memory = "2G" to 2048 MB', async () => { + const dir = join(root, "memory-2g"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 2048); +}); + +test('converts older-schema memory = "512M" to 512 MB', async () => { + const dir = join(root, "memory-512m"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "512M"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 512); +}); + +test("prefers memory_mb when both memory fields are present", async () => { + const dir = join(root, "memory-precedence"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory_mb = 768\nmemory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 768); +}); + +test("uses 15-minute defaults when agent and verifier timeouts are missing", async () => { + const dir = join(root, "default-timeouts"); + const toml = TASK_TOML.replace("[agent]\ntimeout_sec = 1.234\n\n", "").replace( + "[verifier]\ntimeout_sec = 2.789\n\n", + "", + ); + await writeTask(dir, toml); + + const task = await loadTask(dir); + + assert.equal(task.agentTimeoutMs, 900_000); + assert.equal(task.verifierTimeoutMs, 900_000); +}); + +test("allows internet when allow_internet is absent", async () => { + const dir = join(root, "internet-default"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true\n", "")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, true); +}); + +test("disallows internet when allow_internet is false", async () => { + const dir = join(root, "internet-false"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true", "allow_internet = false")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, false); +}); + +test("rejects a task with neither docker_image nor environment/Dockerfile", async () => { + const dir = join(root, "missing-image"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); + +test("loads environment/Dockerfile when docker_image is absent", async () => { + const dir = join(root, "dockerfile"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + await mkdir(join(dir, "environment")); + await writeFile(join(dir, "environment", "Dockerfile"), "FROM scratch\n"); + + const task = await loadTask(dir); + + assert.equal(task.environment.dockerImage, undefined); + assert.equal(task.environment.dockerfileDir, join(dir, "environment")); +}); + +test("rejects a task missing tests/test.sh", async () => { + const dir = join(root, "missing-test"); + await writeTask(dir); + await rm(join(dir, "tests", "test.sh")); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/task.ts b/packages/eve-bench/src/core/task.ts new file mode 100644 index 0000000000..8788b8e74b --- /dev/null +++ b/packages/eve-bench/src/core/task.ts @@ -0,0 +1,82 @@ +import { access, readFile } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import { parseToml, type TomlTable } from "./toml.ts"; + +export interface Task { + readonly name: string; + readonly dir: string; + readonly instruction: string; + readonly agentTimeoutMs: number; + readonly verifierTimeoutMs: number; + readonly buildTimeoutMs: number; + readonly environment: { + readonly dockerImage?: string; + readonly dockerfileDir: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly allowInternet: boolean; + }; +} + +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; + +export async function loadTask(dir: string): Promise { + const toml = parseToml(await readFile(join(dir, "task.toml"), "utf8")); + const instruction = await readFile(join(dir, "instruction.md"), "utf8"); + await access(join(dir, "tests", "test.sh")); + const task = table(toml, "task"); + const agent = table(toml, "agent"); + const verifier = table(toml, "verifier"); + const environment = table(toml, "environment"); + const dockerImage = str(environment, "docker_image"); + const dockerfileDir = join(dir, "environment"); + if (dockerImage === undefined) await access(join(dockerfileDir, "Dockerfile")); + return { + name: shortName(str(task, "name") ?? basename(dir)), + dir, + instruction, + agentTimeoutMs: seconds(agent, "timeout_sec"), + verifierTimeoutMs: seconds(verifier, "timeout_sec"), + buildTimeoutMs: seconds(environment, "build_timeout_sec"), + environment: { + dockerImage, + dockerfileDir, + cpus: num(environment, "cpus"), + memoryMb: num(environment, "memory_mb") ?? memoryMb(str(environment, "memory")), + allowInternet: environment.allow_internet !== false, + }, + }; +} + +function shortName(name: string): string { + return name.includes("/") ? name.slice(name.lastIndexOf("/") + 1) : name; +} + +function table(root: TomlTable, key: string): TomlTable { + const value = root[key]; + return typeof value === "object" && value !== null && !Array.isArray(value) ? value : {}; +} + +function str(table: TomlTable, key: string): string | undefined { + const value = table[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function num(table: TomlTable, key: string): number | undefined { + const value = table[key]; + return typeof value === "number" ? value : undefined; +} + +/** Older tasks declare `memory = "2G"` instead of `memory_mb`. */ +function memoryMb(value: string | undefined): number | undefined { + const match = value?.match(/^(\d+(?:\.\d+)?)\s*([GgMm])/u); + if (!match) return undefined; + const amount = Number(match[1]); + return Math.round(match[2]!.toLowerCase() === "g" ? amount * 1024 : amount); +} + +function seconds(table: TomlTable, key: string): number { + const value = num(table, key); + return value === undefined ? DEFAULT_TIMEOUT_MS : Math.round(value * 1000); +} diff --git a/packages/eve-bench/src/core/toml.test.ts b/packages/eve-bench/src/core/toml.test.ts new file mode 100644 index 0000000000..a2d31ddc6b --- /dev/null +++ b/packages/eve-bench/src/core/toml.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { parseToml } from "./toml.ts"; + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] # trailing comment +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[metadata] +expert_time_estimate_min = 180.0 + +[verifier] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +test("parses the Terminal-Bench task.toml subset", () => { + const parsed = parseToml(TASK_TOML); + assert.equal(parsed.schema_version, "1.1"); + assert.deepEqual(parsed.artifacts, []); + const task = parsed.task as Record; + assert.equal(task.name, "terminal-bench/adaptive-rejection-sampler"); + assert.deepEqual(task.keywords, ["applied-statistics", "simulation"]); + assert.deepEqual(task.authors, [{ name: "jvpoulos", email: "poulos@berkeley.edu" }]); + const environment = parsed.environment as Record; + assert.equal(environment.docker_image, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(environment.cpus, 1); + assert.equal(environment.allow_internet, true); + assert.deepEqual(environment.env, {}); + assert.deepEqual((parsed.verifier as Record).timeout_sec, 900); +}); + +test("handles dotted tables, escapes, and nested arrays", () => { + const parsed = parseToml(` +[a.b] +text = "line\\nbreak \\"quoted\\" # not a comment" +nested = [[1, 2], ["x, y"], []] +literal = 'C:\\path' +`); + const table = (parsed.a as Record).b as Record; + assert.equal(table.text, 'line\nbreak "quoted" # not a comment'); + assert.deepEqual(table.nested, [[1, 2], ["x, y"], []]); + assert.equal(table.literal, "C:\\path"); +}); + +test("rejects unsupported syntax with the line number", () => { + assert.throws(() => parseToml(`ok = 1\nbad = { inline = true }`), /line 2: inline tables/u); + assert.throws(() => parseToml(`ok = 1\nbad = 2024-01-01`), /line 2: unsupported value/u); + assert.throws(() => parseToml(`no equals sign`), /line 1: expected key = value/u); +}); diff --git a/packages/eve-bench/src/core/toml.ts b/packages/eve-bench/src/core/toml.ts new file mode 100644 index 0000000000..a17f39f06d --- /dev/null +++ b/packages/eve-bench/src/core/toml.ts @@ -0,0 +1,159 @@ +/** + * Minimal TOML reader for Terminal-Bench `task.toml` files. Supports tables, + * array-of-tables headers, bare/quoted keys, strings, numbers, booleans, and + * arrays. Anything else fails with the offending line so a dataset change + * surfaces immediately instead of being silently misread. + */ +export type TomlValue = string | number | boolean | TomlValue[] | TomlTable; +export interface TomlTable { + [key: string]: TomlValue; +} + +export function parseToml(source: string): TomlTable { + const root: TomlTable = {}; + let current = root; + const lines = source.split(/\r?\n/u); + for (let index = 0; index < lines.length; index++) { + const line = stripComment(lines[index] ?? "").trim(); + if (line.length === 0) continue; + const lineNumber = index + 1; + if (line.startsWith("[[") && line.endsWith("]]")) { + const path = parseKeyPath(line.slice(2, -2), lineNumber); + const parent = descend(root, path.slice(0, -1)); + const last = path.at(-1)!; + const existing = parent[last]; + const list = Array.isArray(existing) ? existing : []; + parent[last] = list; + current = {}; + list.push(current); + continue; + } + if (line.startsWith("[") && line.endsWith("]")) { + current = descend(root, parseKeyPath(line.slice(1, -1), lineNumber)); + continue; + } + const eq = findUnquoted(line, "="); + if (eq === -1) throw new TomlError(`expected key = value`, lineNumber); + const path = parseKeyPath(line.slice(0, eq), lineNumber); + const raw = line.slice(eq + 1).trim(); + const target = descend(current, path.slice(0, -1)); + target[path.at(-1)!] = parseValue(raw, lineNumber); + } + return root; +} + +export class TomlError extends Error { + constructor(message: string, line: number) { + super(`task.toml line ${line}: ${message}`); + } +} + +function descend(table: TomlTable, path: readonly string[]): TomlTable { + let node = table; + for (const segment of path) { + const next = node[segment]; + if (next === undefined) { + const created: TomlTable = {}; + node[segment] = created; + node = created; + } else if (Array.isArray(next)) { + const last = next.at(-1); + if (!isTable(last)) throw new Error(`cannot descend into array at ${segment}`); + node = last; + } else if (isTable(next)) { + node = next; + } else { + throw new Error(`key ${segment} is not a table`); + } + } + return node; +} + +function isTable(value: unknown): value is TomlTable { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseKeyPath(text: string, line: number): string[] { + const parts = text + .split(".") + .map((part) => part.trim()) + .map((part) => (part.startsWith('"') ? parseString(part, line) : part)); + if (parts.some((part) => part.length === 0)) throw new TomlError(`invalid key ${text}`, line); + return parts; +} + +function parseValue(raw: string, line: number): TomlValue { + if (raw.length === 0) throw new TomlError("missing value", line); + if (raw.startsWith('"') || raw.startsWith("'")) return parseString(raw, line); + if (raw === "true") return true; + if (raw === "false") return false; + if (raw.startsWith("[")) return parseArray(raw, line); + if (raw.startsWith("{")) throw new TomlError("inline tables are not supported", line); + const number = Number(raw.replaceAll("_", "")); + if (Number.isFinite(number)) return number; + throw new TomlError(`unsupported value ${raw}`, line); +} + +function parseString(raw: string, line: number): string { + const quote = raw[0]; + if (quote !== '"' && quote !== "'") throw new TomlError(`expected string`, line); + if (raw.length < 2 || raw.at(-1) !== quote) throw new TomlError(`unterminated string`, line); + const body = raw.slice(1, -1); + if (quote === "'") return body; + return body.replaceAll(/\\(["\\nrt])/gu, (_, code: string) => + code === "n" ? "\n" : code === "r" ? "\r" : code === "t" ? "\t" : code, + ); +} + +function parseArray(raw: string, line: number): TomlValue[] { + if (!raw.endsWith("]")) throw new TomlError("unterminated array", line); + const inner = raw.slice(1, -1).trim(); + if (inner.length === 0) return []; + return splitTopLevel(inner, line).map((item) => parseValue(item.trim(), line)); +} + +function splitTopLevel(text: string, line: number): string[] { + const items: string[] = []; + let depth = 0; + let quote: string | null = null; + let start = 0; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === "[") depth++; + else if (char === "]") depth--; + else if (char === "," && depth === 0) { + items.push(text.slice(start, index)); + start = index + 1; + } + } + if (quote !== null || depth !== 0) throw new TomlError("unbalanced array", line); + const tail = text.slice(start).trim(); + if (tail.length > 0) items.push(tail); + return items; +} + +function findUnquoted(text: string, needle: string): number { + let quote: string | null = null; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === needle) return index; + } + return -1; +} + +function stripComment(line: string): string { + const hash = findUnquoted(line, "#"); + return hash === -1 ? line : line.slice(0, hash); +} diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts new file mode 100644 index 0000000000..372ad58969 --- /dev/null +++ b/packages/eve-bench/src/core/trial.ts @@ -0,0 +1,188 @@ +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Writable } from "node:stream"; + +import { buildImage, Container, containerArch, pullImage, type ProcessResult } from "./docker.ts"; +import type { Harness, HarnessBundle } from "./harness.ts"; +import type { StepResult, TrialResult, Usage } from "./result.ts"; +import type { Task } from "./task.ts"; + +export interface TrialInput { + readonly task: Task; + readonly attempt: number; + readonly harness: Harness; + readonly bundle: HarnessBundle; + readonly model: string; + /** Host env forwarded to the harness exec only (provider credentials). */ + readonly forwardEnv: Readonly>; + readonly dir: string; + readonly signal: AbortSignal; +} + +const INSTALL_DIR = "/installed-agent"; +const LOGS_DIR = "/logs"; +const TESTS_DIR = "/tests"; + +export async function runTrial(input: TrialInput): Promise { + const { task, signal } = input; + const startedAt = new Date().toISOString(); + await mkdir(input.dir, { recursive: true }); + const containerLog = createWriteStream(join(input.dir, "container.log"), { flags: "a" }); + let container: Container | undefined; + let agent: StepResult = skipped(); + let verifier: StepResult = skipped(); + let reward: number | null = null; + let usage: Usage | undefined; + let error: string | undefined; + try { + const image = await resolveImage(task, containerLog, signal); + container = await Container.start( + { + image, + name: `eve-bench-${task.name}-${input.attempt}-${Date.now().toString(36)}`, + cpus: task.environment.cpus, + memoryMb: task.environment.memoryMb, + network: task.environment.allowInternet, + }, + signal, + ); + const workdir = await resolveWorkdir(container); + const bundleArch = input.bundle.arch; + const arch = bundleArch ? await containerArch(container) : undefined; + await container.exec( + `mkdir -p ${INSTALL_DIR} ${LOGS_DIR}/agent ${LOGS_DIR}/verifier ${TESTS_DIR}${arch ? ` ${INSTALL_DIR}/arch` : ""}`, + { timeoutMs: 30_000, signal }, + ); + await container.upload(`${input.bundle.dir}/.`, INSTALL_DIR, signal); + if (arch && bundleArch) { + await container.upload(`${bundleArch[arch]}/.`, `${INSTALL_DIR}/arch`, signal); + } + for (const path of input.harness.stage?.({ taskDir: task.dir }) ?? []) { + await container.upload(path, INSTALL_DIR, signal); + } + const instructionPath = join(input.dir, "instruction.md"); + await writeFile(instructionPath, task.instruction); + await container.upload(instructionPath, `${INSTALL_DIR}/instruction.md`, signal); + + const runContext = { + model: input.model, + installDir: INSTALL_DIR, + instructionPath: `${INSTALL_DIR}/instruction.md`, + taskWorkdir: workdir, + logsDir: `${LOGS_DIR}/agent`, + }; + agent = await step(() => + container!.exec(input.harness.command(runContext), { + cwd: workdir, + env: { ...input.forwardEnv, ...input.harness.env(runContext) }, + timeoutMs: task.agentTimeoutMs, + signal, + log: createWriteStream(join(input.dir, "agent.log")), + }), + ); + + await container.upload(`${join(task.dir, "tests")}/.`, TESTS_DIR, signal); + verifier = await step(() => + container!.exec(`bash ${TESTS_DIR}/test.sh`, { + cwd: workdir, + timeoutMs: task.verifierTimeoutMs, + signal, + log: createWriteStream(join(input.dir, "verifier.log")), + }), + ); + + for (const logs of ["agent", "verifier"]) { + if (!(await container.download(`${LOGS_DIR}/${logs}`, input.dir))) { + containerLog.write(`[eve-bench] could not download ${LOGS_DIR}/${logs}\n`); + } + } + reward = await readReward(join(input.dir, "verifier", "reward.txt")); + usage = await readUsage(join(input.dir, "agent", "result.json")); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } finally { + if (container) await container.remove(); + containerLog.end(); + } + const result: TrialResult = { + task: task.name, + attempt: input.attempt, + harness: input.harness.name, + model: input.model, + reward, + agent, + verifier, + ...(usage ? { usage } : {}), + ...(error ? { error } : {}), + startedAt, + finishedAt: new Date().toISOString(), + }; + await writeFile(join(input.dir, "trial.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +async function resolveImage(task: Task, log: Writable, signal: AbortSignal): Promise { + const options = { timeoutMs: task.buildTimeoutMs, signal, log }; + if (task.environment.dockerImage) { + await pullImage(task.environment.dockerImage, options); + return task.environment.dockerImage; + } + const dockerfile = await readFile(join(task.environment.dockerfileDir, "Dockerfile")); + const tag = `eve-bench/${task.name}:${createHash("sha256").update(dockerfile).digest("hex").slice(0, 12)}`; + await buildImage(tag, task.environment.dockerfileDir, options); + return tag; +} + +async function resolveWorkdir(container: Container): Promise { + const result = await container.exec("pwd", { timeoutMs: 10_000 }); + const workdir = result.stdout.trim(); + if (result.exitCode !== 0 || workdir.length === 0) { + throw new Error(`Could not resolve the task working directory: ${result.stderr.trim()}`); + } + return workdir; +} + +async function step(run: () => Promise): Promise { + const started = Date.now(); + const result = await run(); + return { + status: result.timedOut + ? "timeout" + : result.aborted || result.exitCode !== 0 + ? "failed" + : "completed", + exitCode: result.exitCode, + durationMs: Date.now() - started, + }; +} + +function skipped(): StepResult { + return { status: "skipped", exitCode: null, durationMs: 0 }; +} + +async function readReward(path: string): Promise { + try { + const value = Number((await readFile(path, "utf8")).trim()); + return Number.isFinite(value) ? value : null; + } catch { + return null; + } +} + +async function readUsage(path: string): Promise { + try { + const parsed = JSON.parse(await readFile(path, "utf8")) as { usage?: Partial }; + const usage = parsed.usage; + if (!usage) return undefined; + return { + inputTokens: usage.inputTokens ?? 0, + outputTokens: usage.outputTokens ?? 0, + cachedTokens: usage.cachedTokens ?? 0, + costUsd: usage.costUsd ?? 0, + }; + } catch { + return undefined; + } +} diff --git a/packages/eve-bench/src/harnesses/eve/index.ts b/packages/eve-bench/src/harnesses/eve/index.ts new file mode 100644 index 0000000000..55dcc4dabf --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/index.ts @@ -0,0 +1,247 @@ +import { execFile } from "node:child_process"; +import { createHash, type Hash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { chmod, cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { + Harness, + HarnessBundle, + HarnessPrepareContext, + HarnessRunContext, +} from "../../core/harness.ts"; + +const execFileAsync = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const packageRoot = resolve(here, "../../.."); +const repoRoot = resolve(packageRoot, "../.."); +const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; + +export type EveSource = + | { readonly kind: "local" } + | { readonly kind: "release"; readonly version: string }; + +/** + * Builds the `agent/` app against a local or released eve, and packages the + * Nitro output and runner with architecture-specific Linux Node binaries. + * Cached by content hash. + */ +export function createEveHarness(source: EveSource): Harness { + return { + name: source.kind === "local" ? "eve@local" : `eve@${source.version}`, + prepare: (ctx) => prepare(source, ctx), + command: (ctx) => `sh ${ctx.installDir}/run.sh`, + env: (ctx: HarnessRunContext) => ({ + EVE_PROJECT_DIR: `${ctx.installDir}/app`, + EVE_INSTRUCTION_PATH: ctx.instructionPath, + EVE_AGENT_LOGS_DIR: ctx.logsDir, + EVE_BENCH_MODEL: ctx.model, + EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, + }), + }; +} + +async function prepare(source: EveSource, ctx: HarnessPrepareContext): Promise { + const eve: ResolvedEve = + source.kind === "local" + ? await resolveLocalEve() + : { + dependency: source.version, + identity: source.version, + provenance: { kind: "release", version: source.version }, + }; + + const hash = createHash("sha256"); + hash.update(eve.identity).update(ctx.model).update(NODE_VERSION); + for (const file of await listFiles(join(packageRoot, "agent"))) + hash.update(file).update(await readFile(join(packageRoot, "agent", file))); + hash.update(await readFile(join(here, "runner.mjs"))).update(RUN_SH); + const key = hash.digest("hex").slice(0, 16); + const bundleDir = join(ctx.cacheDir, "bundles", key); + const provenance = { key, model: ctx.model, eve: eve.provenance }; + const nodeDir = join(ctx.cacheDir, `node-v${NODE_VERSION}`); + await Promise.all(ARCHES.map((arch) => ensureNode(nodeDir, arch))); + const arch = Object.fromEntries(ARCHES.map((name) => [name, join(nodeDir, name)])) as Record< + (typeof ARCHES)[number], + string + >; + if (await exists(join(bundleDir, "run.sh"))) return { dir: bundleDir, arch, provenance }; + + const building = `${bundleDir}.building`; + await rm(building, { recursive: true, force: true }); + const app = join(building, "app"); + await mkdir(app, { recursive: true }); + await cp(join(packageRoot, "agent"), join(app, "agent"), { recursive: true }); + await writeFile( + join(app, "package.json"), + `${JSON.stringify({ name: "eve-bench-app", private: true, type: "module", dependencies: { eve: eve.dependency, zod: "4.4.3" } }, null, 2)}\n`, + ); + await writeFile(join(app, "tsconfig.json"), APP_TSCONFIG); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + if (source.kind === "local") { + const artifacts = join(ctx.cacheDir, "artifacts"); + await mkdir(artifacts, { recursive: true }); + await cp(await packLocalEve(artifacts), join(app, "eve.tgz")); + } + const npm = (...args: string[]) => + execFileAsync("npm", [...args, "--ignore-scripts", "--userconfig", join(app, ".npmrc")], { + cwd: app, + maxBuffer: 64 * 1024 * 1024, + }); + await npm("install", "--package-lock-only"); + await npm("ci"); + await execFileAsync(join(app, "node_modules", ".bin", "eve"), ["build"], { + cwd: app, + env: { ...process.env, EVE_BENCH_MODEL: ctx.model, EVE_BENCH_TASK_WORKDIR: "/workspace" }, + maxBuffer: 64 * 1024 * 1024, + }); + await rm(join(app, "agent"), { recursive: true, force: true }); + await rm(join(app, "package-lock.json"), { force: true }); + await rm(join(app, ".npmrc"), { force: true }); + await rm(join(app, "eve.tgz"), { force: true }); + await rm(join(app, "tsconfig.json"), { force: true }); + await rm(join(app, ".eve"), { recursive: true, force: true }); + // Nitro's `.output` is self-contained and the runner speaks plain HTTP. + await rm(join(app, "node_modules"), { recursive: true, force: true }); + await cp(join(here, "runner.mjs"), join(app, "runner.mjs")); + await writeFile(join(building, "run.sh"), RUN_SH, { mode: 0o755 }); + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await rm(bundleDir, { recursive: true, force: true }); + await rename(building, bundleDir); + return { dir: bundleDir, arch, provenance }; +} + +interface ResolvedEve { + readonly dependency: string; + readonly identity: string; + readonly provenance: Record; +} + +async function resolveLocalEve(): Promise { + const evePackage = join(repoRoot, "packages/eve"); + const dist = join(evePackage, "dist"); + if (!(await isDirectory(dist))) { + throw new Error( + "Local eve dist is missing. Run `pnpm build` or `pnpm --filter eve build` first.", + ); + } + + const hash = createHash("sha256"); + hash.update("package.json\0").update(await readFile(join(evePackage, "package.json"))); + for (const file of await listFiles(dist)) { + hash.update("\0").update(file).update("\0"); + await hashFile(hash, join(dist, file)); + } + const distSha256 = hash.digest("hex"); + const [gitSha, status] = await Promise.all([ + execFileAsync("git", ["rev-parse", "HEAD"], { cwd: repoRoot }), + execFileAsync("git", ["status", "--porcelain", "packages/eve"], { cwd: repoRoot }), + ]); + return { + dependency: "file:eve.tgz", + identity: distSha256, + provenance: { + kind: "local", + gitSha: gitSha.stdout.trim(), + dirty: status.stdout.trim().length > 0, + distSha256, + }, + }; +} + +async function packLocalEve(artifacts: string): Promise { + const { stdout } = await execFileAsync( + "pnpm", + ["--filter", "eve", "pack", "--pack-destination", artifacts, "--json"], + { cwd: repoRoot, maxBuffer: 16 * 1024 * 1024 }, + ); + const jsonStart = stdout.indexOf("{"); + const packed = JSON.parse(stdout.slice(jsonStart)) as { filename: string }; + return resolve(artifacts, packed.filename); +} + +async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { + const archDir = join(nodeDir, arch); + const target = join(archDir, "node"); + if (await exists(target)) return; + await mkdir(archDir, { recursive: true }); + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const url = `https://nodejs.org/dist/v${NODE_VERSION}/${name}.tar.xz`; + const response = await fetch(url); + if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status}`); + const archive = join(nodeDir, `${name}.tar.xz`); + await writeFile(archive, Buffer.from(await response.arrayBuffer())); + const extractDir = join(nodeDir, `${name}.extract`); + await mkdir(extractDir, { recursive: true }); + await execFileAsync("tar", [ + "-xJf", + archive, + "--strip-components=2", + "-C", + extractDir, + `${name}/bin/node`, + ]); + await rename(join(extractDir, "node"), target); + await chmod(target, 0o755); + await rm(extractDir, { recursive: true, force: true }); + await rm(archive, { force: true }); +} + +async function listFiles(dir: string, prefix = ""): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) files.push(...(await listFiles(join(dir, entry.name), rel))); + else files.push(rel); + } + return files; +} + +async function hashFile(hash: Hash, path: string): Promise { + for await (const chunk of createReadStream(path)) hash.update(chunk); +} + +async function isDirectory(path: string): Promise { + return stat(path).then( + (entry) => entry.isDirectory(), + () => false, + ); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} + +const RUN_SH = `#!/bin/sh +set -eu +dir=$(cd "$(dirname "$0")" && pwd) +exec "$dir/arch/node" "$dir/app/runner.mjs" +`; + +const APP_TSCONFIG = `${JSON.stringify( + { + compilerOptions: { + target: "ES2024", + lib: ["ES2024"], + module: "NodeNext", + moduleResolution: "NodeNext", + moduleDetection: "force", + verbatimModuleSyntax: true, + noEmit: true, + strict: true, + isolatedModules: true, + skipLibCheck: true, + types: ["node"], + }, + include: ["agent/**/*.ts"], + }, + null, + 2, +)}\n`; diff --git a/packages/eve-bench/src/harnesses/eve/runner.mjs b/packages/eve-bench/src/harnesses/eve/runner.mjs new file mode 100644 index 0000000000..d2e593afca --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/runner.mjs @@ -0,0 +1,242 @@ +// Runs inside the task container: boots the built eve server, sends the task +// instruction as one session, follows the event stream to its boundary, and +// writes /logs/agent/{events.ndjson,result.json,server.log}. Talks to the +// server over plain HTTP so the bundle ships no node_modules. +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const projectDir = requireEnv("EVE_PROJECT_DIR"); +const instructionPath = requireEnv("EVE_INSTRUCTION_PATH"); +const logsDir = process.env.EVE_AGENT_LOGS_DIR ?? "/logs/agent"; +const host = process.env.EVE_HOST ?? "127.0.0.1"; +const port = Number(process.env.EVE_PORT ?? "3024"); +const serverUrl = `http://${host}:${port}`; +const startupTimeoutMs = Number(process.env.EVE_STARTUP_TIMEOUT_MS ?? "60000"); + +await mkdir(logsDir, { recursive: true }); +const eventsPath = join(logsDir, "events.ndjson"); +const resultPath = join(logsDir, "result.json"); +const serverLogPath = join(logsDir, "server.log"); + +let server; +let wroteResult = false; +try { + server = startServer(); + await waitForHealth(server); + + const instruction = await readFile(instructionPath, "utf8"); + const sessionId = await createSession(instruction); + const events = await followStream(sessionId); + + await writeFile(eventsPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); + const status = statusFromEvents(events); + const summary = { + status, + sessionId, + message: finalMessage(events), + inputRequests: events + .filter((event) => event.type === "input.requested") + .flatMap((event) => event.data?.requests ?? []), + usage: summarizeUsage(events), + eventCount: events.length, + failure: failureFromEvents(events) ?? failureFromStatus(status), + }; + await writeFile(resultPath, `${JSON.stringify(summary, null, 2)}\n`); + wroteResult = true; + if (summary.failure) throw new Error(summary.failure.message); +} catch (error) { + if (!wroteResult) { + await writeFile( + resultPath, + `${JSON.stringify( + { + status: "failed", + inputRequests: [], + usage: {}, + eventCount: 0, + failure: { type: "runner.error", message: error?.message ?? String(error) }, + }, + null, + 2, + )}\n`, + ); + } + throw error; +} finally { + if (server) await stopServer(server); +} + +function requireEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} must be set`); + return value; +} + +function startServer() { + const log = createWriteStream(serverLogPath, { flags: "a" }); + const entry = join(projectDir, ".output", "server", "index.mjs"); + const child = spawn(process.execPath, [entry], { + cwd: projectDir, + env: { + ...process.env, + EVE_DEV: "1", + HOST: host, + NITRO_HOST: host, + NITRO_PORT: String(port), + PORT: String(port), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + child.once("exit", (code, signal) => { + log.write(`\n[eve-server exited code=${code ?? ""} signal=${signal ?? ""}]\n`); + }); + return child; +} + +async function waitForHealth(child) { + const deadline = Date.now() + startupTimeoutMs; + let lastError; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`eve server exited before becoming healthy (code ${child.exitCode})`); + } + try { + const response = await fetch(`${serverUrl}/eve/v1/health`); + if (response.ok) return; + lastError = new Error(`health returned ${response.status}`); + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for eve health: ${lastError?.message ?? "unknown"}`); +} + +async function createSession(message) { + const response = await fetch(`${serverUrl}/eve/v1/session`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message }), + }); + if (!response.ok) + throw new Error(`session create failed: ${response.status} ${await response.text()}`); + const payload = await response.json(); + const sessionId = payload?.sessionId ?? response.headers.get("x-eve-session-id"); + if (!sessionId) throw new Error("session create returned no session id"); + return sessionId; +} + +/** Reads the durable NDJSON stream until a session boundary, reconnecting from the cursor if the transport ends first. */ +async function followStream(sessionId) { + const events = []; + for (;;) { + const url = new URL(`${serverUrl}/eve/v1/session/${encodeURIComponent(sessionId)}/stream`); + if (events.length > 0) url.searchParams.set("startIndex", String(events.length)); + const response = await fetch(url); + if (!response.ok || !response.body) + throw new Error(`stream failed: ${response.status} ${await response.text()}`); + for await (const event of ndjson(response.body)) { + events.push(event); + if (isBoundary(event)) return events; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } +} + +async function* ndjson(body) { + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of body) { + buffer += decoder.decode(chunk, { stream: true }); + let newline; + while ((newline = buffer.indexOf("\n")) !== -1) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) yield JSON.parse(line); + } + } + if (buffer.trim()) yield JSON.parse(buffer); +} + +function isBoundary(event) { + return ( + event.type === "session.completed" || + event.type === "session.failed" || + event.type === "session.waiting" + ); +} + +function statusFromEvents(events) { + const boundary = events.findLast(isBoundary); + if (boundary?.type === "session.completed") return "completed"; + if (boundary?.type === "session.waiting") return "waiting"; + if (boundary?.type === "session.failed") return "failed"; + return "unknown"; +} + +function finalMessage(events) { + return events.findLast( + (event) => event.type === "message.completed" && event.data?.finishReason !== "tool-calls", + )?.data?.message; +} + +function failureFromEvents(events) { + for (const event of events) { + if (event.type === "input.requested") { + return { + type: event.type, + message: "eve requested human input during a non-interactive benchmark run", + data: event.data, + }; + } + if (event.type === "turn.failed" || event.type === "session.failed") { + const message = + event.data?.message ?? event.data?.error?.message ?? `${event.type} emitted during eve run`; + return { type: event.type, message, data: event.data }; + } + } + return null; +} + +function failureFromStatus(status) { + if (status === "waiting" || status === "completed") return null; + return { type: "session.unsettled", message: `eve run ended with status: ${status}` }; +} + +function summarizeUsage(events) { + const usage = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: 0 }; + for (const event of events) { + if (event.type !== "step.completed") continue; + const source = event.data?.usage ?? event.data ?? {}; + usage.inputTokens += numberValue(source.inputTokens, source.input_tokens); + usage.outputTokens += numberValue(source.outputTokens, source.output_tokens); + usage.cachedTokens += numberValue( + source.cacheReadTokens, + source.cachedTokens, + source.cached_tokens, + ); + usage.costUsd += numberValue(source.costUsd, source.cost_usd); + } + return usage; +} + +function numberValue(...values) { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return 0; +} + +async function stopServer(child) { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", resolve)), + new Promise((resolve) => setTimeout(resolve, 5000)), + ]); + if (child.exitCode === null) child.kill("SIGKILL"); +} diff --git a/packages/eve-bench/src/harnesses/oracle.ts b/packages/eve-bench/src/harnesses/oracle.ts new file mode 100644 index 0000000000..1389f91a22 --- /dev/null +++ b/packages/eve-bench/src/harnesses/oracle.ts @@ -0,0 +1,23 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import type { Harness } from "../core/harness.ts"; + +/** + * Runs each task's reference `solution/solve.sh`. Validates the runner and + * verifier end to end without a model, and gives the reward ceiling a real + * harness is measured against. + */ +export function createOracleHarness(): Harness { + return { + name: "oracle", + async prepare(ctx) { + const dir = join(ctx.cacheDir, "bundles", "oracle"); + await mkdir(dir, { recursive: true }); + return { dir }; + }, + stage: (ctx) => [join(ctx.taskDir, "solution")], + command: (ctx) => `bash ${ctx.installDir}/solution/solve.sh`, + env: () => ({}), + }; +} diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts new file mode 100644 index 0000000000..7f028379f8 --- /dev/null +++ b/packages/eve-bench/src/options.ts @@ -0,0 +1,98 @@ +import { readdir } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts"; +import type { Harness } from "./core/harness.ts"; +import { createEveHarness } from "./harnesses/eve/index.ts"; +import { createOracleHarness } from "./harnesses/oracle.ts"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +export const paths = { + packageRoot, + datasetsRoot: join(packageRoot, "datasets"), + generatedRoot: join(packageRoot, ".generated"), +} as const; + +export interface TaskSelection { + readonly lock: DatasetLock; + readonly tasks: readonly string[]; +} + +export async function selectTasks(input: { + cohort?: string; + dataset?: string; + task?: readonly string[]; +}): Promise { + if (input.cohort) { + const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); + const entry = await lock(cohort.dataset); + const tasks = input.task?.length + ? cohort.tasks.filter((task) => input.task!.includes(task)) + : cohort.tasks; + return { lock: entry, tasks }; + } + + const entry = await lock(input.dataset ?? (await defaultDataset())); + const tasks = input.task?.length + ? entry.tasks.filter((task) => input.task!.includes(task)) + : entry.tasks; + if (input.task?.length && tasks.length !== input.task.length) { + const missing = input.task.filter((task) => !tasks.includes(task)); + throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); + } + return { lock: entry, tasks }; +} + +export function jobDir(nameOrPath: string): string { + return nameOrPath.includes("/") + ? resolve(nameOrPath) + : join(paths.generatedRoot, "jobs", nameOrPath); +} + +export async function lock(name: string): Promise { + return readDatasetLock(join(paths.datasetsRoot, `${name}.json`)); +} + +export async function allLocks(): Promise { + const files = (await readdir(paths.datasetsRoot)).filter((file) => file.endsWith(".json")); + return Promise.all(files.map((file) => readDatasetLock(join(paths.datasetsRoot, file)))); +} + +export function defaultJobName(label: string): string { + const stamp = new Date().toISOString().replaceAll(/[:.]/gu, "-").slice(0, 19); + return `${stamp}-${label.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}`; +} + +export function selectHarness(name: string, eve: string): Harness { + if (name === "oracle") return createOracleHarness(); + if (name === "eve") { + return createEveHarness( + eve === "local" ? { kind: "local" } : { kind: "release", version: eve }, + ); + } + throw new Error(`unknown harness: ${name}`); +} + +export function forwardedEnv(): Record { + const env: Record = {}; + for (const key of [ + "AI_GATEWAY_API_KEY", + "VERCEL_OIDC_TOKEN", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + ]) { + const value = process.env[key]; + if (value) env[key] = value; + } + return env; +} + +async function defaultDataset(): Promise { + const locks = await allLocks(); + if (locks.length !== 1) { + throw new Error("pass --dataset or --cohort; multiple datasets are available"); + } + return `${locks[0]!.name}-${locks[0]!.version}`; +} diff --git a/packages/eve-bench/src/tools/diff_bench.ts b/packages/eve-bench/src/tools/diff_bench.ts new file mode 100644 index 0000000000..ff59f82988 --- /dev/null +++ b/packages/eve-bench/src/tools/diff_bench.ts @@ -0,0 +1,25 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { diffJobs } from "../core/report.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ + base: z.string().min(1), + candidate: z.string().min(1), +}); + +export default defineTool({ + description: "Compare task rewards between two eve benchmark jobs.", + inputSchema, + approval: never(), + async execute(input) { + const [base, candidate] = await Promise.all([ + readJobResult(jobDir(input.base)), + readJobResult(jobDir(input.candidate)), + ]); + return diffJobs(base, candidate); + }, +}); diff --git a/packages/eve-bench/src/tools/get_bench_result.ts b/packages/eve-bench/src/tools/get_bench_result.ts new file mode 100644 index 0000000000..a3ff347ffc --- /dev/null +++ b/packages/eve-bench/src/tools/get_bench_result.ts @@ -0,0 +1,24 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ job: z.string().min(1) }); + +export default defineTool({ + description: "Read a completed eve benchmark job result.", + inputSchema, + approval: never(), + async execute(input) { + try { + return await readJobResult(jobDir(input.job)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`benchmark result not found for job "${input.job}"`); + } + throw error; + } + }, +}); diff --git a/packages/eve-bench/src/tools/list_bench_tasks.ts b/packages/eve-bench/src/tools/list_bench_tasks.ts new file mode 100644 index 0000000000..ad2460a546 --- /dev/null +++ b/packages/eve-bench/src/tools/list_bench_tasks.ts @@ -0,0 +1,19 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { selectTasks } from "../options.ts"; + +export const inputSchema = z.object({ + cohort: z.string().min(1).optional(), + dataset: z.string().min(1).optional(), +}); + +export default defineTool({ + description: "List task names in an eve benchmark dataset or cohort.", + inputSchema, + approval: never(), + async execute(input) { + return (await selectTasks(input)).tasks; + }, +}); diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts new file mode 100644 index 0000000000..09f410af5f --- /dev/null +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { syncDataset, taskDirs } from "../core/dataset.ts"; +import { runJob } from "../core/job.ts"; +import { loadTask } from "../core/task.ts"; +import { defaultJobName, forwardedEnv, paths, selectHarness, selectTasks } from "../options.ts"; + +export const inputSchema = z + .object({ + cohort: z.string().min(1).optional(), + tasks: z.array(z.string().min(1)).min(1).optional(), + model: z.string().min(1).optional(), + harness: z.enum(["eve", "oracle"]).default("eve"), + eve: z.string().min(1).default("local"), + attempts: z.number().int().positive().default(1), + concurrency: z.number().int().positive().default(4), + job: z.string().min(1).optional(), + }) + .superRefine((input, ctx) => { + if (input.harness !== "oracle" && !input.model) { + ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); + } + }); + +export default defineTool({ + description: "Run an eve benchmark job and return its result.", + inputSchema, + async execute(input, ctx) { + const harness = selectHarness(input.harness, input.eve); + const model = input.model ?? "none"; + const selection = await selectTasks({ cohort: input.cohort, task: input.tasks }); + const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const tasks = await Promise.all( + (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + ); + const name = input.job ?? defaultJobName(`${harness.name}-${model}`); + + return runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks, + harness, + model, + attempts: input.attempts, + concurrency: input.concurrency, + forwardEnv: forwardedEnv(), + signal: ctx.abortSignal, + }); + }, +}); diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts new file mode 100644 index 0000000000..41515610e8 --- /dev/null +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { inputSchema as diffBenchInput } from "./diff_bench.ts"; +import { inputSchema as getBenchResultInput } from "./get_bench_result.ts"; +import { inputSchema as listBenchTasksInput } from "./list_bench_tasks.ts"; +import { inputSchema as runBenchInput } from "./run_bench.ts"; + +describe("run_bench input", () => { + it("accepts valid eve and oracle inputs", () => { + assert.equal( + runBenchInput.safeParse({ + cohort: "smoke", + tasks: ["fix-git"], + model: "zai/glm-5.2", + harness: "eve", + eve: "local", + attempts: 2, + concurrency: 1, + job: "t1", + }).success, + true, + ); + assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); + }); + + it("rejects invalid inputs", () => { + assert.equal(runBenchInput.safeParse({}).success, false); + assert.equal(runBenchInput.safeParse({ harness: "other", model: "model" }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", attempts: 0 }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", tasks: "fix-git" }).success, false); + }); +}); + +describe("get_bench_result input", () => { + it("accepts a job name", () => { + assert.equal(getBenchResultInput.safeParse({ job: "smoke-run" }).success, true); + }); + + it("rejects a missing or empty job name", () => { + assert.equal(getBenchResultInput.safeParse({}).success, false); + assert.equal(getBenchResultInput.safeParse({ job: "" }).success, false); + }); +}); + +describe("diff_bench input", () => { + it("accepts base and candidate jobs", () => { + assert.equal( + diffBenchInput.safeParse({ base: "base-run", candidate: "candidate-run" }).success, + true, + ); + }); + + it("rejects incomplete input", () => { + assert.equal(diffBenchInput.safeParse({ base: "base-run" }).success, false); + assert.equal(diffBenchInput.safeParse({ base: 1, candidate: "candidate-run" }).success, false); + }); +}); + +describe("list_bench_tasks input", () => { + it("accepts empty, cohort, and dataset selections", () => { + assert.equal(listBenchTasksInput.safeParse({}).success, true); + assert.equal(listBenchTasksInput.safeParse({ cohort: "smoke" }).success, true); + assert.equal(listBenchTasksInput.safeParse({ dataset: "terminal-bench-2.0" }).success, true); + }); + + it("rejects invalid selections", () => { + assert.equal(listBenchTasksInput.safeParse({ cohort: "" }).success, false); + assert.equal(listBenchTasksInput.safeParse({ dataset: ["terminal-bench-2.0"] }).success, false); + }); +}); diff --git a/packages/eve-bench/tsconfig.json b/packages/eve-bench/tsconfig.json new file mode 100644 index 0000000000..807ccbdd2d --- /dev/null +++ b/packages/eve-bench/tsconfig.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": ["ES2024"], + "rootDir": ".", + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": false + }, + "include": ["agent/**/*.ts", "src/**/*.ts"], + "exclude": [] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61cc596628..804d5fd3c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1260,6 +1260,22 @@ importers: specifier: 5.0.0 version: 5.0.0(zod@4.4.3) + packages/eve-bench: + dependencies: + eve: + specifier: workspace:* + version: link:../eve + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + typescript: + specifier: 'catalog:' + version: 7.0.2 + packages/eve-buzz-acp-adapter: dependencies: eve: diff --git a/research/eve-bench-runner.md b/research/eve-bench-runner.md new file mode 100644 index 0000000000..e290609aae --- /dev/null +++ b/research/eve-bench-runner.md @@ -0,0 +1,238 @@ +--- +issue: TODO +status: approved +last_updated: "2026-09-09" +--- + +# eve-bench: a zero-dependency benchmark runner + +`packages/eve-bench` currently benchmarks the eve harness on Terminal-Bench +through Harbor: a Python adapter subclasses Harbor's private `Eve` agent, a +Node runner drives the built app, and a materialization script produces a +self-contained app directory (two Node binaries, a rolldown bundle, `npm ci`) +because Harbor copies the agent directory into every task container. + +That stack needs three runtimes (Python, Harbor at a pinned revision, Node), +leaks Harbor-specific env (`HARBOR_MODEL`, `EVE_HARBOR_TASK_WORKDIR`) into the +harness under test, breaks on any upstream refactor of Harbor's private +methods, and has no path to a CLI, an agent tool, or a GitHub Action that +does not drag Python along. + +Per trial, Harbor does five things: resolve a task directory, run a Docker +container, exec the agent under a timeout, exec `tests/test.sh` under a +timeout and read `/logs/verifier/reward.txt`, and write a result record. A +job is that loop over tasks × attempts with concurrency and resume. All of it +fits in a small amount of Node with no third-party dependencies. + +## Proposal + +Replace Harbor with an eve-owned runner. Keep the Terminal-Bench task format +and reward contract unchanged so datasets stay interchangeable and results +remain comparable with published Harbor runs. + +``` +packages/eve-bench/ + src/core/ zero-dependency library; does not import eve + task.ts load task.toml + instruction.md + docker.ts container lifecycle over the docker CLI + trial.ts run one attempt: start → agent → verify → record + job.ts matrix, concurrency, attempts, resume + result.ts trial/job JSON schema + report.ts console, json, junit, diff + src/harnesses/eve/ the eve harness: build once, exec in container + src/harnesses/ oracle (reference solutions) and future harnesses + src/cli.ts eve-bench run | tasks | report | diff + src/tools/ eve tools wrapping core for agent consumers (follow-up) + datasets/ pinned dataset locks and cohorts (JSON) + agent/ the harness under test (unchanged) +``` + +The runner is plain `.ts` executed by Node 24 with type stripping; the root +tsconfig already enforces `erasableSyntaxOnly`. No build step, no bundler. + +### Datasets + +A dataset lock pins a git remote, commit, and the task paths within it: + +```json +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": ["adaptive-rejection-sampler", "fix-git", "..."] +} +``` + +`eve-bench tasks sync ` clones the pinned commit (shallow, sparse) into +`.generated/datasets/-/`. Cohorts are named task subsets in +`datasets/cohorts/*.json` and reference a lock by name; membership changes +are reviewed like code. + +### Tasks + +A task is a Terminal-Bench directory: `task.toml`, `instruction.md`, +`environment/Dockerfile` (optionally with a prebuilt `docker_image`), +`tests/test.sh`, `solution/`. The loader reads only what the runner needs: + +- `task.name` +- `agent.timeout_sec`, `verifier.timeout_sec`, `environment.build_timeout_sec` +- `environment.docker_image`, `cpus`, `memory_mb`, `allow_internet` + +`task.toml` is parsed by a minimal TOML reader covering the subset the +dataset uses (tables, strings, numbers, booleans, arrays). Anything outside +that subset is a load error naming the line. + +### Environment + +One container per trial, always via the `docker` CLI. Prebuilt +`environment.docker_image` is preferred; otherwise the Dockerfile is built +once per task and cached by image tag. Every operation (`build`, `run`, +`exec`, `cp`, `rm`) accepts an `AbortSignal`; SIGINT to the runner aborts +in-flight trials and removes their containers before exiting. + +### Harness contract + +A harness is something that can be placed in a container and told to solve +an instruction. It is a spawned shell, nothing more: + +```ts +interface Harness { + readonly name: string; + /** Runs once per job on the host; returns a directory to place in the container. */ + prepare(ctx: { model: string; cacheDir: string }): Promise<{ dir: string; provenance?: object }>; + /** Optional per-task host paths uploaded into the install directory. */ + stage?(ctx: { taskDir: string }): readonly string[]; + /** Shell command executed inside the container in the task working directory. */ + command(ctx: { + installDir: string; + instructionPath: string; + taskWorkdir: string; + logsDir: string; + }): string; + /** Env visible to the command. Provider credentials are forwarded by the runner. */ + env(ctx: { model: string }): Record; +} +``` + +The bundle directory is uploaded to `/installed-agent`. The eve harness +bundles the Nitro output built from `agent/`, a runner, and Linux Node +binaries, cached by content hash of `agent/`, the eve source, and the model. +The runner drives the server over plain HTTP (`POST /eve/v1/session`, then +`GET /eve/v1/session/:id/stream`), so the bundle carries no `node_modules` +and no bundler is involved. It writes `/logs/agent/events.ndjson` and +`/logs/agent/result.json`. The env the harness reads is `EVE_BENCH_MODEL` and +`EVE_BENCH_TASK_WORKDIR`. + +An `oracle` harness runs each task's `solution/solve.sh` via `stage`. It +validates the runner without a model and is the reward ceiling a real harness +is measured against. A harness wrapping another CLI agent is the same +interface and enables side-by-side comparison. + +### Trial lifecycle + +``` +resolve task ─▶ start container ─▶ upload harness ─▶ exec harness (agent timeout) + │ + record ◀── read reward.txt ◀── exec tests/test.sh ◀─────┘ + (verifier timeout) +``` + +The verifier always runs, including after an agent timeout or failure, so a +trial's reward reflects the filesystem the agent left behind. Container +removal is unconditional. Each trial writes to its own directory: + +``` +.generated/jobs//// + trial.json the result record below + agent/ copied /logs/agent + verifier/ copied /logs/verifier + container.log docker output for the trial +``` + +Resume skips trial directories that already contain a `trial.json`. + +### Result schema + +```ts +interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; // null when the verifier produced none + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: { + inputTokens: number; + outputTokens: number; + cachedTokens: number; + costUsd: number; + }; + readonly error?: string; + readonly startedAt: string; + readonly finishedAt: string; +} + +interface StepResult { + readonly status: "completed" | "timeout" | "failed" | "skipped"; + readonly exitCode: number | null; + readonly durationMs: number; +} + +interface JobResult { + readonly job: string; + readonly dataset: { name: string; version: string; commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: { tasks: number; attempts: number; resolved: number; meanReward: number }; +} +``` + +This schema is the API. The CLI, the GitHub Action, and agent tools all read +and write it; nothing else is stable. + +### CLI + +``` +eve-bench tasks sync fetch a pinned dataset +eve-bench tasks list --cohort smoke print task names +eve-bench run --cohort smoke --model zai/glm-5.2 [--attempts 3] [--concurrency 8] [--task fix-git ...] + [--harness eve|oracle] [--eve local|] [--job ] # same --job resumes +eve-bench report [--format console|json|junit] +eve-bench diff per-task reward deltas +``` + +Every command accepts `--json` and never prompts. Provider credentials come +from the environment (`AI_GATEWAY_API_KEY`) and are forwarded into the +container only for the harness exec, never written to disk. + +### Consumers + +- CI: a workflow runs `eve-bench run --cohort smoke --json`, uploads the job + directory as an artifact, and posts `eve-bench diff` against the base + branch's artifact. +- Humans: the same CLI with the console reporter. +- Agents: `src/tools/` exposes `run_trial`, `get_result`, and `diff` as eve + tools over `core`; they return the schema above. + +## Invariants + +- `src/core` has no dependency on `eve` or any third-party package; only the + eve harness imports eve. +- Task format, verifier contract, and reward semantics are Terminal-Bench's; + eve-bench adds nothing to a task directory. +- A harness never sees runner internals beyond the contract above. +- Trial directories are append-only; resume never rewrites a `trial.json`. + +## Out of scope + +- Non-Docker environments (Vercel Sandbox, remote runners). The `docker.ts` + surface is small enough to add a second implementation later; it is not + abstracted ahead of need. +- Reusing eve's `SandboxBackend` for the environment. The benchmark must not + depend on the surface it measures. +- Uploading results to a hosted dashboard. +- `src/tools/` (eve tools over `core`) and the CI workflow. Both consume the + result schema above and follow once the CLI has run a full dataset. From 67d0a4c84e3996b624923eeb50004d797930c735 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Wed, 9 Sep 2026 14:26:36 -0400 Subject: [PATCH 2/3] feat(eve-bench): container hygiene, local task dirs, and stress scenarios Containers now carry eve-bench labels; a job removes leftovers from a crashed previous run of the same name, cleans up by label after an abort, checks the Docker daemon up front, and caps per-trial logs at 64 MiB. The CLI accepts repeatable --task-dir for local Terminal-Bench task directories, which powers a Docker-backed stress suite (pnpm run test:stress) covering timeouts, bad images, SIGINT abort and resume, and concurrency. The eve-smoke CI job can run pre-merge on PRs labeled bench:eve. Signed-off-by: Rui Conti --- .github/workflows/bench.yml | 12 +- packages/eve-bench/README.md | 39 ++- packages/eve-bench/package.json | 1 + packages/eve-bench/src/cli.ts | 18 +- packages/eve-bench/src/core/docker.test.ts | 35 +++ packages/eve-bench/src/core/docker.ts | 129 ++++++++- packages/eve-bench/src/core/job.ts | 17 +- packages/eve-bench/src/core/trial.ts | 57 ++-- packages/eve-bench/src/options.ts | 15 +- packages/eve-bench/test/stress/stress.test.ts | 266 ++++++++++++++++++ .../tasks/abort-a/environment/Dockerfile | 3 + .../test/stress/tasks/abort-a/instruction.md | 1 + .../stress/tasks/abort-a/solution/solve.sh | 2 + .../test/stress/tasks/abort-a/task.toml | 12 + .../test/stress/tasks/abort-a/tests/test.sh | 3 + .../tasks/abort-b/environment/Dockerfile | 3 + .../test/stress/tasks/abort-b/instruction.md | 1 + .../stress/tasks/abort-b/solution/solve.sh | 2 + .../test/stress/tasks/abort-b/task.toml | 12 + .../test/stress/tasks/abort-b/tests/test.sh | 3 + .../tasks/abort-resume/environment/Dockerfile | 3 + .../stress/tasks/abort-resume/instruction.md | 1 + .../tasks/abort-resume/solution/solve.sh | 2 + .../test/stress/tasks/abort-resume/task.toml | 12 + .../stress/tasks/abort-resume/tests/test.sh | 3 + .../agent-timeout/environment/Dockerfile | 3 + .../stress/tasks/agent-timeout/instruction.md | 1 + .../tasks/agent-timeout/solution/solve.sh | 2 + .../test/stress/tasks/agent-timeout/task.toml | 12 + .../stress/tasks/agent-timeout/tests/test.sh | 3 + .../tasks/bad-image/environment/Dockerfile | 3 + .../stress/tasks/bad-image/instruction.md | 1 + .../stress/tasks/bad-image/solution/solve.sh | 2 + .../test/stress/tasks/bad-image/task.toml | 13 + .../test/stress/tasks/bad-image/tests/test.sh | 3 + .../concurrency-1/environment/Dockerfile | 3 + .../stress/tasks/concurrency-1/instruction.md | 1 + .../tasks/concurrency-1/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-1/task.toml | 12 + .../stress/tasks/concurrency-1/tests/test.sh | 3 + .../concurrency-2/environment/Dockerfile | 3 + .../stress/tasks/concurrency-2/instruction.md | 1 + .../tasks/concurrency-2/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-2/task.toml | 12 + .../stress/tasks/concurrency-2/tests/test.sh | 3 + .../concurrency-3/environment/Dockerfile | 3 + .../stress/tasks/concurrency-3/instruction.md | 1 + .../tasks/concurrency-3/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-3/task.toml | 12 + .../stress/tasks/concurrency-3/tests/test.sh | 3 + .../concurrency-4/environment/Dockerfile | 3 + .../stress/tasks/concurrency-4/instruction.md | 1 + .../tasks/concurrency-4/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-4/task.toml | 12 + .../stress/tasks/concurrency-4/tests/test.sh | 3 + .../concurrency-5/environment/Dockerfile | 3 + .../stress/tasks/concurrency-5/instruction.md | 1 + .../tasks/concurrency-5/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-5/task.toml | 12 + .../stress/tasks/concurrency-5/tests/test.sh | 3 + .../concurrency-6/environment/Dockerfile | 3 + .../stress/tasks/concurrency-6/instruction.md | 1 + .../tasks/concurrency-6/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-6/task.toml | 12 + .../stress/tasks/concurrency-6/tests/test.sh | 3 + .../stress/tasks/happy/environment/Dockerfile | 3 + .../test/stress/tasks/happy/instruction.md | 1 + .../test/stress/tasks/happy/solution/solve.sh | 2 + .../test/stress/tasks/happy/task.toml | 12 + .../test/stress/tasks/happy/tests/test.sh | 3 + .../verifier-timeout/environment/Dockerfile | 3 + .../tasks/verifier-timeout/instruction.md | 1 + .../tasks/verifier-timeout/solution/solve.sh | 2 + .../stress/tasks/verifier-timeout/task.toml | 12 + .../tasks/verifier-timeout/tests/test.sh | 3 + 75 files changed, 827 insertions(+), 36 deletions(-) create mode 100644 packages/eve-bench/src/core/docker.test.ts create mode 100644 packages/eve-bench/test/stress/stress.test.ts create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/happy/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/happy/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/happy/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/happy/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a759cdf859..76f1780fb5 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,6 +2,7 @@ name: eve Benchmarks on: pull_request: + types: [opened, synchronize, reopened, labeled] paths: - "packages/eve-bench/**" - research/eve-bench-runner.md @@ -90,8 +91,9 @@ jobs: } ' + # AI_GATEWAY_API_KEY is available to pull requests from branches in this repository, not forks. eve-smoke: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bench:eve') }} runs-on: ubuntu-latest timeout-minutes: 90 env: @@ -125,19 +127,19 @@ jobs: - name: Run eve smoke benchmark env: - MODEL: ${{ inputs.model }} - COHORT: ${{ inputs.cohort }} + MODEL: ${{ inputs.model || 'zai/glm-5.2' }} + COHORT: ${{ inputs.cohort || 'smoke' }} run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke - name: Download base job artifact - if: ${{ inputs.base_job_artifact != '' }} + if: ${{ inputs.base_job_artifact }} uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ inputs.base_job_artifact }} path: packages/eve-bench/.generated/base-job - name: Compare with base job - if: ${{ inputs.base_job_artifact != '' }} + if: ${{ inputs.base_job_artifact }} run: | pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt { diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md index 149054b23b..b3a41bf612 100644 --- a/packages/eve-bench/README.md +++ b/packages/eve-bench/README.md @@ -27,6 +27,7 @@ pnpm bench tasks list --cohort smoke # print a cohort's task AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8 pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release +pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task pnpm bench report --format junit --out junit.xml pnpm bench diff @@ -37,6 +38,10 @@ Every command accepts `--json` and never prompts. Jobs live in skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight trials and removes their containers. +Pass `--task-dir ` more than once to run one or more local Terminal-Bench +task directories. A local-only run does not sync a dataset and records its +dataset as `local`. + ## How a trial runs 1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). @@ -47,6 +52,32 @@ trials and removes their containers. 4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` to the trial directory, remove the container, and write `trial.json`. +Each runner-owned log (`container.log`, `agent.log`, and `verifier.log`) is +capped at 64 MiB. The runner appends a truncation notice when a log exceeds the +cap. + +Containers carry the labels `eve-bench=1`, `eve-bench.job=`, and +`eve-bench.task=`. Find any leftovers with: + +```bash +docker ps -a --filter label=eve-bench=1 +``` + +At the start of a job, the runner removes stale containers with the same +`eve-bench.job` label before starting new trials. + +## Stress tests + +Run the Docker-backed stress suite separately from the default test suite: + +```bash +pnpm run test:stress +``` + +The suite requires Docker and takes about 30 seconds. It covers the happy path, +agent and verifier timeouts, a bad image, SIGINT abort and resume, and concurrent +trials. + ## Harnesses - `eve` (default): builds `agent/` against the local checkout (`--eve local`) @@ -89,9 +120,11 @@ attempt count, and eve source. touch this package or its design document and requires every task attempt to resolve. It uploads the job directory and `junit.xml` as artifacts. -The `eve-smoke` job runs only through `workflow_dispatch`, accepts `model` and -`cohort` inputs, and requires `AI_GATEWAY_API_KEY`. It uploads its job directory -and can optionally diff the result against a downloaded base-job artifact. +The real-model `eve-smoke` job runs through `workflow_dispatch` or when the +`bench:eve` label is added to a pull request. It accepts `model` and `cohort` +inputs for manual runs and requires `AI_GATEWAY_API_KEY`, which is unavailable +to pull requests from forks. It uploads its job directory and can optionally +diff the result against a downloaded base-job artifact. ## Layout diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index c06941682a..4ad272b0eb 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -10,6 +10,7 @@ "scripts": { "bench": "node src/cli.ts", "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "dependencies": { diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts index 4c63c3d549..95636fad45 100644 --- a/packages/eve-bench/src/cli.ts +++ b/packages/eve-bench/src/cli.ts @@ -22,8 +22,9 @@ const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets eve-bench tasks list [--cohort ] print task names - eve-bench run --model [--cohort ] [--task ...] [--attempts N] [--concurrency N] - [--harness eve|oracle] [--eve local|] [--job ] [--format console|json|junit] + eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] + [--concurrency N] [--harness eve|oracle] [--eve local|] [--job ] + [--format console|json|junit] eve-bench report [--format console|json|junit] eve-bench diff [--json] @@ -90,6 +91,7 @@ async function run(args: string[]): Promise { cohort: { type: "string" }, dataset: { type: "string" }, task: { type: "string", multiple: true }, + "task-dir": { type: "string", multiple: true }, attempts: { type: "string", default: "1" }, concurrency: { type: "string", default: "4" }, eve: { type: "string", default: "local" }, @@ -102,10 +104,15 @@ async function run(args: string[]): Promise { const harness = selectHarness(values.harness, values.eve); const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); if (!model) throw new Error("--model is required"); - const selection = await selectTasks(values); - const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const selection = await selectTasks({ ...values, taskDir: values["task-dir"] }); + const datasetTaskDirs = selection.tasks.length + ? await taskDirs( + await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock), + selection.tasks, + ) + : []; const loaded = await Promise.all( - (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + [...datasetTaskDirs, ...selection.taskDirs].map((dir) => loadTask(dir)), ); const name = values.job ?? defaultJobName(`${harness.name}-${model}`); const controller = new AbortController(); @@ -128,6 +135,7 @@ async function run(args: string[]): Promise { concurrency: Number(values.concurrency), forwardEnv: forwardedEnv(), signal: controller.signal, + onLog: (message) => process.stderr.write(`${message}\n`), onTrial: (trial, resumed) => { if (format !== "console") return; const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); diff --git a/packages/eve-bench/src/core/docker.test.ts b/packages/eve-bench/src/core/docker.test.ts new file mode 100644 index 0000000000..e5221dc165 --- /dev/null +++ b/packages/eve-bench/src/core/docker.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { PassThrough } from "node:stream"; +import { finished } from "node:stream/promises"; + +import { createBoundedWritable } from "./docker.ts"; + +const TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; + +test("bounded writable keeps the head and appends one truncation line", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.write("abc"); + log.write("def"); + log.write("ignored"); + log.end(); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), `abcde${TRUNCATED_LINE}`); +}); + +test("bounded writable does not mark a log within the limit", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.end("abcde"); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), "abcde"); +}); diff --git a/packages/eve-bench/src/core/docker.ts b/packages/eve-bench/src/core/docker.ts index 039ce37451..bb5937b0c2 100644 --- a/packages/eve-bench/src/core/docker.ts +++ b/packages/eve-bench/src/core/docker.ts @@ -1,6 +1,11 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import type { Writable } from "node:stream"; +import { createWriteStream } from "node:fs"; +import { Writable } from "node:stream"; + +const CAPTURE_LIMIT_BYTES = 16 * 1024 * 1024; +const LOG_LIMIT_BYTES = 64 * 1024 * 1024; +const LOG_TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; export interface ProcessResult { readonly exitCode: number | null; @@ -21,11 +26,21 @@ export interface ExecOptions { export interface ContainerOptions { readonly image: string; readonly name: string; + readonly job: string; + readonly task: string; readonly cpus?: number; readonly memoryMb?: number; readonly network: boolean; } +export interface RunnerContainer { + readonly id: string; + readonly name: string; + readonly job: string; + readonly task: string; + readonly createdAt: string; +} + export class Container { readonly name: string; @@ -34,7 +49,20 @@ export class Container { } static async start(options: ContainerOptions, signal?: AbortSignal): Promise { - const args = ["run", "--detach", "--name", options.name, "--entrypoint", "sh"]; + const args = [ + "run", + "--detach", + "--name", + options.name, + "--label", + "eve-bench=1", + "--label", + `eve-bench.job=${options.job}`, + "--label", + `eve-bench.task=${options.task}`, + "--entrypoint", + "sh", + ]; if (options.cpus !== undefined) args.push("--cpus", String(options.cpus)); if (options.memoryMb !== undefined) args.push("--memory", `${options.memoryMb}m`); if (!options.network) args.push("--network", "none"); @@ -87,6 +115,81 @@ export class Container { } } +export async function assertDockerAvailable(): Promise { + try { + await dockerOk(["version", "--format", "{{.Server.Version}}"], { timeoutMs: 10_000 }); + } catch { + throw new Error("Docker is unavailable; start the Docker daemon and retry."); + } +} + +export async function listRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const args = [ + "ps", + "-a", + "--filter", + "label=eve-bench=1", + ...(filter.job ? ["--filter", `label=eve-bench.job=${filter.job}`] : []), + "--format", + "{{json .}}", + ]; + const result = await dockerOk(args, { timeoutMs: 30_000 }); + return result.stdout + .split("\n") + .filter(Boolean) + .map((line) => { + const row = JSON.parse(line) as Record; + return { + id: row.ID ?? "", + name: row.Names ?? "", + job: labelValue(row.Labels, "eve-bench.job"), + task: labelValue(row.Labels, "eve-bench.task"), + createdAt: row.CreatedAt ?? "", + }; + }); +} + +export async function removeRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const containers = await listRunnerContainers(filter); + if (containers.length > 0) { + await dockerOk(["rm", "--force", "--volumes", ...containers.map((container) => container.id)], { + timeoutMs: 60_000, + }); + } + return containers; +} + +export function createBoundedLog(path: string): Writable { + return createBoundedWritable(createWriteStream(path), LOG_LIMIT_BYTES); +} + +export function createBoundedWritable(output: Writable, limitBytes: number): Writable { + let written = 0; + let truncated = false; + return new Writable({ + write(chunk: Buffer | string, encoding, callback) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); + const remaining = Math.max(0, limitBytes - written); + const kept = bytes.subarray(0, remaining); + written += kept.length; + if (kept.length < bytes.length) truncated = true; + if (kept.length === 0) callback(); + else output.write(kept, callback); + }, + final(callback) { + output.end(truncated ? LOG_TRUNCATED_LINE : undefined, callback); + }, + }); +} + export async function imageExists(tag: string): Promise { const result = await docker(["image", "inspect", tag], { timeoutMs: 30_000 }); return result.exitCode === 0; @@ -132,6 +235,8 @@ function docker(args: readonly string[], options: ExecOptions): Promise { - stdout.push(chunk); + stdoutBytes = appendHead(stdout, stdoutBytes, chunk, CAPTURE_LIMIT_BYTES); options.log?.write(chunk); }); child.stderr.on("data", (chunk: Buffer) => { - stderr.push(chunk); + stderrBytes = appendHead(stderr, stderrBytes, chunk, CAPTURE_LIMIT_BYTES); options.log?.write(chunk); }); child.once("error", (error) => { @@ -175,3 +280,19 @@ function docker(args: readonly string[], options: ExecOptions): Promise 0) chunks.push(kept); + return size + kept.length; +} + +function labelValue(labels: string | undefined, key: string): string { + const prefix = `${key}=`; + return ( + labels + ?.split(",") + .find((label) => label.startsWith(prefix)) + ?.slice(prefix.length) ?? "" + ); +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts index d16feead4e..18f21de5ef 100644 --- a/packages/eve-bench/src/core/job.ts +++ b/packages/eve-bench/src/core/job.ts @@ -2,6 +2,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { DatasetLock } from "./dataset.ts"; +import { assertDockerAvailable, listRunnerContainers, removeRunnerContainers } from "./docker.ts"; import type { Harness } from "./harness.ts"; import { summarize, type JobResult, type TrialResult } from "./result.ts"; import type { Task } from "./task.ts"; @@ -19,10 +20,19 @@ export interface JobInput { readonly concurrency: number; readonly forwardEnv: Readonly>; readonly signal: AbortSignal; + readonly onLog?: (message: string) => void; readonly onTrial?: (trial: TrialResult, resumed: boolean) => void; } export async function runJob(input: JobInput): Promise { + await assertDockerAvailable(); + const leftovers = await listRunnerContainers({ job: input.name }); + if (leftovers.length > 0) { + await removeRunnerContainers({ job: input.name }); + for (const container of leftovers) { + input.onLog?.(`Removed leftover container ${container.name} from job ${input.name}`); + } + } await mkdir(input.dir, { recursive: true }); const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); await writeFile( @@ -69,6 +79,7 @@ export async function runJob(input: JobInput): Promise { bundle, model: input.model, forwardEnv: input.forwardEnv, + job: input.name, dir, signal: input.signal, }); @@ -76,7 +87,11 @@ export async function runJob(input: JobInput): Promise { input.onTrial?.(trial, false); } }); - await Promise.all(workers); + const outcomes = await Promise.allSettled(workers); + if (input.signal.aborted) await removeRunnerContainers({ job: input.name }); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } trials.sort((a, b) => a.task.localeCompare(b.task) || a.attempt - b.attempt); const result: JobResult = { diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts index 372ad58969..c91f71eeb1 100644 --- a/packages/eve-bench/src/core/trial.ts +++ b/packages/eve-bench/src/core/trial.ts @@ -1,10 +1,17 @@ import { createHash } from "node:crypto"; -import { createWriteStream } from "node:fs"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Writable } from "node:stream"; +import { finished } from "node:stream/promises"; -import { buildImage, Container, containerArch, pullImage, type ProcessResult } from "./docker.ts"; +import { + buildImage, + Container, + containerArch, + createBoundedLog, + pullImage, + type ProcessResult, +} from "./docker.ts"; import type { Harness, HarnessBundle } from "./harness.ts"; import type { StepResult, TrialResult, Usage } from "./result.ts"; import type { Task } from "./task.ts"; @@ -17,6 +24,7 @@ export interface TrialInput { readonly model: string; /** Host env forwarded to the harness exec only (provider credentials). */ readonly forwardEnv: Readonly>; + readonly job: string; readonly dir: string; readonly signal: AbortSignal; } @@ -29,7 +37,7 @@ export async function runTrial(input: TrialInput): Promise { const { task, signal } = input; const startedAt = new Date().toISOString(); await mkdir(input.dir, { recursive: true }); - const containerLog = createWriteStream(join(input.dir, "container.log"), { flags: "a" }); + const containerLog = createBoundedLog(join(input.dir, "container.log")); let container: Container | undefined; let agent: StepResult = skipped(); let verifier: StepResult = skipped(); @@ -42,6 +50,8 @@ export async function runTrial(input: TrialInput): Promise { { image, name: `eve-bench-${task.name}-${input.attempt}-${Date.now().toString(36)}`, + job: input.job, + task: task.name, cpus: task.environment.cpus, memoryMb: task.environment.memoryMb, network: task.environment.allowInternet, @@ -74,23 +84,27 @@ export async function runTrial(input: TrialInput): Promise { logsDir: `${LOGS_DIR}/agent`, }; agent = await step(() => - container!.exec(input.harness.command(runContext), { - cwd: workdir, - env: { ...input.forwardEnv, ...input.harness.env(runContext) }, - timeoutMs: task.agentTimeoutMs, - signal, - log: createWriteStream(join(input.dir, "agent.log")), - }), + withBoundedLog(join(input.dir, "agent.log"), (log) => + container!.exec(input.harness.command(runContext), { + cwd: workdir, + env: { ...input.forwardEnv, ...input.harness.env(runContext) }, + timeoutMs: task.agentTimeoutMs, + signal, + log, + }), + ), ); await container.upload(`${join(task.dir, "tests")}/.`, TESTS_DIR, signal); verifier = await step(() => - container!.exec(`bash ${TESTS_DIR}/test.sh`, { - cwd: workdir, - timeoutMs: task.verifierTimeoutMs, - signal, - log: createWriteStream(join(input.dir, "verifier.log")), - }), + withBoundedLog(join(input.dir, "verifier.log"), (log) => + container!.exec(`bash ${TESTS_DIR}/test.sh`, { + cwd: workdir, + timeoutMs: task.verifierTimeoutMs, + signal, + log, + }), + ), ); for (const logs of ["agent", "verifier"]) { @@ -105,6 +119,7 @@ export async function runTrial(input: TrialInput): Promise { } finally { if (container) await container.remove(); containerLog.end(); + await finished(containerLog); } const result: TrialResult = { task: task.name, @@ -123,6 +138,16 @@ export async function runTrial(input: TrialInput): Promise { return result; } +async function withBoundedLog(path: string, run: (log: Writable) => Promise): Promise { + const log = createBoundedLog(path); + try { + return await run(log); + } finally { + log.end(); + await finished(log); + } +} + async function resolveImage(task: Task, log: Writable, signal: AbortSignal): Promise { const options = { timeoutMs: task.buildTimeoutMs, signal, log }; if (task.environment.dockerImage) { diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts index 7f028379f8..cf35a1a996 100644 --- a/packages/eve-bench/src/options.ts +++ b/packages/eve-bench/src/options.ts @@ -18,20 +18,31 @@ export const paths = { export interface TaskSelection { readonly lock: DatasetLock; readonly tasks: readonly string[]; + readonly taskDirs: readonly string[]; } export async function selectTasks(input: { cohort?: string; dataset?: string; task?: readonly string[]; + taskDir?: readonly string[]; }): Promise { + const localTaskDirs = input.taskDir?.map((dir) => resolve(dir)) ?? []; + if (localTaskDirs.length > 0 && !input.cohort && !input.dataset && !input.task?.length) { + return { + lock: { name: "local", version: "0", commit: "", gitUrl: "", tasks: [] }, + tasks: [], + taskDirs: localTaskDirs, + }; + } + if (input.cohort) { const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); const entry = await lock(cohort.dataset); const tasks = input.task?.length ? cohort.tasks.filter((task) => input.task!.includes(task)) : cohort.tasks; - return { lock: entry, tasks }; + return { lock: entry, tasks, taskDirs: localTaskDirs }; } const entry = await lock(input.dataset ?? (await defaultDataset())); @@ -42,7 +53,7 @@ export async function selectTasks(input: { const missing = input.task.filter((task) => !tasks.includes(task)); throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); } - return { lock: entry, tasks }; + return { lock: entry, tasks, taskDirs: localTaskDirs }; } export function jobDir(nameOrPath: string): string { diff --git a/packages/eve-bench/test/stress/stress.test.ts b/packages/eve-bench/test/stress/stress.test.ts new file mode 100644 index 0000000000..85977542bd --- /dev/null +++ b/packages/eve-bench/test/stress/stress.test.ts @@ -0,0 +1,266 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFile, rm } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const tasksRoot = join(packageRoot, "test", "stress", "tasks"); +const jobsRoot = join(packageRoot, ".generated", "jobs"); +const timeout = 300_000; + +interface StepResult { + status: "completed" | "timeout" | "failed" | "skipped"; +} + +interface TrialResult { + task: string; + reward: number | null; + agent: StepResult; + verifier: StepResult; + error?: string; +} + +interface JobResult { + dataset: { name: string; version: string; commit: string }; + trials: TrialResult[]; +} + +interface CliResult { + code: number | null; + stdout: string; + stderr: string; + durationMs: number; +} + +function taskDir(name: string): string { + return join(tasksRoot, name); +} + +function cliArgs( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], +): string[] { + return [ + "src/cli.ts", + "run", + "--harness", + "oracle", + "--job", + job, + "--json", + ...taskDirs.flatMap((dir) => ["--task-dir", dir]), + ...extra, + ]; +} + +async function runCli( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], + fresh = true, +): Promise { + if (fresh) await rm(join(jobsRoot, job), { recursive: true, force: true }); + const started = Date.now(); + const child = spawn(process.execPath, cliArgs(job, taskDirs, extra), { + cwd: packageRoot, + stdio: ["ignore", "pipe", "pipe"], + }); + return collect(child, started); +} + +function collect(child: ReturnType, started = Date.now()): Promise { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout?.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk)); + return new Promise((resolvePromise, reject) => { + child.once("error", reject); + child.once("close", (code) => { + resolvePromise({ + code, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + durationMs: Date.now() - started, + }); + }); + }); +} + +async function readResult(job: string): Promise { + return JSON.parse(await readFile(join(jobsRoot, job, "result.json"), "utf8")) as JobResult; +} + +async function readTrial(job: string, task: string): Promise { + return JSON.parse( + await readFile(join(jobsRoot, job, task, "1", "trial.json"), "utf8"), + ) as TrialResult; +} + +async function dockerNames(filter: string): Promise { + const result = await runCommand("docker", [ + "ps", + "-a", + "--filter", + `name=eve-bench-${filter}`, + "--format", + "{{.Names}}", + ]); + assert.equal(result.code, 0, result.stderr); + return result.stdout.trim() ? result.stdout.trim().split("\n") : []; +} + +async function cleanupContainers(filter: string): Promise { + const names = await dockerNames(filter); + if (names.length === 0) return; + const result = await runCommand("docker", ["rm", "--force", ...names]); + assert.equal(result.code, 0, result.stderr); +} + +async function runCommand(command: string, args: readonly string[]): Promise { + const started = Date.now(); + return collect( + spawn(command, args, { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }), + started, + ); +} + +async function waitFor( + condition: () => Promise, + waitMs: number, + description: string, +): Promise { + const deadline = Date.now() + waitMs; + while (Date.now() < deadline) { + if (await condition()) return; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 250)); + } + assert.fail(`timed out waiting for ${description}`); +} + +function assertPassed(result: CliResult): void { + assert.equal(result.code, 0, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`); +} + +test("happy: records completed steps and reward 1", { timeout }, async () => { + const job = "stress-happy"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("happy")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-happy"); + assert.equal(trial.reward, 1); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "completed"); + assert.deepEqual((await readResult(job)).dataset, { name: "local", version: "0", commit: "" }); +}); + +test("agent-timeout: verifier still runs and the container is removed", { timeout }, async () => { + const job = "stress-agent-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("agent-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-agent-timeout"); + assert.equal(trial.agent.status, "timeout"); + assert.equal(trial.verifier.status, "completed"); + assert.equal(trial.reward, 0); + assert.deepEqual(await dockerNames(job), []); +}); + +test("verifier-timeout: records timeout and null reward", { timeout }, async () => { + const job = "stress-verifier-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("verifier-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-verifier-timeout"); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "timeout"); + assert.equal(trial.reward, null); +}); + +test( + "bad-image: records the setup error, skips both steps, and leaves no container", + { timeout }, + async () => { + const job = "stress-bad-image"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("bad-image")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-bad-image"); + assert.ok(trial.error); + assert.equal(trial.agent.status, "skipped"); + assert.equal(trial.verifier.status, "skipped"); + assert.deepEqual(await dockerNames(job), []); + }, +); + +test( + "abort: exits 130, writes a partial result, cleans up, and resumes with a new task set", + { timeout }, + async () => { + const job = "stress-abort"; + await cleanupContainers(job); + await rm(join(jobsRoot, job), { recursive: true, force: true }); + const child = spawn( + process.execPath, + cliArgs(job, [taskDir("abort-a"), taskDir("abort-b")], ["--concurrency", "2"]), + { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }, + ); + const completed = collect(child); + + try { + await waitFor( + async () => (await dockerNames(job)).length === 2, + 120_000, + "both abort containers", + ); + child.kill("SIGINT"); + const aborted = await completed; + assert.equal(aborted.code, 130, `stdout:\n${aborted.stdout}\nstderr:\n${aborted.stderr}`); + + const partial = await readResult(job); + assert.ok(partial.trials.length <= 2); + await waitFor(async () => (await dockerNames(job)).length === 0, 15_000, "abort cleanup"); + + const resumed = await runCli(job, [taskDir("abort-resume")], [], false); + assertPassed(resumed); + const result = await readResult(job); + assert.equal(result.trials.length, 1); + assert.equal(result.trials[0]?.task, "stress-abort-resume"); + assert.equal(result.trials[0]?.reward, 1); + } finally { + if (child.exitCode === null) child.kill("SIGKILL"); + await cleanupContainers(job); + } + }, +); + +test( + "concurrency: six trials finish substantially faster than serial execution", + { timeout }, + async () => { + const dirs = Array.from({ length: 6 }, (_, index) => taskDir(`concurrency-${index + 1}`)); + await cleanupContainers("stress-concurrency"); + + const single = await runCli("stress-concurrency-single", [dirs[0]!], ["--concurrency", "1"]); + assertPassed(single); + const concurrent = await runCli("stress-concurrency", dirs, ["--concurrency", "6"]); + assertPassed(concurrent); + + const result = await readResult("stress-concurrency"); + assert.equal(result.trials.length, 6); + assert.deepEqual( + result.trials.map((trial) => trial.reward), + [1, 1, 1, 1, 1, 1], + ); + assert.ok( + concurrent.durationMs < single.durationMs * 3, + `single=${single.durationMs}ms concurrent=${concurrent.durationMs}ms`, + ); + }, +); diff --git a/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-a/instruction.md b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-a/task.toml b/packages/eve-bench/test/stress/tasks/abort-a/task.toml new file mode 100644 index 0000000000..d0a9a638a5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-a" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-b/instruction.md b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-b/task.toml b/packages/eve-bench/test/stress/tasks/abort-b/task.toml new file mode 100644 index 0000000000..cf271a1f1a --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-b" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/task.toml b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml new file mode 100644 index 0000000000..cb0de4aea3 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-resume" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh new file mode 100644 index 0000000000..eee73eb76e --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 60; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml new file mode 100644 index 0000000000..58831360d4 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-agent-timeout" + +[agent] +timeout_sec = 3 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/bad-image/instruction.md b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/bad-image/task.toml b/packages/eve-bench/test/stress/tasks/bad-image/task.toml new file mode 100644 index 0000000000..d74d09f84c --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/task.toml @@ -0,0 +1,13 @@ +[task] +name = "stress-bad-image" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false +docker_image = "eve-bench/does-not-exist:1" diff --git a/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh new file mode 100644 index 0000000000..169b2c15e2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +echo 1 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml new file mode 100644 index 0000000000..257a3f2044 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-1" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml new file mode 100644 index 0000000000..21e6f19aeb --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-2" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml new file mode 100644 index 0000000000..c0faeb0b80 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-3" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml new file mode 100644 index 0000000000..0b2d2ba227 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-4" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml new file mode 100644 index 0000000000..52baba26c5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-5" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml new file mode 100644 index 0000000000..15ae72e375 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-6" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/happy/instruction.md b/packages/eve-bench/test/stress/tasks/happy/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/happy/task.toml b/packages/eve-bench/test/stress/tasks/happy/task.toml new file mode 100644 index 0000000000..eaf9709812 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-happy" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/happy/tests/test.sh b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml new file mode 100644 index 0000000000..7672b068e9 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-verifier-timeout" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 3 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh new file mode 100644 index 0000000000..fb0ab7d284 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +sleep 60; echo 1 > /logs/verifier/reward.txt From 262529a80994e469950633639d8931e694b1d81e Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Mon, 14 Sep 2026 14:02:27 -0400 Subject: [PATCH 3/3] feat(eve-bench): benchmark local and CLI agents Signed-off-by: Rui Conti --- .gitignore | 2 - packages/eve-bench/README.md | 41 +- packages/eve-bench/agent/task-path.ts | 11 +- packages/eve-bench/agent/tools/bash.ts | 20 +- packages/eve-bench/package.json | 5 +- packages/eve-bench/src/cli.ts | 52 +- packages/eve-bench/src/core/harness.ts | 2 + packages/eve-bench/src/core/id.test.ts | 15 + packages/eve-bench/src/core/id.ts | 8 + .../eve-bench/src/core/job-identity.test.ts | 92 +++ packages/eve-bench/src/core/job-identity.ts | 129 +++++ packages/eve-bench/src/core/job.ts | 165 +++++- packages/eve-bench/src/core/result.ts | 31 + packages/eve-bench/src/core/task.ts | 3 +- packages/eve-bench/src/core/trial.ts | 15 +- packages/eve-bench/src/harnesses/cli/index.ts | 48 ++ .../eve-bench/src/harnesses/cli/output.ts | 53 ++ .../eve-bench/src/harnesses/cli/plan.test.ts | 207 +++++++ packages/eve-bench/src/harnesses/cli/plan.ts | 208 +++++++ .../eve-bench/src/harnesses/cli/prepare.ts | 247 ++++++++ .../eve-bench/src/harnesses/cli/runner.ts | 241 ++++++++ .../src/harnesses/e0/container-backend.ts | 394 +++++++++++++ packages/eve-bench/src/harnesses/e0/index.ts | 250 ++++++++ .../src/harnesses/e0/snapshot.test.ts | 46 ++ .../eve-bench/src/harnesses/e0/snapshot.ts | 53 ++ packages/eve-bench/src/harnesses/eve/index.ts | 5 +- packages/eve-bench/src/options.test.ts | 44 ++ packages/eve-bench/src/options.ts | 77 ++- packages/eve-bench/src/tools/run_bench.ts | 31 +- packages/eve-bench/src/tools/tools.test.ts | 21 + .../test/scenarios/cli-adapters.test.ts | 311 ++++++++++ .../test/scenarios/e0-container.test.ts | 539 ++++++++++++++++++ .../test/scenarios/job-resume.test.ts | 366 ++++++++++++ packages/eve-bench/test/stress/stress.test.ts | 10 +- research/eve-bench-runner.md | 25 +- 35 files changed, 3671 insertions(+), 96 deletions(-) create mode 100644 packages/eve-bench/src/core/id.test.ts create mode 100644 packages/eve-bench/src/core/id.ts create mode 100644 packages/eve-bench/src/core/job-identity.test.ts create mode 100644 packages/eve-bench/src/core/job-identity.ts create mode 100644 packages/eve-bench/src/harnesses/cli/index.ts create mode 100644 packages/eve-bench/src/harnesses/cli/output.ts create mode 100644 packages/eve-bench/src/harnesses/cli/plan.test.ts create mode 100644 packages/eve-bench/src/harnesses/cli/plan.ts create mode 100644 packages/eve-bench/src/harnesses/cli/prepare.ts create mode 100644 packages/eve-bench/src/harnesses/cli/runner.ts create mode 100644 packages/eve-bench/src/harnesses/e0/container-backend.ts create mode 100644 packages/eve-bench/src/harnesses/e0/index.ts create mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.test.ts create mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.ts create mode 100644 packages/eve-bench/src/options.test.ts create mode 100644 packages/eve-bench/test/scenarios/cli-adapters.test.ts create mode 100644 packages/eve-bench/test/scenarios/e0-container.test.ts create mode 100644 packages/eve-bench/test/scenarios/job-resume.test.ts diff --git a/.gitignore b/.gitignore index adebbfbd91..621fc0c100 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ coverage/ .pnpm-store/ .bun/ .npm/ -__pycache__/ # Test artifacts .vitest/ @@ -33,7 +32,6 @@ packages/eve-buzz-acp-adapter/NOTICE # Logs *.log -/jobs/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md index b3a41bf612..a33ef56b68 100644 --- a/packages/eve-bench/README.md +++ b/packages/eve-bench/README.md @@ -29,19 +29,36 @@ pnpm bench run --harness oracle --cohort smoke # reference solutions; pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task +# Build and cache a local e0 snapshot without calling a model. +pnpm bench prepare --harness e0 --agent /path/to/internal-agents/agents/e0 \ + --model openai/gpt-5.6-terra + +# Run one e0 smoke task. The source app must have installed workspace dependencies. +AI_GATEWAY_API_KEY=... pnpm bench run --harness e0 \ + --agent /path/to/internal-agents/agents/e0 \ + --model deepseek/deepseek-v4-pro --cohort smoke --task fix-git --job e0-deepseek + pnpm bench report --format junit --out junit.xml pnpm bench diff ``` Every command accepts `--json` and never prompts. Jobs live in -`.generated/jobs//`; re-running with the same `--job` name resumes it, -skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight -trials and removes their containers. +`.generated/jobs//`. Re-running the same job resumes only when the model, +harness, bundle provenance, dataset, tasks, and attempt count match. Invalid or +mismatched trial records fail closed instead of being overwritten. Ctrl-C +aborts in-flight trials and removes their containers. Pass `--task-dir ` more than once to run one or more local Terminal-Bench task directories. A local-only run does not sync a dataset and records its dataset as `local`. +On an ARM host, an amd64-only prebuilt task image can make native CLI agents run +through emulation. Pass `--native-build` to build the selected task's reviewed +`environment/Dockerfile` for the host architecture instead. Use the flag for +every comparison arm, and do not compare its scores or latency directly with +prebuilt-image runs. `job.json` records `local-dockerfile` instead of the +prebuilt image name. + ## How a trial runs 1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). @@ -93,6 +110,22 @@ trials. `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass eve's nested sandbox on purpose. +- `e0`: snapshots an installed local e0 app and its source-backed `eve-code` + extension. The bundle records source, dependency, and installed eve hashes. + It preserves e0's instructions, coding capabilities, tools, skills, and + authored worker model. The adapter replaces production sandbox and transport + integrations with the task-container backend and a local eve channel. It + removes production Connect configuration, channels, connections, schedules, + instrumentation, and sandbox bootstrap. `--model` changes the root model, + not e0's authored worker model. + +- `pi`, `opencode`, and `codex`: prepare explicitly versioned CLI bundles on the + host and route full provider-qualified model IDs through an HTTPS API base. + Pass an exact `--version`. The default base is Vercel AI Gateway. Codex uses + the Responses protocol; pi and OpenCode use chat completions. Hermes fails + explicitly until a pinned Python/uv bundle and verified custom-provider + routing are implemented. + - `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner and as the reward ceiling. @@ -130,7 +163,7 @@ diff the result against a downloaded base-job artifact. - `agent/`: harness behavior under evaluation (model binding, instructions, tools) - `src/core/`: runner library with no dependency on eve or third parties -- `src/harnesses/`: `eve` and `oracle` +- `src/harnesses/`: `eve`, `e0`, CLI competitors, and `oracle` - `src/tools/`: eve tool definitions over the runner core - `src/options.ts`: run options shared by the CLI and tools - `src/cli.ts`: `eve-bench` CLI diff --git a/packages/eve-bench/agent/task-path.ts b/packages/eve-bench/agent/task-path.ts index 8303fe355f..06d9e700d7 100644 --- a/packages/eve-bench/agent/task-path.ts +++ b/packages/eve-bench/agent/task-path.ts @@ -1,13 +1,18 @@ -import { isAbsolute, resolve } from "node:path"; +import { resolve } from "node:path"; export function taskRoot(): string { const root = process.env.EVE_BENCH_TASK_WORKDIR; if (root === undefined || root.length === 0) { throw new Error("EVE_BENCH_TASK_WORKDIR must be set by the eve-bench harness."); } - return root; + return resolve(root); } export function resolveTaskPath(path = "."): string { - return isAbsolute(path) ? path : resolve(taskRoot(), path); + const root = taskRoot(); + const target = resolve(root, path); + if (target !== root && !target.startsWith(`${root}/`)) { + throw new Error("Task paths must stay inside the task working directory."); + } + return target; } diff --git a/packages/eve-bench/agent/tools/bash.ts b/packages/eve-bench/agent/tools/bash.ts index 2d8121c64d..24ac9f0db4 100644 --- a/packages/eve-bench/agent/tools/bash.ts +++ b/packages/eve-bench/agent/tools/bash.ts @@ -8,6 +8,24 @@ import { z } from "zod"; import { taskRoot } from "../task-path.js"; const execAsync = promisify(exec); +const ENV_KEYS = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "TMP", + "TEMP", + "TZ", + "TERM", +]; + +function commandEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of ENV_KEYS) if (process.env[key] !== undefined) env[key] = process.env[key]; + return env; +} export default defineTool({ approval: never(), @@ -20,7 +38,7 @@ export default defineTool({ try { const { stdout, stderr } = await execAsync(command, { cwd: taskRoot(), - env: process.env, + env: commandEnv(), maxBuffer: 10 * 1024 * 1024, signal: ctx.abortSignal, }); diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index 4ad272b0eb..b9d7a9b8ac 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -9,7 +9,10 @@ "type": "module", "scripts": { "bench": "node src/cli.ts", - "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "test": "pnpm run test:unit && pnpm run test:integration", + "test:unit": "pnpm run typecheck && node --test src/*.test.ts src/core/*.test.ts src/harnesses/**/*.test.ts src/tools/*.test.ts", + "test:integration": "node --test test/scenarios/*.test.ts", + "test:scenario": "node --test test/scenarios/*.test.ts", "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts index 95636fad45..67ddd3ce32 100644 --- a/packages/eve-bench/src/cli.ts +++ b/packages/eve-bench/src/cli.ts @@ -15,6 +15,7 @@ import { lock, paths, selectHarness, + selectNativeBuild, selectTasks, } from "./options.ts"; @@ -23,8 +24,11 @@ const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets eve-bench tasks list [--cohort ] print task names eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] - [--concurrency N] [--harness eve|oracle] [--eve local|] [--job ] + [--concurrency N] [--harness eve|e0|oracle|pi|opencode|codex] [--job ] + [--eve local|] [--agent ] [--version ] + [--reasoning ] [--base-url ] [--native-build] [--format console|json|junit] + eve-bench prepare --harness --model [harness options] build/cache without model calls eve-bench report [--format console|json|junit] eve-bench diff [--json] @@ -38,8 +42,9 @@ try { case "tasks": await tasks(subcommand, rest); break; + case "prepare": case "run": - await run(subcommand === undefined ? [] : [subcommand, ...rest]); + await run(subcommand === undefined ? [] : [subcommand, ...rest], command === "prepare"); break; case "report": await report(subcommand, rest); @@ -83,7 +88,7 @@ async function tasks(sub: string | undefined, args: string[]): Promise { throw new Error(`unknown tasks subcommand: ${sub ?? "(none)"}\n${USAGE}`); } -async function run(args: string[]): Promise { +async function run(args: string[], prepareOnly = false): Promise { const { values } = parseArgs({ args, options: { @@ -96,14 +101,35 @@ async function run(args: string[]): Promise { concurrency: { type: "string", default: "4" }, eve: { type: "string", default: "local" }, harness: { type: "string", default: "eve" }, + agent: { type: "string" }, + version: { type: "string" }, + "base-url": { type: "string" }, + reasoning: { type: "string" }, + "native-build": { type: "boolean", default: false }, job: { type: "string" }, format: { type: "string", default: "console" }, json: { type: "boolean" }, }, }); - const harness = selectHarness(values.harness, values.eve); + const attempts = positiveInteger(values.attempts, "--attempts"); + const concurrency = positiveInteger(values.concurrency, "--concurrency"); + const harness = selectHarness(values.harness, values.eve, { + ...values, + baseUrl: values["base-url"], + }); const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); if (!model) throw new Error("--model is required"); + if (prepareOnly) { + const bundle = await harness.prepare({ model, cacheDir: join(paths.generatedRoot, "cache") }); + process.stdout.write(`${JSON.stringify(bundle, null, 2)}\n`); + return; + } + const forwardEnv = forwardedEnv(harness.credentials ?? []); + if (harness.credentials?.length && Object.keys(forwardEnv).length === 0) { + throw new Error( + "No model credentials exported. Set AI_GATEWAY_API_KEY (or the eve model's provider credentials); use prepare for a model-free build.", + ); + } const selection = await selectTasks({ ...values, taskDir: values["task-dir"] }); const datasetTaskDirs = selection.tasks.length ? await taskDirs( @@ -111,8 +137,9 @@ async function run(args: string[]): Promise { selection.tasks, ) : []; - const loaded = await Promise.all( - [...datasetTaskDirs, ...selection.taskDirs].map((dir) => loadTask(dir)), + const loaded = await selectNativeBuild( + await Promise.all([...datasetTaskDirs, ...selection.taskDirs].map((dir) => loadTask(dir))), + values["native-build"], ); const name = values.job ?? defaultJobName(`${harness.name}-${model}`); const controller = new AbortController(); @@ -131,9 +158,9 @@ async function run(args: string[]): Promise { tasks: loaded, harness, model, - attempts: Number(values.attempts), - concurrency: Number(values.concurrency), - forwardEnv: forwardedEnv(), + attempts, + concurrency, + forwardEnv, signal: controller.signal, onLog: (message) => process.stderr.write(`${message}\n`), onTrial: (trial, resumed) => { @@ -177,6 +204,13 @@ async function diff(base: string | undefined, args: string[]): Promise { process.stdout.write(values.json ? `${JSON.stringify(deltas, null, 2)}\n` : formatDiff(deltas)); } +function positiveInteger(value: string, name: string): number { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1) + throw new Error(`${name} must be a positive integer`); + return parsed; +} + function reportFormat(value: string | undefined): ReportFormat { if (value === "console" || value === "json" || value === "junit") return value; throw new Error(`unknown format: ${value}`); diff --git a/packages/eve-bench/src/core/harness.ts b/packages/eve-bench/src/core/harness.ts index 797d481d27..e3555c71cc 100644 --- a/packages/eve-bench/src/core/harness.ts +++ b/packages/eve-bench/src/core/harness.ts @@ -5,6 +5,8 @@ */ export interface Harness { readonly name: string; + /** Host environment names allowed to cross into this harness process. */ + readonly credentials?: readonly string[]; prepare(ctx: HarnessPrepareContext): Promise; /** Optional per-task host paths uploaded into the install directory before `command` runs. */ stage?(ctx: { readonly taskDir: string }): readonly string[]; diff --git a/packages/eve-bench/src/core/id.test.ts b/packages/eve-bench/src/core/id.test.ts new file mode 100644 index 0000000000..cccf1ed792 --- /dev/null +++ b/packages/eve-bench/src/core/id.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertSafeId } from "./id.ts"; + +test("accepts bounded path- and Docker-safe identifiers", () => { + for (const value of ["job", "task-1", "a.b_c", `a${"x".repeat(127)}`]) { + assert.equal(assertSafeId(value, "job"), value); + } +}); + +test("rejects traversal, separators, labels, whitespace, and oversized identifiers", () => { + for (const value of ["", ".", "..", "../job", "a/b", "a:b", "two words", `a${"x".repeat(128)}`]) { + assert.throws(() => assertSafeId(value, "task"), /path- and Docker-safe/); + } +}); diff --git a/packages/eve-bench/src/core/id.ts b/packages/eve-bench/src/core/id.ts new file mode 100644 index 0000000000..54bc39ecc5 --- /dev/null +++ b/packages/eve-bench/src/core/id.ts @@ -0,0 +1,8 @@ +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; + +export function assertSafeId(value: string, kind: "job" | "task"): string { + if (!SAFE_ID.test(value) || value === "." || value === "..") { + throw new Error(`${kind} name must be 1-128 path- and Docker-safe characters`); + } + return value; +} diff --git a/packages/eve-bench/src/core/job-identity.test.ts b/packages/eve-bench/src/core/job-identity.test.ts new file mode 100644 index 0000000000..dd61bd45dd --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { assertJobIdentity, createJobIdentity, identityHash } from "./job-identity.ts"; + +const input = { + name: "resume", + harness: "eve@local", + model: "provider/model", + attempts: 2, + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a", "b"], + }, + tasks: [ + { name: "a", hash: "aaa" }, + { name: "b", hash: "bbb" }, + ], + provenance: { key: "bundle", build: { version: "1", sha: "abc" } }, +}; + +test("identity is stable across object key and selected task order, and JSON round trips", () => { + const expected = createJobIdentity(input); + const reordered = createJobIdentity({ + ...input, + tasks: [...input.tasks].reverse(), + provenance: { build: { sha: "abc", version: "1" }, key: "bundle" }, + }); + assert.deepEqual(reordered, expected); + assertJobIdentity(JSON.parse(JSON.stringify(expected)), reordered); + assert.equal( + identityHash({ b: [1, { z: 2, a: 3 }], a: true }), + identityHash({ a: true, b: [1, { a: 3, z: 2 }] }), + ); + assert.notEqual(identityHash([1, 2]), identityHash([2, 1])); + assert.notEqual(identityHash(["ab", "c"]), identityHash(["a", "bc"])); +}); + +for (const [field, changes] of [ + ["job", { name: "different" }], + ["harness", { harness: "oracle" }], + ["model", { model: "provider/other" }], + ["attempts", { attempts: 3 }], + ["tasks", { tasks: [{ name: "renamed", hash: "aaa" }, input.tasks[1]!] }], + ["tasks", { tasks: [{ name: "a", hash: "changed" }, input.tasks[1]!] }], + ["tasks", { tasks: input.tasks.slice(0, 1) }], + ["provenance", { provenance: { key: "bundle", build: { version: "1", sha: "changed" } } }], + ...(["name", "version", "gitUrl", "commit"] as const).map( + (key) => ["dataset", { dataset: { ...input.dataset, [key]: "changed" } }] as const, + ), +] as const) { + test(`changed ${field} refuses resume with an actionable error`, () => { + assert.throws( + () => + assertJobIdentity(createJobIdentity(input), createJobIdentity({ ...input, ...changes })), + new RegExp(`differs \\(.*${field}.*\\).*new job name`), + ); + }); +} + +test("unselected dataset task inventory does not change selected content identity", () => { + assert.deepEqual( + createJobIdentity({ ...input, dataset: { ...input.dataset, tasks: ["a", "b", "unselected"] } }), + createJobIdentity(input), + ); +}); + +test("legacy, unsupported, and incomplete identities fail closed", () => { + const expected = createJobIdentity(input); + for (const saved of [undefined, null, {}, "bad", { ...expected, version: 2 }, { version: 1 }]) { + assert.throws(() => assertJobIdentity(saved, expected), /new job name/); + } +}); + +test("duplicate task names cannot share trial directories", () => { + assert.throws( + () => createJobIdentity({ ...input, tasks: [input.tasks[0]!, input.tasks[0]!] }), + /unique/, + ); +}); + +test("absent provenance is stable and distinct from recorded provenance", () => { + const { provenance: _provenance, ...without } = input; + assert.deepEqual(createJobIdentity(without), createJobIdentity({ ...without, provenance: {} })); + assert.throws( + () => assertJobIdentity(createJobIdentity(without), createJobIdentity(input)), + /provenance/, + ); +}); diff --git a/packages/eve-bench/src/core/job-identity.ts b/packages/eve-bench/src/core/job-identity.ts new file mode 100644 index 0000000000..c4b0001dc0 --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.ts @@ -0,0 +1,129 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, opendir } from "node:fs/promises"; +import { isAbsolute, join, relative, sep } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import type { Task } from "./task.ts"; + +export interface JobIdentity { + readonly version: 1; + readonly job: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: string; + readonly tasks: string; + readonly provenance: string; +} + +export function identityHash(value: unknown): string { + const json = JSON.stringify(value, (_key, item: unknown) => { + if (item !== null && typeof item === "object" && !Array.isArray(item)) { + return Object.fromEntries( + Object.entries(item).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + } + return item; + }); + if (json === undefined) throw new Error("Job identity must be JSON serializable."); + return createHash("sha256").update(json).digest("hex"); +} + +export function createJobIdentity(input: { + readonly name: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: DatasetLock; + readonly tasks: readonly { readonly name: string; readonly hash: string }[]; + readonly provenance?: Record; +}): JobIdentity { + const { name, version, gitUrl, commit } = input.dataset; + const tasks = [...input.tasks].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + if (new Set(tasks.map((task) => task.name)).size !== tasks.length) { + throw new Error("Selected task names must be unique to resume safely."); + } + return { + version: 1, + job: input.name, + harness: input.harness, + model: input.model, + attempts: input.attempts, + dataset: identityHash({ name, version, gitUrl, commit }), + tasks: identityHash(tasks), + provenance: identityHash(input.provenance ?? {}), + }; +} + +export function assertJobIdentity(saved: unknown, expected: JobIdentity): void { + if (!saved || typeof saved !== "object" || !("version" in saved) || saved.version !== 1) { + throw new Error( + `Job ${expected.job} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + const fields = Object.keys(expected) as (keyof JobIdentity)[]; + const changed = fields.filter( + (key) => !(key in saved) || Reflect.get(saved, key) !== expected[key], + ); + if (changed.length > 0) { + throw new Error( + `Job ${expected.job} resume identity differs (${changed.join(", ")}). Use a new job name for changed inputs.`, + ); + } +} + +/** Hash only the selected tree, independent of its location, mtimes, and traversal order. */ +export async function hashTask(task: Task): Promise { + const dockerfileDir = relative(task.dir, task.environment.dockerfileDir); + if (isAbsolute(dockerfileDir) || dockerfileDir === ".." || dockerfileDir.startsWith(`..${sep}`)) { + throw new Error(`Task ${task.name}: environment must be inside the selected task tree.`); + } + const entries: { path: string; kind: string; executable: number; hash?: string }[] = []; + let bytes = 0; + let count = 0; + async function visit(path: string, depth: number): Promise { + if (depth > 64 || ++count > 100_000) { + throw new Error(`Task ${task.name}: identity scan exceeds 64 levels or 100,000 entries.`); + } + const absolute = join(task.dir, path); + const stat = await lstat(absolute); + // Do not follow links into unselected tasks (or hash a link but execute its changed target). + if (!stat.isDirectory() && !stat.isFile()) { + throw new Error( + `Task ${task.name}: identity requires regular files/directories, not ${path || "."}.`, + ); + } + const entry = { + path: path.split(sep).join("/"), + kind: stat.isDirectory() ? "directory" : "file", + executable: stat.mode & 0o111, + }; + if (stat.isDirectory()) { + entries.push(entry); + for await (const child of await opendir(absolute)) { + await visit(join(path, child.name), depth + 1); + } + } else { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(absolute)) { + bytes += chunk.length; + if (bytes > 1024 * 1024 * 1024) { + throw new Error(`Task ${task.name}: identity scan exceeds 1 GiB.`); + } + hash.update(chunk); + } + entries.push({ ...entry, hash: hash.digest("hex") }); + } + } + await visit("", 0); + entries.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + return identityHash({ + entries, + instruction: task.instruction, + agentTimeoutMs: task.agentTimeoutMs, + verifierTimeoutMs: task.verifierTimeoutMs, + buildTimeoutMs: task.buildTimeoutMs, + environment: { ...task.environment, dockerfileDir: dockerfileDir.split(sep).join("/") }, + }); +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts index 18f21de5ef..87367afb5a 100644 --- a/packages/eve-bench/src/core/job.ts +++ b/packages/eve-bench/src/core/job.ts @@ -1,10 +1,13 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, open, readFile, readdir, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import type { DatasetLock } from "./dataset.ts"; import { assertDockerAvailable, listRunnerContainers, removeRunnerContainers } from "./docker.ts"; import type { Harness } from "./harness.ts"; -import { summarize, type JobResult, type TrialResult } from "./result.ts"; +import { assertSafeId } from "./id.ts"; +import { assertJobIdentity, createJobIdentity, hashTask, identityHash } from "./job-identity.ts"; +import { assertTrialResult, summarize, type JobResult, type TrialResult } from "./result.ts"; import type { Task } from "./task.ts"; import { runTrial } from "./trial.ts"; @@ -25,37 +28,89 @@ export interface JobInput { } export async function runJob(input: JobInput): Promise { + assertSafeId(input.name, "job"); + if (!Number.isSafeInteger(input.attempts) || input.attempts < 1) { + throw new Error("attempts must be a positive integer"); + } + if (!Number.isSafeInteger(input.concurrency) || input.concurrency < 1) { + throw new Error("concurrency must be a positive integer"); + } + if (input.tasks.length === 0) throw new Error("benchmark requires at least one task"); + for (const task of input.tasks) assertSafeId(task.name, "task"); + // Containers are labeled by name, so different output directories must share the lock. + const lockPath = join(tmpdir(), `eve-bench-job-${identityHash(input.name)}.lock`); + const lock = await open(lockPath, "wx", 0o600).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + throw new Error( + `Job ${input.name} is locked. Wait for the active run or use a new job name. If the owner crashed, confirm it has stopped before removing ${lockPath}.`, + ); + }); + try { + await lock.writeFile(`${JSON.stringify({ pid: process.pid, job: input.name })}\n`); + return await runLockedJob(input); + } finally { + await lock.close(); + await unlink(lockPath); + } +} + +async function runLockedJob(input: JobInput): Promise { + await mkdir(input.dir, { recursive: true }); + const savedIdentity = await readSavedIdentity(input.dir, input.name); + const taskHashes = []; + for (const task of input.tasks) taskHashes.push({ name: task.name, hash: await hashTask(task) }); + const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); + const identity = createJobIdentity({ + ...input, + harness: input.harness.name, + tasks: taskHashes, + provenance: bundle.provenance, + }); + if (savedIdentity !== undefined) assertJobIdentity(savedIdentity, identity); + await assertDockerAvailable(); const leftovers = await listRunnerContainers({ job: input.name }); + if (savedIdentity === undefined && leftovers.length > 0) { + throw new Error( + `Job ${input.name} has containers but no resume identity here. Use a new job name or resume from the original job directory.`, + ); + } + if (savedIdentity === undefined) { + await writeFile( + join(input.dir, "job.json"), + `${JSON.stringify( + { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + attempts: input.attempts, + tasks: input.tasks.map((task) => task.name), + taskEnvironments: input.tasks.map((task) => ({ + task: task.name, + image: task.environment.dockerImage ?? null, + source: task.environment.dockerImage ? "prebuilt" : "local-dockerfile", + })), + bundle: bundle.provenance ?? {}, + identity, + startedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + { flag: "wx" }, + ); + } if (leftovers.length > 0) { await removeRunnerContainers({ job: input.name }); for (const container of leftovers) { input.onLog?.(`Removed leftover container ${container.name} from job ${input.name}`); } } - await mkdir(input.dir, { recursive: true }); - const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); - await writeFile( - join(input.dir, "job.json"), - `${JSON.stringify( - { - job: input.name, - dataset: { - name: input.dataset.name, - version: input.dataset.version, - commit: input.dataset.commit, - }, - harness: input.harness.name, - model: input.model, - attempts: input.attempts, - tasks: input.tasks.map((task) => task.name), - bundle: bundle.provenance ?? {}, - startedAt: new Date().toISOString(), - }, - null, - 2, - )}\n`, - ); const queue = input.tasks.flatMap((task) => Array.from({ length: input.attempts }, (_, index) => ({ task, attempt: index + 1 })), @@ -66,7 +121,12 @@ export async function runJob(input: JobInput): Promise { const next = queue.shift(); if (!next || input.signal.aborted) return; const dir = join(input.dir, next.task.name, String(next.attempt)); - const existing = await readTrial(dir); + const existing = await readTrial(dir, { + task: next.task.name, + attempt: next.attempt, + harness: input.harness.name, + model: input.model, + }); if (existing) { trials.push(existing); input.onTrial?.(existing, true); @@ -114,10 +174,57 @@ export async function readJobResult(dir: string): Promise { return JSON.parse(await readFile(join(dir, "result.json"), "utf8")) as JobResult; } -async function readTrial(dir: string): Promise { +async function readSavedIdentity(dir: string, name: string): Promise { + let text: string; try { - return JSON.parse(await readFile(join(dir, "trial.json"), "utf8")) as TrialResult; + text = await readFile(join(dir, "job.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if ((await readdir(dir)).length === 0) return undefined; + throw new Error( + `Job ${name} has existing files but no job.json. Use a new job name to run safely.`, + ); + } + let saved; + try { + saved = JSON.parse(text); } catch { - return undefined; + throw new Error(`Job ${name} has invalid job.json. Use a new job name to run safely.`); + } + if (saved?.identity?.version !== 1) { + throw new Error( + `Job ${name} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + return saved.identity; +} + +async function readTrial( + dir: string, + expected: Pick, +): Promise { + let text: string; + try { + text = await readFile(join(dir, "trial.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error( + `Trial ${expected.task}#${expected.attempt} has invalid JSON; remove the job or use a new job name.`, + ); + } + assertTrialResult(parsed); + for (const key of ["task", "attempt", "harness", "model"] as const) { + if (parsed[key] !== expected[key]) { + throw new Error( + `Trial ${expected.task}#${expected.attempt} has mismatched ${key}; use a new job name.`, + ); + } } + return parsed; } diff --git a/packages/eve-bench/src/core/result.ts b/packages/eve-bench/src/core/result.ts index d42e58a798..ffec8037dc 100644 --- a/packages/eve-bench/src/core/result.ts +++ b/packages/eve-bench/src/core/result.ts @@ -43,6 +43,37 @@ export interface JobSummary { readonly meanReward: number; } +export function assertTrialResult(value: unknown): asserts value is TrialResult { + if (!value || typeof value !== "object") throw new Error("trial record must be an object"); + const trial = value as Partial; + const step = (value: unknown) => { + const item = value as Partial | undefined; + return ( + item !== undefined && + ["completed", "timeout", "failed", "skipped"].includes(item.status ?? "") && + (item.exitCode === null || Number.isInteger(item.exitCode)) && + typeof item.durationMs === "number" && + Number.isFinite(item.durationMs) + ); + }; + if ( + typeof trial.task !== "string" || + !Number.isInteger(trial.attempt) || + typeof trial.harness !== "string" || + typeof trial.model !== "string" || + !( + trial.reward === null || + (typeof trial.reward === "number" && Number.isFinite(trial.reward)) + ) || + !step(trial.agent) || + !step(trial.verifier) || + typeof trial.startedAt !== "string" || + typeof trial.finishedAt !== "string" + ) { + throw new Error("trial record has an invalid shape"); + } +} + export function summarize(trials: readonly TrialResult[]): JobSummary { const tasks = new Set(trials.map((trial) => trial.task)).size; const rewards = trials.map((trial) => trial.reward ?? 0); diff --git a/packages/eve-bench/src/core/task.ts b/packages/eve-bench/src/core/task.ts index 8788b8e74b..d2a2b61859 100644 --- a/packages/eve-bench/src/core/task.ts +++ b/packages/eve-bench/src/core/task.ts @@ -1,6 +1,7 @@ import { access, readFile } from "node:fs/promises"; import { basename, join } from "node:path"; +import { assertSafeId } from "./id.ts"; import { parseToml, type TomlTable } from "./toml.ts"; export interface Task { @@ -33,7 +34,7 @@ export async function loadTask(dir: string): Promise { const dockerfileDir = join(dir, "environment"); if (dockerImage === undefined) await access(join(dockerfileDir, "Dockerfile")); return { - name: shortName(str(task, "name") ?? basename(dir)), + name: assertSafeId(shortName(str(task, "name") ?? basename(dir)), "task"), dir, instruction, agentTimeoutMs: seconds(agent, "timeout_sec"), diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts index c91f71eeb1..4730ace546 100644 --- a/packages/eve-bench/src/core/trial.ts +++ b/packages/eve-bench/src/core/trial.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Writable } from "node:stream"; import { finished } from "node:stream/promises"; @@ -129,12 +129,19 @@ export async function runTrial(input: TrialInput): Promise { reward, agent, verifier, - ...(usage ? { usage } : {}), - ...(error ? { error } : {}), startedAt, finishedAt: new Date().toISOString(), }; - await writeFile(join(input.dir, "trial.json"), `${JSON.stringify(result, null, 2)}\n`); + if (usage) Object.assign(result, { usage }); + if (error) Object.assign(result, { error }); + const target = join(input.dir, "trial.json"); + const temporary = join(input.dir, `trial.${process.pid}.tmp`); + try { + await writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`, { flag: "wx" }); + await rename(temporary, target); + } finally { + await rm(temporary, { force: true }); + } return result; } diff --git a/packages/eve-bench/src/harnesses/cli/index.ts b/packages/eve-bench/src/harnesses/cli/index.ts new file mode 100644 index 0000000000..09cbf90849 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/index.ts @@ -0,0 +1,48 @@ +import type { Harness } from "../../core/harness.ts"; +import { prepareCliBundle } from "./prepare.ts"; +import { + GATEWAY_BASE_URL, + shellQuote, + validateModel, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; + +export function createCliHarness( + name: "hermes" | "pi" | "opencode" | "codex", + options: { version: string; baseUrl?: string; reasoning?: string }, +): Harness { + const supported = validateOptions(name, options); + const pinned = { ...options }; + return { + name: `${name}@${pinned.version}`, + credentials: ["AI_GATEWAY_API_KEY"], + prepare: async (ctx) => { + validateModel(ctx.model); + const bundle = await prepareCliBundle(supported, pinned, ctx); + return { + ...bundle, + provenance: { + ...bundle.provenance, + model: ctx.model, + baseUrl: pinned.baseUrl ?? GATEWAY_BASE_URL, + reasoning: pinned.reasoning ?? null, + protocol: supported === "codex" ? "responses" : "chat-completions", + }, + }; + }, + command: (ctx) => + `exec ${shellQuote(`${ctx.installDir}/arch/node`)} ${shellQuote(`${ctx.installDir}/runner.ts`)}`, + env: (ctx) => { + validateModel(ctx.model); + return { + EVE_BENCH_CLI_NAME: name, + EVE_BENCH_CLI_OPTIONS: JSON.stringify(pinned), + EVE_BENCH_CLI_CONTEXT: JSON.stringify(ctx), + }; + }, + }; +} + +export type { CliName, CliOptions }; diff --git a/packages/eve-bench/src/harnesses/cli/output.ts b/packages/eve-bench/src/harnesses/cli/output.ts new file mode 100644 index 0000000000..7c5e8eb7f7 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/output.ts @@ -0,0 +1,53 @@ +import { StringDecoder } from "node:string_decoder"; + +/** Retain a suffix so even a credential split across pipe chunks is redacted. */ +export class SecretRedactor { + private readonly decoder = new StringDecoder("utf8"); + private pending = ""; + private readonly secrets: string[]; + private readonly keep: number; + + constructor(secret: string) { + this.secrets = [ + ...new Set([secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]), + ] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + this.keep = Math.max(1, ...this.secrets.map((value) => value.length)); + } + + push(chunk: Buffer): string { + this.pending += this.decoder.write(chunk); + return this.drain(false); + } + + finish(): string { + this.pending += this.decoder.end(); + return this.drain(true); + } + + private drain(final: boolean): string { + let output = ""; + while (this.pending.length > 0) { + const safe = final ? this.pending.length : Math.max(0, this.pending.length - this.keep + 1); + if (!safe) break; + let start = this.pending.length; + let matched = ""; + for (const secret of this.secrets) { + const index = this.pending.indexOf(secret); + if (index >= 0 && index < start) { + start = index; + matched = secret; + } + } + if (matched && start < safe) { + output += this.pending.slice(0, start) + "[REDACTED]"; + this.pending = this.pending.slice(start + matched.length); + } else { + output += this.pending.slice(0, safe); + this.pending = this.pending.slice(safe); + } + } + return output; + } +} diff --git a/packages/eve-bench/src/harnesses/cli/plan.test.ts b/packages/eve-bench/src/harnesses/cli/plan.test.ts new file mode 100644 index 0000000000..7be2b4233c --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createCliHarness } from "./index.ts"; +import { + executionPlan, + packageName, + validateModel, + validateOptions, + type SupportedCliName, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const ctx = { + model: "deepseek/deepseek-v4", + installDir: "/installed-agent", + instructionPath: "/instruction.md", + taskWorkdir: "/workspace", + logsDir: "/logs/agent", +}; + +for (const name of ["pi", "opencode", "codex"] as const) { + test(`${name} requires a pin and rejects credential-bearing endpoints`, () => { + for (const version of [ + "", + "latest", + "next", + "^1.2.3", + "1", + "1.2", + "*", + "1.2.3; echo unsafe", + "v1.2.3", + "01.2.3", + ]) { + assert.throws(() => createCliHarness(name, { version }), /explicit exact version/); + } + assert.equal(createCliHarness(name, { version: "1.2.3-rc.1" }).name, `${name}@1.2.3-rc.1`); + for (const baseUrl of [ + "https://user:key@example.com/v1", + "https://example.com/v1?key=secret", + "https://example.com/#secret", + "http://example.com", + "not a URL", + ]) { + assert.throws( + () => createCliHarness(name, { version: "1.2.3", baseUrl }), + /without credentials/, + ); + } + }); + + test(`${name} retains full provider-qualified model IDs`, () => { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6", "openai/acme/served-model"]) { + const plan = executionPlan( + name, + { version: "1.2.3", reasoning: "high" }, + { ...ctx, model }, + "/runtime", + "--a 'quote'\n$(touch /never)", + ); + assert.ok( + plan.args.includes(model) || plan.args.includes(`--model=eve-bench-gateway/${model}`), + ); + if (name === "pi") { + assert.equal(plan.stdin, "--a 'quote'\n$(touch /never)"); + assert.ok(!plan.args.includes("--")); + } else { + assert.equal(plan.args.at(-1), "--a 'quote'\n$(touch /never)"); + assert.equal(plan.args.at(-2), "--"); + } + assert.match(plan.config, /https:\/\/ai-gateway.vercel.sh\/v1/); + assert.ok( + !JSON.stringify( + createCliHarness(name, { version: "1.2.3" }).env({ ...ctx, model }), + ).includes("API_KEY"), + ); + } + }); +} + +test("pi custom models use runtime environment references and explicit thinking capability", () => { + const plan = executionPlan( + "pi", + { version: "0.74.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "chat-completions"); + assert.deepEqual(JSON.parse(plan.config).providers["eve-bench-gateway"], { + baseUrl: "https://ai-gateway.vercel.sh/v1", + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [{ id: ctx.model, reasoning: true }], + }); + assert.deepEqual(plan.args.slice(0, 4), ["--print", "--mode", "json", "--session-dir"]); + assert.ok(plan.args.includes("--thinking")); +}); + +test("OpenCode uses a custom compatible provider, not OpenAI's Responses loader", () => { + const plan = executionPlan( + "opencode", + { version: "1.2.3", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + const provider = JSON.parse(plan.config).provider["eve-bench-gateway"]; + assert.equal(provider.npm, "@ai-sdk/openai-compatible"); + assert.deepEqual(provider.env, ["OPENAI_API_KEY"]); + assert.equal(provider.options.baseURL, "https://ai-gateway.vercel.sh/v1"); + assert.deepEqual(provider.models[ctx.model].variants, { high: { reasoningEffort: "high" } }); + assert.ok(plan.args.includes("--dangerously-skip-permissions")); + assert.ok(plan.args.includes("--variant")); +}); + +test("Codex has env-key auth, a Responses wire API, and no auth.json", () => { + const plan = executionPlan( + "codex", + { version: "0.118.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "responses"); + assert.match(plan.config, /env_key = "AI_GATEWAY_API_KEY"/); + assert.match(plan.config, /requires_openai_auth = false/); + assert.match(plan.config, /wire_api = "responses"/); + assert.doesNotMatch(plan.config, /auth.json/); + assert.ok(plan.args.includes("--dangerously-bypass-approvals-and-sandbox")); + assert.ok(plan.args.includes("model_reasoning_effort=high")); +}); + +test("no reasoning option means no imposed reasoning setting", () => { + for (const name of ["pi", "opencode", "codex"] as const) { + const plan = executionPlan(name, { version: "1.2.3" }, ctx, "/runtime", "instruction"); + assert.doesNotMatch(plan.config, /reasoningEffort|model_reasoning_effort|"reasoning"/); + assert.ok(!plan.args.includes("--variant")); + } + assert.throws(() => validateOptions("pi", { version: "1.2.3", reasoning: "max" }), /reasoning/); + assert.throws( + () => validateOptions("codex", { version: "1.2.3", reasoning: "off" }), + /reasoning/, + ); +}); + +test("package names follow Harbor's pi rename boundary", () => { + assert.equal(packageName("pi", "0.73.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0-rc.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("pi", "1.0.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("codex", "0.118.0"), "@openai/codex"); + assert.equal(packageName("opencode", "1.2.3"), "opencode-ai"); +}); + +test("Hermes fails explicitly before installation, without a substitute agent", () => { + assert.throws( + () => createCliHarness("hermes", { version: "v2026.7.20" }), + /Hermes is not supported.*pinned, audited Python\/uv bundle/, + ); +}); + +test("model validation rejects unqualified IDs and command/control injection", () => { + for (const model of [ + "gpt-5.6", + "openai/", + "openai/gpt\nmodel", + "openai/$(cmd)", + "openai/gpt model", + ]) { + assert.throws(() => validateModel(model), /full provider-qualified/); + } +}); + +test("constructor snapshots options instead of accepting later mutation", () => { + const options = { version: "1.2.3" }; + const harness = createCliHarness("pi", options); + options.version = "latest"; + assert.match(harness.env(ctx).EVE_BENCH_CLI_OPTIONS!, /1.2.3/); +}); + +test("redaction covers every byte boundary including escaped and Unicode secrets", () => { + for (const secret of ["fake-key-abcdef", 'fake-"key', "fake-秘密"]) { + for (const form of [secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]) { + const input = Buffer.from(`prefix ${form} suffix ${form}!`); + for (let split = 0; split <= input.length; split++) { + const redactor = new SecretRedactor(secret); + const output = + redactor.push(input.subarray(0, split)) + + redactor.push(input.subarray(split)) + + redactor.finish(); + assert.equal(output, "prefix [REDACTED] suffix [REDACTED]!"); + } + } + } +}); + +// Keep the supported surface explicit in the pure tests, independently of parent wiring. +const supported: SupportedCliName[] = ["pi", "opencode", "codex"]; +test("all supported adapters implement the shared Harness contract", () => { + for (const name of supported) { + const harness = createCliHarness(name, { version: "1.2.3" }); + assert.equal(typeof harness.prepare, "function"); + assert.match(harness.command({ ...ctx, installDir: "/has space/and'quote" }), /^exec '/); + } +}); diff --git a/packages/eve-bench/src/harnesses/cli/plan.ts b/packages/eve-bench/src/harnesses/cli/plan.ts new file mode 100644 index 0000000000..36731d671e --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.ts @@ -0,0 +1,208 @@ +import type { HarnessRunContext } from "../../core/harness.ts"; + +export type CliName = "hermes" | "pi" | "opencode" | "codex"; +export type SupportedCliName = Exclude; +export interface CliOptions { + readonly version: string; + readonly baseUrl?: string; + readonly reasoning?: string; +} + +export const GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"; +const EXACT_VERSION = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/; + +export function validateOptions(name: CliName, options: CliOptions): SupportedCliName { + if (name === "hermes") { + throw new Error( + "Hermes is not supported by this adapter: Harbor installs a mutable main/scripts/install.sh and only establishes OPENAI_BASE_URL routing for openai/*, not Gateway DeepSeek. A pinned, audited Python/uv bundle and verified custom-provider routing are required; no fallback agent is run.", + ); + } + if (!["pi", "opencode", "codex"].includes(name)) throw new Error("Unsupported CLI harness"); + if (!EXACT_VERSION.test(options.version ?? "")) { + throw new Error( + "CLI harness requires an explicit exact version (for example 1.2.3); tags, ranges, and latest are not allowed.", + ); + } + let url: URL; + try { + url = new URL(options.baseUrl ?? GATEWAY_BASE_URL); + } catch { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (options.reasoning !== undefined) { + const allowed = + name === "pi" + ? ["off", "minimal", "low", "medium", "high", "xhigh"] + : name === "codex" + ? ["none", "minimal", "low", "medium", "high", "xhigh"] + : undefined; + if ( + allowed + ? !allowed.includes(options.reasoning) + : !/^[a-zA-Z0-9_-]{1,64}$/.test(options.reasoning) + ) { + throw new Error(`Unsupported ${name} reasoning value`); + } + } + return name; +} + +export function validateModel(model: string): void { + if ( + model.length > 256 || + !/^[a-zA-Z0-9][a-zA-Z0-9._-]*\/[a-zA-Z0-9][a-zA-Z0-9/._:-]*$/.test(model) + ) { + throw new Error( + "CLI model must be a full provider-qualified model ID, without whitespace or control characters.", + ); + } +} + +export function packageName(name: SupportedCliName, version: string): string { + if (name === "codex") return "@openai/codex"; + if (name === "opencode") return "opencode-ai"; + const [major, minor] = version.split(".").map(Number); + // Harbor pi.py changes package namespace at 0.74.0 (prereleases precede it). + return major === 0 && (minor! < 74 || (minor === 74 && version.startsWith("0.74.0-"))) + ? "@mariozechner/pi-coding-agent" + : "@earendil-works/pi-coding-agent"; +} + +export interface ExecutionPlan { + readonly args: string[]; + readonly stdin?: string; + readonly env: Record; + readonly configPath: string; + readonly config: string; + readonly protocol: "chat-completions" | "responses"; +} + +/** Native commands/config follow Harbor installed/{pi,opencode,codex}.py at + * 2fe1615503fed39ad82b7ce09b22497996b30f1f. Gateway model IDs deliberately stay intact. + */ +export function executionPlan( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessRunContext, + runtimeDir: string, + instruction: string, +): ExecutionPlan { + validateOptions(name, options); + validateModel(ctx.model); + const baseUrl = options.baseUrl ?? GATEWAY_BASE_URL; + const json = (value: unknown) => `${JSON.stringify(value, null, 2)}\n`; + if (name === "pi") { + return { + protocol: "chat-completions", + args: [ + "--print", + "--mode", + "json", + "--session-dir", + `${runtimeDir}/pi/sessions`, + "--provider", + "eve-bench-gateway", + "--model", + ctx.model, + ...(options.reasoning ? ["--thinking", options.reasoning] : []), + ], + // pi 0.80.3's parseArgs treats -- as an unknown flag. main.readPipedStdin + // passes stdin to prepareInitialMessage without flag or @file interpretation. + stdin: instruction, + env: { PI_CODING_AGENT_DIR: `${runtimeDir}/pi` }, + configPath: `${runtimeDir}/pi/models.json`, + config: json({ + providers: { + "eve-bench-gateway": { + baseUrl, + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [ + options.reasoning + ? { id: ctx.model, reasoning: options.reasoning !== "off" } + : { id: ctx.model }, + ], + }, + }, + }), + }; + } + if (name === "opencode") { + return { + protocol: "chat-completions", + // Only the first component selects the CLI provider; the API ID stays intact. + args: [ + `--model=eve-bench-gateway/${ctx.model}`, + "run", + "--format=json", + ...(options.reasoning ? ["--variant", options.reasoning] : []), + "--thinking", + "--dangerously-skip-permissions", + "--", + instruction, + ], + env: { + OPENCODE_FAKE_VCS: "git", + XDG_DATA_HOME: `${runtimeDir}/opencode/xdg-data`, + XDG_STATE_HOME: `${runtimeDir}/opencode/xdg-state`, + }, + configPath: `${runtimeDir}/home/.config/opencode/opencode.json`, + // Harbor's mimo.py (OpenCode derivative) documents the compatible SDK override. + // OpenCode provider.ts resolves single-entry provider.env into the SDK API key. + config: json({ + provider: { + "eve-bench-gateway": { + npm: "@ai-sdk/openai-compatible", + env: ["OPENAI_API_KEY"], + options: { baseURL: baseUrl }, + models: { + [ctx.model]: options.reasoning + ? { + reasoning: true, + variants: { [options.reasoning]: { reasoningEffort: options.reasoning } }, + } + : {}, + }, + }, + }, + }), + }; + } + return { + protocol: "responses", + args: [ + "exec", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--model", + ctx.model, + "--json", + "--enable", + "unified_exec", + ...(options.reasoning ? ["-c", `model_reasoning_effort=${options.reasoning}`] : []), + "--", + instruction, + ], + env: { CODEX_HOME: `${runtimeDir}/codex`, OPENAI_BASE_URL: baseUrl }, + configPath: `${runtimeDir}/codex/config.toml`, + // Unlike Harbor's auth.json, Codex's native env_key needs no persisted credential. + // Source: openai/codex, codex-rs/model-provider-info/src/lib.rs, ModelProviderInfo. + config: + `model_provider = "eve-bench-gateway"\nopenai_base_url = ${JSON.stringify(baseUrl)}\n` + + `[model_providers.eve-bench-gateway]\nname = "eve-bench-gateway"\n` + + `base_url = ${JSON.stringify(baseUrl)}\nwire_api = "responses"\n` + + `env_key = "AI_GATEWAY_API_KEY"\nrequires_openai_auth = false\n`, + }; +} + +export function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} diff --git a/packages/eve-bench/src/harnesses/cli/prepare.ts b/packages/eve-bench/src/harnesses/cli/prepare.ts new file mode 100644 index 0000000000..9fd28ff377 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/prepare.ts @@ -0,0 +1,247 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; +import { packageName, type CliOptions, type SupportedCliName } from "./plan.ts"; + +const exec = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; +const SOURCES = ["plan.ts", "runner.ts", "output.ts", "prepare.ts", "index.ts"]; + +/** Installation is host-only, serial, and credential-free. Both Linux/glibc + * architectures are prepared because the Harness contract selects arch only after upload. + * Lifecycle scripts are disabled; native CLI launchers use their optional platform packages. + */ +export async function prepareCliBundle( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessPrepareContext, +): Promise { + const cache = resolve(ctx.cacheDir, "cli"); + await mkdir(cache, { recursive: true }); + const scratch = await mkdtemp(join(cache, ".prepare-")); + const cleanEnv = { PATH: process.env.PATH, HOME: join(scratch, "home"), CI: "true" }; + await mkdir(cleanEnv.HOME, { recursive: true }); + try { + let pnpmVersion: string; + try { + pnpmVersion = ( + await exec("pnpm", ["--version"], { env: cleanEnv, cwd: scratch }) + ).stdout.trim(); + } catch { + throw new Error( + "CLI bundles require pnpm 11+ on the host; pnpm could not run. No container installation was attempted.", + ); + } + if (!/^(?:1[1-9]|[2-9]\d)\./.test(pnpmVersion)) { + throw new Error( + "CLI bundles require pnpm 11+ for explicit Linux CPU/libc installs. No container installation was attempted.", + ); + } + const sources = await Promise.all(SOURCES.map((file) => readFile(join(here, file)))); + const sourceSha256 = sha256(Buffer.concat(sources)); + const pkg = packageName(name, options.version); + const key = sha256( + JSON.stringify({ pkg, version: options.version, pnpmVersion, NODE_VERSION, sourceSha256 }), + ); + const dir = join(cache, key); + if (await exists(join(dir, "provenance.json"))) return await bundleAt(dir); + + const started = Date.now(); + const building = join(scratch, "bundle"); + const common = join(building, "common"); + await mkdir(common, { recursive: true }); + for (const source of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(here, source), join(common, source)); + const lockSha256: Record = {}; + const nodeSha256: Record = {}; + for (const arch of ARCHES) { + const archDir = join(building, arch); + const app = join(archDir, "app"); + await mkdir(app, { recursive: true }); + await writeFile( + join(app, "package.json"), + JSON.stringify({ + name: "eve-bench-cli-bundle", + private: true, + type: "module", + dependencies: { [pkg]: options.version }, + }), + ); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + try { + await exec( + "pnpm", + [ + "install", + "--ignore-workspace", + "--ignore-scripts", + "--ignore-pnpmfile", + "--no-frozen-lockfile", + "--prod", + "--os=linux", + `--cpu=${arch}`, + "--libc=glibc", + "--package-import-method=copy", + "--network-concurrency=2", + "--child-concurrency=1", + "--store-dir", + join(cache, "store"), + "--reporter=silent", + ], + { cwd: app, env: cleanEnv, maxBuffer: 1024 * 1024 }, + ); + } catch { + // Do not embed package-manager stdout/errors: registry diagnostics may contain credentials. + throw new Error( + `Host pnpm install failed for ${pkg}@${options.version} (linux/${arch}, scripts disabled). No CLI setup was moved into the task timeout.`, + ); + } + const manifest = JSON.parse( + await readFile(join(app, "node_modules", pkg, "package.json"), "utf8"), + ) as { + version: string; + bin?: string | Record; + }; + if (manifest.version !== options.version) + throw new Error("Installed CLI version does not match the explicit pin"); + const bin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.[name]; + if (!bin || isAbsolute(bin)) + throw new Error("Pinned CLI package has no supported relative executable entry"); + const packageRoot = join(app, "node_modules", pkg); + let entry = resolve(packageRoot, bin); + let native = false; + if (name === "opencode") { + const platform = arch === "x64" ? "opencode-linux-x64-baseline" : "opencode-linux-arm64"; + entry = join( + app, + "node_modules/.pnpm", + `${platform}@${options.version}`, + "node_modules", + platform, + "bin/opencode", + ); + native = true; + } + if (!(await exists(entry))) { + throw new Error( + "Pinned CLI package executable is missing for the requested Linux platform", + ); + } + if (!native && relative(packageRoot, entry).startsWith("..")) { + throw new Error("Pinned CLI package executable escapes its package"); + } + await writeFile( + join(archDir, "entry.json"), + JSON.stringify({ entry: relative(archDir, entry), native }), + ); + lockSha256[arch] = sha256(await readFile(join(app, "pnpm-lock.yaml"))); + nodeSha256[arch] = await installNode(archDir, arch, scratch); + } + const provenance = { + key, + name, + version: options.version, + package: pkg, + sourceSha256, + installLocation: "host", + containerInstallMs: 0, + hostPrepareMs: Date.now() - started, + pnpmVersion, + nodeVersion: NODE_VERSION, + platforms: ["linux-x64-glibc", "linux-arm64-glibc"], + lifecycleScripts: false, + lockSha256, + nodeSha256, + adapterSources: { + harbor: + "https://github.com/harbor-framework/harbor/tree/2fe1615503fed39ad82b7ce09b22497996b30f1f/src/harbor/agents/installed", + codexEnvAuth: + "https://github.com/openai/codex/blob/1aaa453ce2868e14c10089c226ebec265657cb27/codex-rs/model-provider-info/src/lib.rs", + opencodeProviders: + "https://github.com/anomalyco/opencode/blob/ac1758c0e6b8e7368be133e87e8e27a5f60dec04/packages/opencode/src/provider/provider.ts", + }, + limitations: [ + "Native CLI/platform compatibility is not inferred from mock tests.", + "Host pnpm 11+ and Linux glibc containers are required; no musl/system-package bootstrap.", + "CLI-owned startup work is included in invocation wall time, not independently measured.", + "Custom model context/output limits use CLI defaults; reasoning settings are not normalized between agents.", + "Codex requires a Responses-compatible endpoint/model; no chat-completions fallback.", + "Native state stays ephemeral; only redacted stdout/stderr is exported. Usage is not normalized.", + ], + }; + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await cp(join(building, "provenance.json"), join(common, "provenance.json")); + try { + await rename(building, dir); + } catch (error) { + if (!(await exists(join(dir, "provenance.json")))) throw error; + } + return await bundleAt(dir); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function bundleAt(dir: string): Promise { + // Keep platform trees out of the common upload (which otherwise doubles every transfer). + return { + dir: join(dir, "common"), + arch: { x64: join(dir, "x64"), arm64: join(dir, "arm64") }, + provenance: JSON.parse(await readFile(join(dir, "provenance.json"), "utf8")) as Record< + string, + unknown + >, + }; +} + +async function installNode(archDir: string, arch: string, scratch: string): Promise { + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const base = `https://nodejs.org/dist/v${NODE_VERSION}`; + const sums = (await download(`${base}/SHASUMS256.txt`, 1024 * 1024)).toString("utf8"); + const expected = sums + .split("\n") + .find((line) => line.trim().endsWith(` ${name}.tar.xz`)) + ?.split(/\s+/)[0]; + if (!expected || !/^[a-f0-9]{64}$/.test(expected)) + throw new Error("Pinned Node checksum is missing"); + const archive = await download(`${base}/${name}.tar.xz`, 128 * 1024 * 1024); + if (sha256(archive) !== expected) throw new Error("Pinned Node archive checksum mismatch"); + const path = join(scratch, `${name}.tar.xz`); + await writeFile(path, archive); + await exec("tar", ["-xJf", path, "--strip-components=2", "-C", archDir, `${name}/bin/node`]); + await chmod(join(archDir, "node"), 0o755); + await rm(path); + return expected; +} + +async function download(url: string, limit: number): Promise { + const response = await fetch(url, { signal: AbortSignal.timeout(120_000), redirect: "error" }); + if (!response.ok || !response.body) + throw new Error(`Pinned Node download failed (HTTP ${response.status})`); + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > limit) throw new Error("Pinned Node download exceeds size limit"); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/eve-bench/src/harnesses/cli/runner.ts b/packages/eve-bench/src/harnesses/cli/runner.ts new file mode 100644 index 0000000000..9662e74016 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/runner.ts @@ -0,0 +1,241 @@ +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { finished } from "node:stream/promises"; + +import type { HarnessRunContext } from "../../core/harness.ts"; +import { + GATEWAY_BASE_URL, + executionPlan, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const OUTPUT_LIMIT = 64 * 1024 * 1024; +const INSTRUCTION_LIMIT = 1024 * 1024; + +async function main(): Promise { + const ctx = JSON.parse(process.env.EVE_BENCH_CLI_CONTEXT!) as HarnessRunContext; + const options = JSON.parse(process.env.EVE_BENCH_CLI_OPTIONS!) as CliOptions; + const name = validateOptions(process.env.EVE_BENCH_CLI_NAME as CliName, options); + const start = Date.now(); + let agentStart: number | undefined; + let exitCode = 1; + let failure: string | undefined; + let signal: string | null = null; + await mkdir(ctx.logsDir, { recursive: true }); + const runtimeDir = await mkdtemp(join(ctx.installDir, ".cli-runtime-")); + const secret = process.env.AI_GATEWAY_API_KEY; + try { + if (!secret?.trim()) + throw new Error("AI_GATEWAY_API_KEY must be forwarded to the CLI runtime environment."); + if (secret.length > 8192 || /[\r\n\0]/.test(secret)) + throw new Error("AI_GATEWAY_API_KEY has an invalid format."); + if ((await stat(ctx.instructionPath)).size > INSTRUCTION_LIMIT) + throw new Error("CLI instruction exceeds 1 MiB."); + const instruction = await readFile(ctx.instructionPath, "utf8"); + const plan = executionPlan(name, options, ctx, runtimeDir, instruction); + const home = join(runtimeDir, "home"); + await mkdir(home, { recursive: true }); + await mkdir(dirname(plan.configPath), { recursive: true }); + await writeFile(plan.configPath, plan.config, { mode: 0o600 }); + const { entry, native = false } = JSON.parse( + await readFile(join(ctx.installDir, "arch/entry.json"), "utf8"), + ) as { entry: string; native?: boolean }; + const executable = join(ctx.installDir, "arch", entry); + const baseEnv = { + PATH: `${ctx.installDir}/arch:${process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin"}`, + HOME: home, + XDG_CONFIG_HOME: `${home}/.config`, + XDG_CACHE_HOME: `${runtimeDir}/cache`, + XDG_DATA_HOME: `${runtimeDir}/data`, + XDG_STATE_HOME: `${runtimeDir}/state`, + LANG: "C.UTF-8", + NO_COLOR: "1", + ...plan.env, + }; + const version = await run( + executable, + ["--version"], + baseEnv, + ctx, + "cli-setup.log", + secret, + false, + undefined, + native, + ); + const escapedVersion = options.version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if ( + version.code !== 0 || + !new RegExp(`(?:^|\\s)${escapedVersion}(?:\\s|$)`).test(version.text) + ) { + throw new Error( + "Pinned CLI failed its --version check; native package/runtime compatibility must be checked before benchmarking.", + ); + } + agentStart = Date.now(); + const result = await run( + executable, + plan.args, + { ...baseEnv, AI_GATEWAY_API_KEY: secret, OPENAI_API_KEY: secret }, + ctx, + `${name}.jsonl`, + secret, + name !== "codex", + plan.stdin, + native, + ); + exitCode = result.code; + signal = result.signal; + if (result.errorEvent && exitCode === 0) { + exitCode = 1; + failure = `${name} emitted an error event despite exiting zero.`; + } + } catch (error) { + // Only our actionable messages are emitted; filesystem/spawn diagnostics can contain arbitrary data. + const message = error instanceof Error ? error.message : ""; + failure = + /^(AI_GATEWAY_API_KEY |CLI instruction |Pinned CLI |CLI model |Unsupported .* reasoning)/.test( + message, + ) + ? message + : "CLI setup or execution failed; inspect the redacted CLI logs and the pinned bundle."; + } finally { + const end = Date.now(); + await rm(runtimeDir, { recursive: true, force: true }); + const runtimeResult: Record = { + name, + version: options.version, + model: ctx.model, + baseUrl: options.baseUrl ?? GATEWAY_BASE_URL, + reasoning: options.reasoning ?? null, + protocol: name === "codex" ? "responses" : "chat-completions", + installLocation: "host", + containerInstallMs: 0, + setupMs: (agentStart ?? end) - start, + agentMs: agentStart === undefined ? 0 : end - agentStart, + exitCode, + signal, + }; + if (failure) runtimeResult.failure = failure; + await writeFile( + join(ctx.logsDir, "cli-runtime.json"), + `${JSON.stringify(runtimeResult, null, 2)}\n`, + ); + } + if (failure) process.stderr.write(`[eve-bench] ${failure}\n`); + return exitCode; +} + +async function run( + entry: string, + args: string[], + env: Record, + ctx: HarnessRunContext, + logName: string, + secret: string, + detectErrors = false, + input?: string, + native = false, +): Promise<{ code: number; signal: string | null; text: string; errorEvent: boolean }> { + const log = createWriteStream(join(ctx.logsDir, logName), { mode: 0o600 }); + let bytes = 0; + let text = ""; + let line = ""; + let skipLine = false; + let errorEvent = false; + let truncated = false; + const emit = (chunk: string, stdout: boolean) => { + if (stdout && detectErrors) { + for (const piece of chunk.split(/(?<=\n)/)) { + if (!skipLine) line += piece; + if (line.length > INSTRUCTION_LIMIT) { + line = ""; + skipLine = true; + } + if (piece.endsWith("\n")) { + if (!skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Native diagnostics need not be JSON. */ + } + } + line = ""; + skipLine = false; + } + } + } + if (text.length < 65536) text += chunk.slice(0, 65536 - text.length); + const buffer = Buffer.from(chunk); + const kept = buffer.subarray(0, Math.max(0, OUTPUT_LIMIT - bytes)); + bytes += kept.length; + if (kept.length > 0) { + log.write(kept); + process.stdout.write(kept); + } + if (!truncated && kept.length < buffer.length) { + truncated = true; + log.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + process.stdout.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + } + }; + const child = spawn(native ? entry : process.execPath, native ? args : [entry, ...args], { + cwd: ctx.taskWorkdir, + env, + stdio: ["pipe", "pipe", "pipe"], + }); + // A CLI may exit before consuming stdin; its exit status remains authoritative. + child.stdin.on("error", () => {}); + child.stdin.end(input); + const forwardTerm = () => child.kill("SIGTERM"); + const forwardInt = () => child.kill("SIGINT"); + process.on("SIGTERM", forwardTerm); + process.on("SIGINT", forwardInt); + const stdout = new SecretRedactor(secret); + const stderr = new SecretRedactor(secret); + child.stdout.on("data", (chunk: Buffer) => emit(stdout.push(chunk), true)); + child.stderr.on("data", (chunk: Buffer) => emit(stderr.push(chunk), false)); + let spawnFailed = false; + child.on("error", () => { + spawnFailed = true; + }); + const result = await new Promise<{ code: number; signal: string | null }>((resolve) => { + child.on("close", (code, signal) => resolve({ code: spawnFailed ? 1 : (code ?? 1), signal })); + }); + process.off("SIGTERM", forwardTerm); + process.off("SIGINT", forwardInt); + emit(stdout.finish(), true); + emit(stderr.finish(), false); + if (detectErrors && line && !skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Partial diagnostic. */ + } + } + log.end(); + await finished(log); + return { ...result, text, errorEvent }; +} + +function isErrorEvent(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const event = value as { type?: string; errorMessage?: unknown; message?: unknown }; + if (event.type === "error" || typeof event.errorMessage === "string") return true; + return event.message !== undefined && isErrorEvent(event.message); +} + +main().then( + (code) => { + process.exitCode = code; + }, + () => { + process.stderr.write("[eve-bench] Invalid CLI runtime context or unwritable logs directory.\n"); + process.exitCode = 1; + }, +); diff --git a/packages/eve-bench/src/harnesses/e0/container-backend.ts b/packages/eve-bench/src/harnesses/e0/container-backend.ts new file mode 100644 index 0000000000..3f15a9c6ec --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/container-backend.ts @@ -0,0 +1,394 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rm } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import type { Readable } from "node:stream"; +import type { SandboxBackend, SandboxProcess, SandboxSession } from "eve/sandbox"; + +const MAX_BYTES = 16 * 1024 * 1024; +const ENV_KEYS = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "TMP", + "TEMP", + "TZ", + "TERM", +]; + +function checkSize(bytes: number, operation: string): void { + if (bytes > MAX_BYTES) throw new Error(`${operation} exceeds 16 MiB`); +} + +/** + * Attach to the task container; the runner owns isolation, networking and deletion. + * Explicit roots permit local tests, not an isolated host sandbox. No symlinks are + * created. File paths and cwd map /workspace to root, but shell source is never + * rewritten: commands (including code__apply_patch's root) must use the actual + * task directory or relative paths, not literal /workspace paths in shell text. + * Built-in glob/grep embed paths in shell commands: pass the actual root as path + * instead of relying on their /workspace default. + */ +export function containerBackend( + root = process.env.EVE_BENCH_TASK_WORKDIR!, + { commandTimeoutMs = 120_000 }: { commandTimeoutMs?: number } = {}, +): SandboxBackend { + if (!root?.trim() || !isAbsolute(root)) + throw new Error( + "EVE_BENCH_TASK_WORKDIR must be an absolute task directory (or pass an explicit test root)", + ); + if (arguments[0] === undefined && process.env.EVE_BENCH_CONTAINER !== "1") + throw new Error("The e0 backend requires EVE_BENCH_CONTAINER=1 inside a task container"); + if ( + !Number.isFinite(commandTimeoutMs) || + commandTimeoutMs <= 0 || + commandTimeoutMs > 2_147_483_647 + ) + throw new Error("commandTimeoutMs must be positive and at most 2147483647"); + root = resolve(root); + const name = "e0-task-container"; + const children = new Set(); + const skillsHome = () => process.env.HOME && resolve(process.env.HOME, ".agents/skills"); + const within = (parent: string, path: string) => path === parent || path.startsWith(`${parent}/`); + const resolvePath = (path: string): string => { + let target: string; + if (path === "$HOME" || path.startsWith("$HOME/")) { + if (!process.env.HOME) throw new Error("HOME is unavailable in the task container"); + target = resolve(process.env.HOME, path.slice(6)); + } else if (path === "/workspace" || path.startsWith("/workspace/")) { + target = resolve(root, path === "/workspace" ? "." : path.slice(11)); + } else { + target = resolve(root, path); + } + const skills = skillsHome(); + if (!within(root, target) && !(skills && within(skills, target))) { + throw new Error("Sandbox paths must stay inside the task workspace or seeded skills"); + } + return target; + }; + const rejectSymlinkPath = async (target: string) => { + const base = within(root, target) ? root : skillsHome()!; + let current = base; + for (const component of relative(base, target).split(sep).filter(Boolean)) { + current = resolve(current, component); + try { + if ((await lstat(current)).isSymbolicLink()) { + throw new Error("Sandbox file operations do not follow symbolic links"); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + } + }; + const stop = async () => { + await Promise.all([...children].map((child) => child.kill())); + }; + const session: SandboxSession & { readonly description: string } = { + id: `${name}:${root}`, + description: `Task container. Work in ${JSON.stringify(root)}; use this actual root for code__apply_patch. File operations and cwd map /workspace here, but shell command text does not. Use relative paths or the actual task directory in bash. Always pass the actual task directory as glob/grep's path; their /workspace default is not remapped. Skill files are under $HOME/.agents/skills.`, + resolvePath, + async spawn({ command, workingDirectory, env = {}, abortSignal }) { + abortSignal?.throwIfAborted(); + const shellEnv: NodeJS.ProcessEnv = {}; + for (const key of ENV_KEYS) { + if (process.env[key] !== undefined) shellEnv[key] = process.env[key]; + } + // Neither inherit credentials nor let login profiles reload production env. + const child = spawn("/bin/bash", ["--noprofile", "--norc", "-c", command], { + cwd: resolvePath(workingDirectory ?? "."), + env: { ...shellEnv, ...env }, + detached: true, + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let killed = false; + let timedOut = false; + let failure: Error | undefined; + let bytes = 0; + const killGroup = () => { + if (settled || killed || child.pid === undefined) return; + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + killed = true; + }; + const abort = () => killGroup(); + abortSignal?.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(() => { + timedOut = true; + killGroup(); + }, commandTimeoutMs); + const stream = (source: Readable) => { + let closed = false; + return new ReadableStream({ + start(controller) { + source.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength; + if (bytes > MAX_BYTES && !failure) { + failure = new Error("Command output exceeds 16 MiB"); + killGroup(); + } + if (closed) return; + if (failure) { + closed = true; + controller.error(failure); + } else controller.enqueue(chunk); + }); + source.once("error", (error) => { + failure ??= error; + killGroup(); + if (!closed) { + closed = true; + controller.error(error); + } + }); + source.once("close", () => { + if (!closed) { + closed = true; + controller.close(); + } + }); + }, + cancel() { + closed = true; + killGroup(); + }, + }); + }; + const stdout = stream(child.stdout); + const stderr = stream(child.stderr); + // A shell can exit while a background descendant has redirected its pipes. + // Kill the group on exit as well as timeout, before dropping its PID. + child.once("exit", killGroup); + const completed = new Promise<{ exitCode: number }>((resolve, reject) => { + child.once("error", (error) => { + failure ??= error; + }); + child.once("close", (code) => { + settled = true; + clearTimeout(timer); + abortSignal?.removeEventListener("abort", abort); + children.delete(handle); + if (abortSignal?.aborted) reject(abortSignal.reason); + else if (failure) reject(failure); + else resolve({ exitCode: timedOut ? 124 : (code ?? 137) }); + }); + }); + // Stream consumers may wait only after draining output, or never wait at all. + void completed.catch(() => {}); + const handle: SandboxProcess = { + pid: child.pid, + stdout, + stderr, + wait: () => completed, + async kill() { + killGroup(); + await completed.catch(() => {}); + }, + }; + children.add(handle); + if (abortSignal?.aborted) killGroup(); + return handle; + }, + async run(input) { + const child = await session.spawn(input); + const collect = async (stream: ReadableStream) => { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); + }; + try { + const [stdout, stderr, result] = await Promise.all([ + collect(child.stdout), + collect(child.stderr), + child.wait(), + ]); + return { + ...result, + stdout, + stderr: + result.exitCode === 124 + ? `${stderr}\nCommand ended with exit 124 (timeout limit ${commandTimeoutMs}ms); its process group was stopped.\n` + : stderr, + }; + } finally { + await child.kill(); + } + }, + async readBinaryFile({ path, abortSignal }) { + abortSignal?.throwIfAborted(); + let file; + try { + // Nonblocking open + regular-file check avoid hangs on FIFOs/devices. + const target = resolvePath(path); + await rejectSymlinkPath(target); + file = await open(target, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + try { + const stat = await file.stat(); + if (!stat.isFile()) throw new Error("Only regular files can be read"); + checkSize(stat.size, "File read"); + const chunks: Uint8Array[] = []; + let bytes = 0; + for (;;) { + abortSignal?.throwIfAborted(); + const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_BYTES + 1 - bytes)); + const { bytesRead } = await file.read(buffer); + abortSignal?.throwIfAborted(); + if (bytesRead === 0) return Buffer.concat(chunks); + bytes += bytesRead; + checkSize(bytes, "File read"); + chunks.push(buffer.subarray(0, bytesRead)); + } + } finally { + await file.close(); + } + }, + async readFile(options) { + const bytes = await session.readBinaryFile(options); + return bytes === null + ? null + : new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + }, + async readTextFile({ encoding = "utf-8", startLine, endLine, ...options }) { + if ( + (startLine !== undefined && (!Number.isInteger(startLine) || startLine < 1)) || + (endLine !== undefined && (!Number.isInteger(endLine) || endLine < (startLine ?? 1))) + ) + throw new Error("Invalid 1-based line range"); + const bytes = await session.readBinaryFile(options); + if (bytes === null) return null; + const text = + encoding === "utf-8" || encoding === "utf8" + ? new TextDecoder("utf-8", { fatal: true }).decode(bytes) + : Buffer.from(bytes).toString(bufferEncoding(encoding)); + return startLine === undefined && endLine === undefined + ? text + : text + .split("\n") + .slice((startLine ?? 1) - 1, endLine) + .join("\n"); + }, + async writeBinaryFile({ path, content, abortSignal }) { + abortSignal?.throwIfAborted(); + checkSize(content.byteLength, "File write"); + const target = resolvePath(path); + await rejectSymlinkPath(target); + await mkdir(dirname(target), { recursive: true }); + await rejectSymlinkPath(target); + abortSignal?.throwIfAborted(); + const file = await open( + target, + constants.O_WRONLY | constants.O_CREAT | constants.O_NONBLOCK, + ); + try { + if (!(await file.stat()).isFile()) throw new Error("Only regular files can be written"); + abortSignal?.throwIfAborted(); + await file.truncate(0); + await file.writeFile(content, { signal: abortSignal }); + } finally { + await file.close(); + } + }, + async writeTextFile({ content, encoding = "utf8", ...options }) { + const codec = bufferEncoding(encoding); + checkSize(Buffer.byteLength(content, codec), "File write"); + await session.writeBinaryFile({ ...options, content: Buffer.from(content, codec) }); + }, + async writeFile({ content, ...options }) { + options.abortSignal?.throwIfAborted(); + const reader = content.getReader(); + const abort = () => { + void reader.cancel().catch(() => {}); + }; + options.abortSignal?.addEventListener("abort", abort, { once: true }); + const chunks: Uint8Array[] = []; + let bytes = 0; + try { + for (;;) { + options.abortSignal?.throwIfAborted(); + const next = await reader.read(); + options.abortSignal?.throwIfAborted(); + if (next.done) break; + bytes += next.value.byteLength; + checkSize(bytes, "File write"); + if (next.value.byteLength) chunks.push(Buffer.from(next.value)); + } + await session.writeBinaryFile({ ...options, content: Buffer.concat(chunks) }); + } finally { + options.abortSignal?.removeEventListener("abort", abort); + void reader.cancel().catch(() => {}); + reader.releaseLock(); + } + }, + async removePath({ path, recursive = false, force = false, abortSignal }) { + abortSignal?.throwIfAborted(); + const target = resolvePath(path); + await rejectSymlinkPath(target); + await rm(target, { recursive, force }); + }, + async setNetworkPolicy(policy) { + if (policy !== "allow-all") + throw new Error( + "The runner owns network policy; this backend cannot change the firewall or broker credentials", + ); + }, + }; + return { + name, + async prewarm({ seedFiles, bootstrap }) { + if (bootstrap) + throw new Error("Production sandbox bootstrap must not run in the task container"); + for (const file of seedFiles) { + const path = resolvePath(file.path); + const skills = skillsHome(); + if (!(path.startsWith(`${root}/`) || (skills && path.startsWith(`${skills}/`)))) + throw new Error("Unexpected seed path: use the task workspace or $HOME/.agents/skills"); + checkSize( + typeof file.content === "string" + ? Buffer.byteLength(file.content) + : file.content.byteLength, + "Seed file", + ); + } + await mkdir(root, { recursive: true }); + for (const file of seedFiles) { + await session.writeBinaryFile({ + path: file.path, + content: typeof file.content === "string" ? Buffer.from(file.content) : file.content, + }); + } + return { reused: false }; + }, + async create({ sessionKey }) { + await mkdir(root, { recursive: true }); + const liveSession = { ...session, id: sessionKey }; + return { + session: liveSession, + useSessionFn: async () => liveSession, + captureState: async () => ({ backendName: name, metadata: {}, sessionKey }), + stop, + shutdown: stop, + }; + }, + }; +} + +function bufferEncoding(encoding: string): BufferEncoding { + if (!Buffer.isEncoding(encoding)) throw new Error(`Unsupported encoding: ${encoding}`); + return encoding; +} diff --git a/packages/eve-bench/src/harnesses/e0/index.ts b/packages/eve-bench/src/harnesses/e0/index.ts new file mode 100644 index 0000000000..4e24b66b76 --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/index.ts @@ -0,0 +1,250 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import type { Harness, HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; +import { ensureNode, NODE_VERSION } from "../eve/index.ts"; +import { copySource, hashTree } from "./snapshot.ts"; + +const exec = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const REASONING = ["provider-default", "none", "minimal", "low", "medium", "high", "xhigh"]; + +export interface E0Options { + readonly agent: string; + readonly reasoning?: string; +} + +export function createE0Harness(options: E0Options): Harness { + if (!options.agent?.trim()) + throw new Error( + "--agent must point to the installed e0 application (containing agent/ and package.json)", + ); + if (options.reasoning && !REASONING.includes(options.reasoning)) + throw new Error(`Unsupported e0 reasoning: ${options.reasoning}`); + const agent = resolve(options.agent); + return { + name: "e0", + credentials: ["AI_GATEWAY_API_KEY"], + prepare: (ctx) => prepareE0({ ...options, agent }, ctx), + command: (ctx) => `exec '${ctx.installDir}/arch/node' '${ctx.installDir}/app/runner.mjs'`, + env: (ctx) => { + const env: Record = { + EVE_PROJECT_DIR: `${ctx.installDir}/app`, + EVE_INSTRUCTION_PATH: ctx.instructionPath, + EVE_AGENT_LOGS_DIR: ctx.logsDir, + EVE_BENCH_CONTAINER: "1", + EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, + E0_MODEL: ctx.model, + }; + if (options.reasoning) env.E0_REASONING = options.reasoning; + return env; + }, + }; +} + +export async function adaptE0(app: string): Promise { + const agent = join(app, "agent"); + // These integrations are outside the self-contained terminal workload. Authored + // coding behavior remains intact, including the extension's worker model. + for (const slot of ["sandbox", "channels", "schedules", "connections", "instrumentation"]) { + await rm(join(agent, slot), { recursive: true, force: true }); + await rm(join(agent, `${slot}.ts`), { force: true }); + } + await mkdir(join(agent, "lib"), { recursive: true }); + await cp(join(here, "container-backend.ts"), join(agent, "lib/eve-bench-backend.ts")); + await writeFile( + join(agent, "sandbox.ts"), + `import { defineSandbox } from "eve/sandbox";\nimport { containerBackend } from "./lib/eve-bench-backend.ts";\nexport default defineSandbox({ backend: () => containerBackend() });\n`, + ); + await mkdir(join(agent, "channels"), { recursive: true }); + await writeFile( + join(agent, "channels/eve.ts"), + `import { eveChannel } from "eve/channels/eve";\nimport { localDev } from "eve/channels/auth";\nexport default eveChannel({ auth: [localDev()] });\n`, + ); + await writeFile( + join(agent, "extensions/code.ts"), + `import code from "eve-code";\nexport default code({});\n`, + ); +} + +async function sourceIdentity(agent: string, extension: string) { + const pkg = JSON.parse(await readFile(join(agent, "package.json"), "utf8")) as { + dependencies: Record; + }; + const dependencies: Record = {}; + for (const name of Object.keys(pkg.dependencies).sort()) { + const installed = JSON.parse( + await readFile(join(agent, "node_modules", name, "package.json"), "utf8"), + ) as { version: string }; + dependencies[name] = installed.version; + } + const gitRoot = ( + await exec("git", ["rev-parse", "--show-toplevel"], { cwd: agent }) + ).stdout.trim(); + const gitSha = (await exec("git", ["rev-parse", "HEAD"], { cwd: agent })).stdout.trim(); + const scope = [ + relative(gitRoot, agent), + relative(gitRoot, extension), + "pnpm-lock.yaml", + "pnpm-workspace.yaml", + ]; + const conflicts = ( + await exec("git", ["diff", "--name-only", "--diff-filter=U", "--", ...scope], { cwd: gitRoot }) + ).stdout.trim(); + if (conflicts) throw new Error(`e0 source has unresolved merge conflicts:\n${conflicts}`); + const status = ( + await exec("git", ["status", "--porcelain", "--", ...scope], { cwd: gitRoot }) + ).stdout + .split("\n") + .filter((line) => !line.includes("/.eve-bench-")) + .join("\n"); + const metadata = createHash("sha256"); + for (const file of [ + join(agent, "package.json"), + join(agent, "tsconfig.json"), + join(extension, "package.json"), + join(extension, "tsconfig.json"), + join(gitRoot, "pnpm-lock.yaml"), + join(gitRoot, "pnpm-workspace.yaml"), + ]) { + metadata + .update(relative(gitRoot, file)) + .update("\0") + .update(await readFile(file)); + } + return { + gitSha, + dirty: status.trim().length > 0, + agentSha256: await hashTree(join(agent, "agent")), + extensionSha256: await hashTree(join(extension, "extension")), + metadataSha256: metadata.digest("hex"), + eveDistSha256: await hashTree(join(agent, "node_modules/eve/dist")), + dependencies, + }; +} + +async function prepareE0(options: E0Options, ctx: HarnessPrepareContext): Promise { + const agent = await realpath(options.agent); + let extension: string; + try { + extension = await realpath(join(agent, "node_modules/eve-code")); + await stat(join(extension, "extension/extension.ts")); + await stat(join(agent, "node_modules/.bin/eve")); + await stat(join(agent, "agent/extensions/code.ts")); + } catch { + throw new Error( + "e0 requires installed workspace dependencies and source-backed eve-code. Run pnpm install in internal-agents first.", + ); + } + const source = await sourceIdentity(agent, extension); + const adapter = createHash("sha256"); + for (const file of [ + "index.ts", + "snapshot.ts", + "container-backend.ts", + "../eve/index.ts", + "../eve/runner.mjs", + ]) + adapter.update(await readFile(join(here, file))); + const identity = { + source, + model: ctx.model, + reasoning: options.reasoning ?? "authored", + workerModel: "authored (not overridden)", + nodeVersion: NODE_VERSION, + adapterSha256: adapter.digest("hex"), + adaptation: + "Task-container sandbox; local eve channel; production extension configuration removed. No production channels, schedules, connections, instrumentation, sandbox bootstrap, or Connect configuration. Instructions, coding capabilities, tools, skills, and authored worker preserved.", + }; + const key = createHash("sha256").update(JSON.stringify(identity)).digest("hex"); + const cache = resolve(ctx.cacheDir); + const destination = join(cache, "e0", key); + const nodeDir = join(cache, `node-v${NODE_VERSION}`); + await Promise.all((["x64", "arm64"] as const).map((arch) => ensureNode(nodeDir, arch))); + const bundle = { + dir: destination, + arch: { x64: join(nodeDir, "x64"), arm64: join(nodeDir, "arm64") }, + provenance: { key, ...identity }, + }; + if (await exists(join(destination, "provenance.json"))) return bundle; + + // An adjacent disposable directory resolves the already-installed dependency + // graph without modifying source files, installing new versions, or adding links. + const scratch = await mkdtemp(join(agent, ".eve-bench-")); + const buildEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + CI: "true", + E0_MODEL: ctx.model, + }; + if (options.reasoning) buildEnv.E0_REASONING = options.reasoning; + try { + await copySource(join(agent, "agent"), join(scratch, "agent")); + for (const file of ["package.json", "tsconfig.json"]) + await cp(join(agent, file), join(scratch, file)); + const stagedExtension = join(scratch, "node_modules/eve-code"); + await mkdir(stagedExtension, { recursive: true }); + await copySource(join(extension, "extension"), join(stagedExtension, "extension")); + for (const file of ["package.json", "tsconfig.json"]) + await cp(join(extension, file), join(stagedExtension, file)); + const eve = join(agent, "node_modules/.bin/eve"); + await exec(eve, ["extension", "build"], { + cwd: stagedExtension, + env: buildEnv, + maxBuffer: 8 * 1024 * 1024, + }); + await adaptE0(scratch); + await exec(eve, ["build"], { cwd: scratch, env: buildEnv, maxBuffer: 16 * 1024 * 1024 }); + const after = await sourceIdentity(agent, extension); + if (JSON.stringify(after) !== JSON.stringify(source)) { + throw new Error( + "e0 source or dependencies changed during preparation; retry after edits settle", + ); + } + await mkdir(join(cache, "e0"), { recursive: true }); + const publishing = await mkdtemp(join(cache, "e0/.publish-")); + try { + await mkdir(join(publishing, "app")); + await cp(join(scratch, ".output"), join(publishing, "app/.output"), { + recursive: true, + dereference: true, + }); + await cp(join(here, "../eve/runner.mjs"), join(publishing, "app/runner.mjs")); + await writeFile( + join(publishing, "provenance.json"), + `${JSON.stringify(bundle.provenance, null, 2)}\n`, + ); + try { + await rename(publishing, destination); + } catch (error) { + if (!(await exists(join(destination, "provenance.json")))) throw error; + } + } finally { + await rm(publishing, { recursive: true, force: true }); + } + return bundle; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.test.ts b/packages/eve-bench/src/harnesses/e0/snapshot.test.ts new file mode 100644 index 0000000000..01174b85b4 --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/snapshot.test.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { copySource, hashTree, includeSource } from "./snapshot.ts"; + +test("source filter excludes generated and secret-bearing files", () => { + for (const path of [ + "node_modules", + ".git", + ".eve", + ".output", + ".next", + "dist", + ".env", + ".env.local", + ".eve-bench-work", + ]) { + assert.equal(includeSource(path), false, path); + } + assert.equal(includeSource("agent.ts"), true); +}); + +test("hashTree is content-sensitive and copySource preserves its identity", async (t) => { + const root = await mkdtemp(join(tmpdir(), "eve-bench-source-")); + t.after(() => rm(root, { recursive: true, force: true })); + const source = join(root, "source"); + const destination = join(root, "destination"); + await mkdir(join(source, "nested"), { recursive: true }); + await writeFile(join(source, "agent.ts"), "export default 1;\n"); + await writeFile(join(source, "nested/file.txt"), "one\n"); + const before = await hashTree(source); + await copySource(source, destination); + assert.equal(await hashTree(destination), before); + await writeFile(join(source, "nested/file.txt"), "two\n"); + assert.notEqual(await hashTree(source), before); +}); + +test("hashTree rejects symlinks", async (t) => { + const root = await mkdtemp(join(tmpdir(), "eve-bench-source-link-")); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, "target"), "data"); + await symlink(join(root, "target"), join(root, "link")); + await assert.rejects(hashTree(root), /does not support symlinks/); +}); diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.ts b/packages/eve-bench/src/harnesses/e0/snapshot.ts new file mode 100644 index 0000000000..dfddddec4e --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/snapshot.ts @@ -0,0 +1,53 @@ +import { createHash } from "node:crypto"; +import { cp, lstat, readdir, readFile } from "node:fs/promises"; +import { basename, join, relative } from "node:path"; + +const EXCLUDED = new Set(["node_modules", ".git", ".eve", ".output", ".next", "dist"]); + +export function includeSource(path: string): boolean { + const name = basename(path); + return !EXCLUDED.has(name) && !name.startsWith(".env") && !name.startsWith(".eve-bench-"); +} + +/** Source snapshots never follow links into credentials, build output, or another checkout. */ +export async function hashTree(root: string): Promise { + const hash = createHash("sha256"); + let count = 0; + let bytes = 0; + async function walk(dir: string, depth: number): Promise { + if (depth > 32) throw new Error("Source tree exceeds 32 directory levels"); + for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name), + )) { + if (!includeSource(entry.name)) continue; + const path = join(dir, entry.name); + if (++count > 20_000) throw new Error("Source tree exceeds 20000 entries"); + if (entry.isSymbolicLink()) + throw new Error(`Source snapshot does not support symlinks: ${path}`); + if (entry.isDirectory()) await walk(path, depth + 1); + else if (entry.isFile()) { + const info = await lstat(path); + bytes += info.size; + if (bytes > 128 * 1024 * 1024) throw new Error("Source tree exceeds 128 MiB"); + hash + .update(relative(root, path)) + .update("\0") + .update(String(info.mode & 0o777)) + .update("\0"); + hash.update(await readFile(path)).update("\0"); + } else throw new Error(`Source snapshot requires regular files: ${path}`); + } + } + await walk(root, 0); + return hash.digest("hex"); +} + +export async function copySource(source: string, destination: string): Promise { + const before = await hashTree(source); + await cp(source, destination, { recursive: true, filter: includeSource }); + if ((await hashTree(destination)) !== before || (await hashTree(source)) !== before) { + throw new Error( + "Source changed while preparing the benchmark snapshot; retry after edits settle", + ); + } +} diff --git a/packages/eve-bench/src/harnesses/eve/index.ts b/packages/eve-bench/src/harnesses/eve/index.ts index 55dcc4dabf..3bf856bff1 100644 --- a/packages/eve-bench/src/harnesses/eve/index.ts +++ b/packages/eve-bench/src/harnesses/eve/index.ts @@ -17,7 +17,7 @@ const execFileAsync = promisify(execFile); const here = dirname(fileURLToPath(import.meta.url)); const packageRoot = resolve(here, "../../.."); const repoRoot = resolve(packageRoot, "../.."); -const NODE_VERSION = "24.19.0"; +export const NODE_VERSION = "24.19.0"; const ARCHES = ["x64", "arm64"] as const; export type EveSource = @@ -32,6 +32,7 @@ export type EveSource = export function createEveHarness(source: EveSource): Harness { return { name: source.kind === "local" ? "eve@local" : `eve@${source.version}`, + credentials: ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"], prepare: (ctx) => prepare(source, ctx), command: (ctx) => `sh ${ctx.installDir}/run.sh`, env: (ctx: HarnessRunContext) => ({ @@ -163,7 +164,7 @@ async function packLocalEve(artifacts: string): Promise { return resolve(artifacts, packed.filename); } -async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { +export async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { const archDir = join(nodeDir, arch); const target = join(archDir, "node"); if (await exists(target)) return; diff --git a/packages/eve-bench/src/options.test.ts b/packages/eve-bench/src/options.test.ts new file mode 100644 index 0000000000..85fd0eaac4 --- /dev/null +++ b/packages/eve-bench/src/options.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import type { Task } from "./core/task.ts"; +import { jobDir, lock, selectNativeBuild, selectTasks } from "./options.ts"; + +function task(dir: string): Task { + return { + name: "example", + dir, + instruction: "test", + agentTimeoutMs: 1, + verifierTimeoutMs: 1, + buildTimeoutMs: 1, + environment: { + dockerImage: "registry.example/task:pinned", + dockerfileDir: join(dir, "environment"), + allowInternet: true, + }, + }; +} + +test("job, dataset, and cohort selections reject traversal and missing tasks", async () => { + assert.throws(() => jobDir("../../outside"), /Docker-safe/); + await assert.rejects(lock("../outside"), /Docker-safe/); + await assert.rejects(selectTasks({ cohort: "smoke", task: ["missing-task"] }), /tasks not in/); +}); + +test("native build replaces a prebuilt image only when a Dockerfile exists", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "eve-bench-native-")); + t.after(() => rm(dir, { recursive: true, force: true })); + await mkdir(join(dir, "environment")); + await writeFile(join(dir, "environment/Dockerfile"), "FROM scratch\n"); + const original = task(dir); + assert.equal( + (await selectNativeBuild([original], false))[0]?.environment.dockerImage, + original.environment.dockerImage, + ); + assert.equal((await selectNativeBuild([original], true))[0]?.environment.dockerImage, undefined); + await rm(join(dir, "environment/Dockerfile")); + await assert.rejects(selectNativeBuild([original], true), /has no Dockerfile/); +}); diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts index cf35a1a996..f48c537713 100644 --- a/packages/eve-bench/src/options.ts +++ b/packages/eve-bench/src/options.ts @@ -1,11 +1,15 @@ -import { readdir } from "node:fs/promises"; +import { access, readdir } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts"; import type { Harness } from "./core/harness.ts"; +import { assertSafeId } from "./core/id.ts"; +import type { Task } from "./core/task.ts"; import { createEveHarness } from "./harnesses/eve/index.ts"; import { createOracleHarness } from "./harnesses/oracle.ts"; +import { createE0Harness } from "./harnesses/e0/index.ts"; +import { createCliHarness } from "./harnesses/cli/index.ts"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -37,11 +41,13 @@ export async function selectTasks(input: { } if (input.cohort) { - const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); + const cohortName = assertSafeId(input.cohort, "job"); + const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${cohortName}.json`)); const entry = await lock(cohort.dataset); const tasks = input.task?.length ? cohort.tasks.filter((task) => input.task!.includes(task)) : cohort.tasks; + assertSelectedTasks(input.task, tasks, cohort.dataset); return { lock: entry, tasks, taskDirs: localTaskDirs }; } @@ -49,21 +55,16 @@ export async function selectTasks(input: { const tasks = input.task?.length ? entry.tasks.filter((task) => input.task!.includes(task)) : entry.tasks; - if (input.task?.length && tasks.length !== input.task.length) { - const missing = input.task.filter((task) => !tasks.includes(task)); - throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); - } + assertSelectedTasks(input.task, tasks, `${entry.name}@${entry.version}`); return { lock: entry, tasks, taskDirs: localTaskDirs }; } -export function jobDir(nameOrPath: string): string { - return nameOrPath.includes("/") - ? resolve(nameOrPath) - : join(paths.generatedRoot, "jobs", nameOrPath); +export function jobDir(name: string): string { + return join(paths.generatedRoot, "jobs", assertSafeId(name, "job")); } export async function lock(name: string): Promise { - return readDatasetLock(join(paths.datasetsRoot, `${name}.json`)); + return readDatasetLock(join(paths.datasetsRoot, `${assertSafeId(name, "job")}.json`)); } export async function allLocks(): Promise { @@ -76,7 +77,23 @@ export function defaultJobName(label: string): string { return `${stamp}-${label.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}`; } -export function selectHarness(name: string, eve: string): Harness { +export function selectHarness( + name: string, + eve: string, + options: { agent?: string; version?: string; baseUrl?: string; reasoning?: string } = {}, +): Harness { + if (name !== "e0" && options.agent) throw new Error("--agent is only supported by --harness e0"); + if (name === "e0") + return createE0Harness({ agent: options.agent ?? "", reasoning: options.reasoning }); + if (name === "pi" || name === "opencode" || name === "codex" || name === "hermes") { + return createCliHarness(name, { + version: options.version ?? "", + baseUrl: options.baseUrl, + reasoning: options.reasoning, + }); + } + if (options.version || options.baseUrl || options.reasoning) + throw new Error("--version, --base-url and --reasoning require a supporting harness"); if (name === "oracle") return createOracleHarness(); if (name === "eve") { return createEveHarness( @@ -86,20 +103,42 @@ export function selectHarness(name: string, eve: string): Harness { throw new Error(`unknown harness: ${name}`); } -export function forwardedEnv(): Record { +export async function selectNativeBuild(tasks: readonly Task[], enabled: boolean): Promise { + if (!enabled) return [...tasks]; + return Promise.all( + tasks.map(async (task) => { + try { + await access(join(task.environment.dockerfileDir, "Dockerfile")); + } catch { + throw new Error(`Task ${task.name} has no Dockerfile for --native-build`); + } + return { + ...task, + environment: { ...task.environment, dockerImage: undefined }, + }; + }), + ); +} + +export function forwardedEnv(names: readonly string[]): Record { const env: Record = {}; - for (const key of [ - "AI_GATEWAY_API_KEY", - "VERCEL_OIDC_TOKEN", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - ]) { + for (const key of names) { const value = process.env[key]; if (value) env[key] = value; } return env; } +function assertSelectedTasks( + requested: readonly string[] | undefined, + selected: readonly string[], + source: string, +): void { + if (!requested?.length || selected.length === requested.length) return; + const missing = requested.filter((task) => !selected.includes(task)); + throw new Error(`tasks not in ${source}: ${missing.join(", ")}`); +} + async function defaultDataset(): Promise { const locks = await allLocks(); if (locks.length !== 1) { diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts index 09f410af5f..72c24bffd0 100644 --- a/packages/eve-bench/src/tools/run_bench.ts +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -6,14 +6,26 @@ import { z } from "zod"; import { syncDataset, taskDirs } from "../core/dataset.ts"; import { runJob } from "../core/job.ts"; import { loadTask } from "../core/task.ts"; -import { defaultJobName, forwardedEnv, paths, selectHarness, selectTasks } from "../options.ts"; +import { + defaultJobName, + forwardedEnv, + paths, + selectHarness, + selectNativeBuild, + selectTasks, +} from "../options.ts"; export const inputSchema = z .object({ cohort: z.string().min(1).optional(), tasks: z.array(z.string().min(1)).min(1).optional(), model: z.string().min(1).optional(), - harness: z.enum(["eve", "oracle"]).default("eve"), + harness: z.enum(["eve", "e0", "oracle", "pi", "opencode", "codex", "hermes"]).default("eve"), + agent: z.string().min(1).optional(), + version: z.string().min(1).optional(), + baseUrl: z.string().url().optional(), + reasoning: z.string().min(1).optional(), + nativeBuild: z.boolean().default(false), eve: z.string().min(1).default("local"), attempts: z.number().int().positive().default(1), concurrency: z.number().int().positive().default(4), @@ -23,18 +35,25 @@ export const inputSchema = z if (input.harness !== "oracle" && !input.model) { ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); } + if (input.harness === "e0" && !input.agent) { + ctx.addIssue({ code: "custom", path: ["agent"], message: "agent is required for e0" }); + } + if (["pi", "opencode", "codex", "hermes"].includes(input.harness) && !input.version) { + ctx.addIssue({ code: "custom", path: ["version"], message: "version is required" }); + } }); export default defineTool({ description: "Run an eve benchmark job and return its result.", inputSchema, async execute(input, ctx) { - const harness = selectHarness(input.harness, input.eve); + const harness = selectHarness(input.harness, input.eve, input); const model = input.model ?? "none"; const selection = await selectTasks({ cohort: input.cohort, task: input.tasks }); const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); - const tasks = await Promise.all( - (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + const tasks = await selectNativeBuild( + await Promise.all((await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir))), + input.nativeBuild, ); const name = input.job ?? defaultJobName(`${harness.name}-${model}`); @@ -48,7 +67,7 @@ export default defineTool({ model, attempts: input.attempts, concurrency: input.concurrency, - forwardEnv: forwardedEnv(), + forwardEnv: forwardedEnv(harness.credentials ?? []), signal: ctx.abortSignal, }); }, diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts index 41515610e8..1e5933f587 100644 --- a/packages/eve-bench/src/tools/tools.test.ts +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -22,6 +22,19 @@ describe("run_bench input", () => { true, ); assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); + assert.equal( + runBenchInput.safeParse({ harness: "e0", agent: "/apps/e0", model: "openai/gpt-5.6-terra" }) + .success, + true, + ); + assert.equal( + runBenchInput.safeParse({ + harness: "pi", + version: "0.80.3", + model: "deepseek/deepseek-v4-pro", + }).success, + true, + ); }); it("rejects invalid inputs", () => { @@ -29,6 +42,14 @@ describe("run_bench input", () => { assert.equal(runBenchInput.safeParse({ harness: "other", model: "model" }).success, false); assert.equal(runBenchInput.safeParse({ harness: "oracle", attempts: 0 }).success, false); assert.equal(runBenchInput.safeParse({ harness: "oracle", tasks: "fix-git" }).success, false); + assert.equal( + runBenchInput.safeParse({ harness: "e0", model: "openai/gpt-5.6-terra" }).success, + false, + ); + assert.equal( + runBenchInput.safeParse({ harness: "codex", model: "openai/gpt-5.6-terra" }).success, + false, + ); }); }); diff --git a/packages/eve-bench/test/scenarios/cli-adapters.test.ts b/packages/eve-bench/test/scenarios/cli-adapters.test.ts new file mode 100644 index 0000000000..76f3d8fd9a --- /dev/null +++ b/packages/eve-bench/test/scenarios/cli-adapters.test.ts @@ -0,0 +1,311 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { createCliHarness } from "../../src/harnesses/cli/index.ts"; +import { shellQuote, type SupportedCliName } from "../../src/harnesses/cli/plan.ts"; +import type { Harness, HarnessBundle, HarnessRunContext } from "../../src/core/harness.ts"; + +const exec = promisify(execFile); +const sourceDir = fileURLToPath(new URL("../../src/harnesses/cli/", import.meta.url)); +const FAKE_KEY = 'model-free-fake-key-"abcdef'; + +async function temp(t: TestContext): Promise { + const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-cli-"))); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} + +const mockCli = `import { readFile } from 'node:fs/promises'; +const args = process.argv.slice(2); +if (args[0] === '--version') { + console.log('mock-cli 1.2.3'); +} else { + const name = JSON.parse(await readFile(new URL('./identity.json', import.meta.url), 'utf8')).name; + const path = name === 'pi' ? process.env.PI_CODING_AGENT_DIR + '/models.json' + : name === 'codex' ? process.env.CODEX_HOME + '/config.toml' + : process.env.HOME + '/.config/opencode/opencode.json'; + const config = await readFile(path, 'utf8'); + let input = ''; + for await (const chunk of process.stdin) input += chunk.toString(); + const prompt = name === 'pi' ? input : args.at(-1); + if (name === 'pi' && args.includes('--')) throw new Error('pi does not support a -- separator'); + console.log(JSON.stringify({ type: 'mock', args, prompt, cwd: process.cwd(), config, + env: { PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR, CODEX_HOME: process.env.CODEX_HOME, + OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, OPENCODE_FAKE_VCS: process.env.OPENCODE_FAKE_VCS }, + credentialMapped: process.env.AI_GATEWAY_API_KEY === ${JSON.stringify(FAKE_KEY)} && process.env.OPENAI_API_KEY === process.env.AI_GATEWAY_API_KEY, + unrelatedCredentialAbsent: !process.env.UNRELATED_TEST_API_KEY })); + const key = process.env.AI_GATEWAY_API_KEY; + process.stdout.write(key.slice(0, 12)); + await new Promise(r => setTimeout(r, 10)); + process.stdout.write(key.slice(12) + '\\n'); + process.stderr.write(JSON.stringify(key) + '\\n'); + if (prompt === '__opencode_error__') process.stdout.write(JSON.stringify({ type: 'error', error: 'mock failure' })); + if (prompt === '__exit_7__') process.exitCode = 7; + if (prompt === '__signal__') process.kill(process.pid, 'SIGTERM'); +} +`; + +async function mockInstalled(t: TestContext, name: SupportedCliName) { + const root = await temp(t); + const installDir = join(root, "install with 'quote"); + await mkdir(join(installDir, "arch"), { recursive: true }); + for (const file of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(sourceDir, file), join(installDir, file)); + await writeFile( + join(installDir, "arch/node"), + `#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`, + { mode: 0o755 }, + ); + await writeFile(join(installDir, "arch/mock.mjs"), mockCli); + await writeFile(join(installDir, "arch/identity.json"), JSON.stringify({ name })); + await writeFile(join(installDir, "arch/entry.json"), JSON.stringify({ entry: "mock.mjs" })); + const ctx = { + model: "deepseek/deepseek-v4", + installDir, + instructionPath: join(root, "instruction with 'quote.md"), + taskWorkdir: join(root, "task with space"), + logsDir: join(root, "logs"), + }; + await mkdir(ctx.taskWorkdir); + return { root, ctx, harness: createCliHarness(name, { version: "1.2.3", reasoning: "high" }) }; +} + +async function invoke( + harness: Harness, + ctx: HarnessRunContext, + instruction: string, + key: string | undefined = FAKE_KEY, +) { + await writeFile(ctx.instructionPath, instruction); + try { + const result = await exec("sh", ["-c", harness.command(ctx)], { + env: { + PATH: process.env.PATH, + AI_GATEWAY_API_KEY: key, + UNRELATED_TEST_API_KEY: "must-not-forward", + ...harness.env(ctx), + }, + maxBuffer: 4 * 1024 * 1024, + }); + return { code: 0, ...result }; + } catch (error) { + const failed = error as Error & { code: number; stdout: string; stderr: string }; + return { code: failed.code, stdout: failed.stdout, stderr: failed.stderr }; + } +} + +for (const name of ["pi", "opencode", "codex"] as const) { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6"]) { + test(`${name} mock binary receives native flags, full ${model}, safe config and runtime credentials`, async (t) => { + const { ctx, harness } = await mockInstalled(t, name); + ctx.model = model; + const prompt = "--untrusted 'quote'\n$(touch SHOULD_NOT_EXIST)"; + const result = await invoke(harness, ctx, prompt); + assert.equal(result.code, 0, result.stderr); + const log = await readFile(join(ctx.logsDir, `${name}.jsonl`), "utf8"); + const record = JSON.parse(log.split("\n")[0]!); + assert.equal(record.cwd, ctx.taskWorkdir); + assert.equal(record.prompt, prompt); + assert.equal(record.credentialMapped, true); + assert.equal(record.unrelatedCredentialAbsent, true); + assert.ok( + record.args.includes(model) || record.args.includes(`--model=eve-bench-gateway/${model}`), + ); + assert.match(record.config, /ai-gateway.vercel.sh/); + if (name === "pi") { + assert.equal(JSON.parse(record.config).providers["eve-bench-gateway"].models[0].id, model); + assert.match(record.config, /\$AI_GATEWAY_API_KEY/); + } else if (name === "opencode") { + assert.ok(JSON.parse(record.config).provider["eve-bench-gateway"].models[model]); + assert.equal(record.env.OPENCODE_FAKE_VCS, "git"); + } else { + assert.match(record.config, /wire_api = "responses"/); + assert.match(record.config, /env_key = "AI_GATEWAY_API_KEY"/); + } + for (const file of await readdir(ctx.logsDir)) { + const content = await readFile(join(ctx.logsDir, file), "utf8"); + assert.ok(!content.includes(FAKE_KEY)); + assert.ok(!content.includes(JSON.stringify(FAKE_KEY).slice(1, -1))); + } + assert.ok(!result.stdout.includes(FAKE_KEY)); + assert.ok(!result.stderr.includes(FAKE_KEY)); + assert.match(log, /\[REDACTED\]/); + assert.deepEqual(await readdir(ctx.taskWorkdir), []); + assert.ok(!(await readdir(ctx.installDir)).some((file) => file.startsWith(".cli-runtime-"))); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.installLocation, "host"); + assert.equal(metrics.containerInstallMs, 0); + assert.ok(metrics.setupMs >= 0); + assert.ok(metrics.agentMs > 0); + assert.equal(metrics.exitCode, 0); + }); + } +} + +test("native nonzero exits, signals, and OpenCode zero-exit error events fail the harness", async (t) => { + for (const [name, instruction, expected] of [ + ["pi", "__exit_7__", 7], + ["codex", "__signal__", 1], + ["opencode", "__opencode_error__", 1], + ] as const) { + const { ctx, harness } = await mockInstalled(t, name); + const result = await invoke(harness, ctx, instruction); + assert.equal(result.code, expected); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.exitCode, expected); + if (instruction === "__signal__") assert.equal(metrics.signal, "SIGTERM"); + if (instruction === "__opencode_error__") assert.match(metrics.failure, /error event/); + } +}); + +test("missing key, oversized prompt, and version mismatch fail before the agent starts", async (t) => { + const { ctx, harness } = await mockInstalled(t, "pi"); + const missing = await invoke(harness, ctx, "do not run", ""); + assert.equal(missing.code, 1); + assert.match(missing.stderr, /AI_GATEWAY_API_KEY must be forwarded/); + assert.deepEqual(await readdir(ctx.logsDir), ["cli-runtime.json"]); + const oversized = await invoke(harness, ctx, "x".repeat(1024 * 1024 + 1)); + assert.equal(oversized.code, 1); + assert.match(oversized.stderr, /exceeds 1 MiB/); + const wrong = await invoke(createCliHarness("pi", { version: "1.2.30" }), ctx, "do not run"); + assert.equal(wrong.code, 1); + assert.match(wrong.stderr, /failed its --version check/); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.agentMs, 0); +}); + +test("host prepare uses pinned pnpm installs, verifies archives, caches across models, and relocates", async (t) => { + const root = await temp(t); + const bin = join(root, "bin"); + await mkdir(bin); + const calls = join(root, "pnpm-calls.jsonl"); + const shebang = `#!${process.execPath}\n`; + await writeFile( + join(bin, "pnpm"), + shebang + + ` +import { readFile, writeFile, mkdir, appendFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +if (process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY || process.env.UNRELATED_TEST_API_KEY) process.exit(91); +if (args[0] === '--version') { console.log('11.21.0'); } +else { + await appendFile(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); + const manifest = JSON.parse(await readFile('package.json', 'utf8')); + const [pkg, version] = Object.entries(manifest.dependencies)[0]; + const dir = join(process.cwd(), 'node_modules', pkg); + const name = pkg.includes('pi-coding-agent') ? 'pi' : pkg === 'opencode-ai' ? 'opencode' : 'codex'; + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'package.json'), JSON.stringify({ version, bin: { [name]: 'cli.mjs' } })); + await writeFile(join(dir, 'cli.mjs'), ${JSON.stringify(mockCli)}); + await writeFile(join(dir, 'identity.json'), JSON.stringify({ name })); + await writeFile('pnpm-lock.yaml', 'lockfileVersion: 9.0\\n'); +} +`, + { mode: 0o755 }, + ); + await writeFile( + join(bin, "tar"), + shebang + + ` +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +await writeFile(join(args[args.indexOf('-C') + 1], 'node'), ${JSON.stringify(`#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`)}); +`, + { mode: 0o755 }, + ); + const originalPath = process.env.PATH; + const originalKey = process.env.AI_GATEWAY_API_KEY; + process.env.PATH = `${bin}:${originalPath}`; + process.env.AI_GATEWAY_API_KEY = FAKE_KEY; + t.after(() => { + process.env.PATH = originalPath; + if (originalKey === undefined) delete process.env.AI_GATEWAY_API_KEY; + else process.env.AI_GATEWAY_API_KEY = originalKey; + }); + const archive = Buffer.from("mock node archive"); + const hash = createHash("sha256").update(archive).digest("hex"); + let fetches = 0; + t.mock.method(globalThis, "fetch", async (url: string) => { + fetches++; + assert.match(url, /^https:\/\/nodejs.org\/dist\/v24\.19\.0\//); + return new Response( + url.endsWith("SHASUMS256.txt") + ? `${hash} node-v24.19.0-linux-x64.tar.xz\n${hash} node-v24.19.0-linux-arm64.tar.xz\n` + : archive, + ); + }); + const harness = createCliHarness("pi", { version: "1.2.3" }); + const cacheDir = join(root, "cache"); + const bundle = await harness.prepare({ model: "deepseek/deepseek-v4", cacheDir }); + assert.equal(fetches, 4); + const sameBundle = await harness.prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(sameBundle.dir, bundle.dir); + assert.equal(bundle.provenance?.model, "deepseek/deepseek-v4"); + assert.equal(sameBundle.provenance?.model, "openai/gpt-5.6"); + const differentConfig = await createCliHarness("pi", { + version: "1.2.3", + reasoning: "high", + baseUrl: "https://gateway.example.test/v1", + }).prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(differentConfig.dir, bundle.dir); + assert.equal(differentConfig.provenance?.reasoning, "high"); + assert.equal(differentConfig.provenance?.baseUrl, "https://gateway.example.test/v1"); + assert.equal(fetches, 4); + const installs = (await readFile(calls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + assert.equal(installs.length, 2); + for (const [i, args] of installs.entries()) { + assert.ok(args.includes("--ignore-scripts")); + assert.ok(args.includes("--ignore-workspace")); + assert.ok(args.includes("--os=linux")); + assert.ok(args.includes(`--cpu=${i === 0 ? "x64" : "arm64"}`)); + assert.ok(!args.some((arg) => arg.includes("latest"))); + } + assert.equal(bundle.provenance?.installLocation, "host"); + assert.equal(bundle.provenance?.version, "1.2.3"); + assert.equal(bundle.provenance?.containerInstallMs, 0); + assert.match(JSON.stringify(bundle.provenance?.lockSha256), /[a-f0-9]{64}/); + assert.ok(!JSON.stringify(bundle.provenance).includes(FAKE_KEY)); + for (const arch of ["x64", "arm64"] as const) await assertRelocated(bundle, harness, root, arch); + + // A failed build must not publish a cache entry or retain partial build trees. + t.mock.method(globalThis, "fetch", async () => new Response("bad checksum")); + await assert.rejects( + createCliHarness("codex", { version: "1.2.3" }).prepare({ model: "openai/gpt-5.6", cacheDir }), + /checksum is missing/, + ); + assert.ok(!(await readdir(join(cacheDir, "cli"))).some((name) => name.startsWith(".prepare-"))); +}); + +async function assertRelocated( + bundle: HarnessBundle, + harness: Harness, + root: string, + arch: "x64" | "arm64", +) { + const installDir = join(root, `relocated ${arch}`); + await cp(bundle.dir, installDir, { recursive: true }); + await cp(bundle.arch![arch], join(installDir, "arch"), { recursive: true }); + const ctx = { + model: "openai/gpt-5.6", + installDir, + taskWorkdir: root, + instructionPath: join(root, `${arch}-instruction.md`), + logsDir: join(root, `${arch}-logs`), + }; + const result = await invoke(harness, ctx, "mock relocation"); + assert.equal(result.code, 0, result.stderr); + assert.match(await readFile(join(ctx.logsDir, "pi.jsonl"), "utf8"), /mock relocation/); + assert.equal(dirname(bundle.dir), dirname(bundle.arch![arch])); +} diff --git a/packages/eve-bench/test/scenarios/e0-container.test.ts b/packages/eve-bench/test/scenarios/e0-container.test.ts new file mode 100644 index 0000000000..28afd82ee1 --- /dev/null +++ b/packages/eve-bench/test/scenarios/e0-container.test.ts @@ -0,0 +1,539 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + lstat, + mkdir, + mkdtemp, + open, + readFile, + readdir, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import type { SandboxBackend, SandboxSession } from "eve/sandbox"; +import { containerBackend } from "../../src/harnesses/e0/container-backend.ts"; + +const MAX_BYTES = 16 * 1024 * 1024; +const exec = promisify(execFile); +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +const node = (script: string) => `${quote(process.execPath)} -e ${quote(script)}`; + +function env(t: TestContext, values: Record) { + const previous = Object.fromEntries(Object.keys(values).map((key) => [key, process.env[key]])); + const set = (entries: typeof values) => { + for (const [key, value] of Object.entries(entries)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }; + set(values); + t.after(() => set(previous)); +} + +async function fixture(t: TestContext, commandTimeoutMs = 3_000) { + const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-e0-"))); + const root = join(dir, "task with 'quote"); + const backend: SandboxBackend = containerBackend(root, { commandTimeoutMs }); + const input = { sessionKey: `test:${root}`, templateKey: null, runtimeContext: { appRoot: dir } }; + const handle = await backend.create(input); + t.after(async () => { + await handle.shutdown(); + await rm(dir, { recursive: true, force: true }); + }); + return { dir, root, backend, handle, session: handle.session, input }; +} + +async function collect(stream: ReadableStream) { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); +} + +async function ready(root: string) { + for (let i = 0; i < 100; i++) { + try { + return Number(await readFile(join(root, "descendant.pid"), "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await delay(10); + } + throw new Error("Descendant did not start"); +} + +async function stopped(pid: number) { + for (let i = 0; i < 100; i++) { + try { + const { stdout } = await exec("ps", ["-o", "stat=", "-p", String(pid)]); + // Linux init may reap a killed orphan later; a zombie cannot execute work. + if (stdout.trim().startsWith("Z") || !stdout.trim()) return; + } catch (error) { + if ((error as { code?: number }).code === 1) return; + throw error; + } + await delay(10); + } + assert.fail(`Process ${pid} is still running`); +} + +const descendant = "(sleep 0.5; printf leaked > escaped) & echo $! > descendant.pid; wait"; + +test("default root requires workdir and container guard; explicit roots support local tests", async (t) => { + const { root } = await fixture(t); + env(t, { EVE_BENCH_TASK_WORKDIR: undefined, EVE_BENCH_CONTAINER: "1" }); + assert.throws(() => containerBackend(), /EVE_BENCH_TASK_WORKDIR/); + process.env.EVE_BENCH_TASK_WORKDIR = root; + delete process.env.EVE_BENCH_CONTAINER; + assert.throws(() => containerBackend(), /EVE_BENCH_CONTAINER=1/); + assert.throws(() => containerBackend(undefined, {}), /EVE_BENCH_CONTAINER=1/); + assert.doesNotThrow(() => containerBackend(root)); + process.env.EVE_BENCH_CONTAINER = "1"; + assert.doesNotThrow(() => containerBackend()); + for (const invalid of ["", "relative"]) + assert.throws(() => containerBackend(invalid), /absolute task directory/); + for (const commandTimeoutMs of [0, -1, NaN, Infinity, 2_147_483_648]) + assert.throws(() => containerBackend(root, { commandTimeoutMs }), /commandTimeoutMs/); +}); + +test("workspace paths, cwd, actual roots and skill home work without symlinks or shell rewriting", async (t) => { + const { session, backend, dir, root } = await fixture(t); + const home = join(dir, "home"); + env(t, { HOME: home }); + await mkdir(home); + const prewarm = { + templateKey: "test", + runtimeContext: { appRoot: dir }, + seedFiles: [ + { path: "/workspace/nested/seed.txt", content: "seed\n" }, + { path: "$HOME/.agents/skills/test/SKILL.md", content: "skill\n" }, + { path: join(home, ".agents/skills/test/reference.md"), content: Buffer.from("reference\n") }, + ], + }; + assert.deepEqual(await backend.prewarm(prewarm), { reused: false }); + assert.equal(session.resolvePath("/workspace"), root); + assert.equal(session.resolvePath("/workspace/nested"), join(root, "nested")); + assert.equal(session.resolvePath("nested/seed.txt"), join(root, "nested/seed.txt")); + for (const escaped of [ + "/workspace-other", + "/proc/self/environ", + "/installed-agent", + "../../outside", + ]) { + assert.throws(() => session.resolvePath(escaped), /must stay inside/); + await assert.rejects(session.readTextFile({ path: escaped }), /must stay inside/); + } + assert.equal(await session.readTextFile({ path: "/workspace/nested/seed.txt" }), "seed\n"); + await symlink("/proc", join(root, "proc-link")); + await assert.rejects( + session.readTextFile({ path: "/workspace/proc-link/self/environ" }), + /do not follow symbolic links/, + ); + await rm(join(root, "proc-link")); + assert.equal( + await session.readTextFile({ path: "$HOME/.agents/skills/test/SKILL.md" }), + "skill\n", + ); + const shell = await session.run({ + command: 'printf "%s\\n" "$PWD"; cat seed.txt; cat "$HOME/.agents/skills/test/reference.md"', + workingDirectory: "/workspace/nested", + }); + assert.equal(shell.exitCode, 0, shell.stderr); + assert.equal(shell.stdout, `${root}/nested\nseed\nreference\n`); + const literal = await session.run({ command: "printf '%s' '/workspace/literal'" }); + assert.equal(literal.stdout, "/workspace/literal"); + const description = (session as SandboxSession & { description: string }).description; + assert.ok(description.includes(JSON.stringify(root))); + assert.match(description, /shell command text does not/); + assert.equal((await lstat(root)).isSymbolicLink(), false); + assert.deepEqual(await readdir(root), ["nested"]); + await assert.rejects( + backend.prewarm({ ...prewarm, bootstrap: () => assert.fail("must not bootstrap") }), + /Production sandbox bootstrap/, + ); + await assert.rejects( + backend.prewarm({ ...prewarm, seedFiles: [{ path: join(dir, "unexpected"), content: "no" }] }), + /must stay inside/, + ); +}); + +test("shell inherits only operational env, not provider/production secrets or startup hooks", async (t) => { + const { session, dir } = await fixture(t); + const startup = join(dir, "startup.sh"); + await writeFile(startup, "export INJECTED_SECRET=should-not-run\n"); + env(t, { + AI_GATEWAY_API_KEY: "fake-provider", + OPENAI_API_KEY: "fake-openai", + ANTHROPIC_API_KEY: "fake-anthropic", + D0_JUDGE_API_KEY: "fake-judge", + VERCEL_TOKEN: "fake-vercel", + AWS_SECRET_ACCESS_KEY: "fake-aws", + DATABASE_URL: "fake-database", + CUSTOM_PRODUCTION_VALUE: "fake-custom", + GITHUB_TOKEN: "fake-github", + BASH_ENV: startup, + ENV: startup, + NODE_OPTIONS: "--trace-warnings", + LANG: "C", + }); + const result = await session.run({ + command: node("console.log(JSON.stringify(process.env))"), + env: { EVE_CODE_DIAGNOSTICS_REQUEST: "explicit-tool-data" }, + }); + assert.equal(result.exitCode, 0, result.stderr); + const forwarded = JSON.parse(result.stdout) as Record; + for (const key of [ + "AI_GATEWAY_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "D0_JUDGE_API_KEY", + "VERCEL_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "DATABASE_URL", + "CUSTOM_PRODUCTION_VALUE", + "GITHUB_TOKEN", + "BASH_ENV", + "ENV", + "NODE_OPTIONS", + "INJECTED_SECRET", + ]) + assert.equal(forwarded[key], undefined, key); + assert.equal(forwarded.PATH, process.env.PATH); + assert.equal(forwarded.LANG, "C"); + assert.equal(forwarded.EVE_CODE_DIAGNOSTICS_REQUEST, "explicit-tool-data"); + const override = await session.run({ command: 'printf "%s" "$LANG"', env: { LANG: "explicit" } }); + assert.equal(override.stdout, "explicit"); +}); + +test("binary, text and streaming file operations preserve bytes, ranges, encodings and removal", async (t) => { + const { session } = await fixture(t); + for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) + assert.equal(await session[method]({ path: "missing" }), null); + await session.writeTextFile({ path: "/workspace/text", content: "one\n😀two\nthree\n" }); + assert.equal(await session.readTextFile({ path: "text", startLine: 2, endLine: 2 }), "😀two"); + assert.equal(await session.readTextFile({ path: "text", startLine: 3, endLine: 99 }), "three\n"); + await assert.rejects( + async () => session.readTextFile({ path: "text", startLine: 0 }), + /line range/, + ); + await assert.rejects( + async () => session.readTextFile({ path: "text", startLine: 3, endLine: 2 }), + /line range/, + ); + await session.writeTextFile({ path: "latin", content: "café", encoding: "latin1" }); + assert.equal(await session.readTextFile({ path: "latin", encoding: "latin1" }), "café"); + await assert.rejects(async () => session.readTextFile({ path: "latin" }), /encoded data/); + const bytes = Uint8Array.from([0, 255, 128, 1]); + await session.writeBinaryFile({ path: "binary", content: bytes }); + assert.deepEqual( + Buffer.from((await session.readBinaryFile({ path: "binary" }))!), + Buffer.from(bytes), + ); + const stream = await session.readFile({ path: "text" }); + assert.ok(stream); + await session.writeFile({ path: "copy/nested", content: stream }); + assert.equal(await session.readTextFile({ path: "copy/nested" }), "one\n😀two\nthree\n"); + await session.removePath({ path: "/workspace/copy", recursive: true }); + await session.removePath({ path: "missing", force: true }); + assert.equal(await session.readFile({ path: "copy/nested" }), null); +}); + +test("all file surfaces are bounded and oversized streamed writes leave existing files untouched", async (t) => { + const { session, root } = await fixture(t); + const file = await open(join(root, "large"), "w"); + await file.truncate(MAX_BYTES + 1); + await file.close(); + for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) + await assert.rejects( + async () => session[method]({ path: "large" }), + /File read exceeds 16 MiB/, + ); + await assert.rejects(async () => session.readBinaryFile({ path: "." }), /regular files/); + await session.writeTextFile({ path: "target", content: "unchanged" }); + await assert.rejects( + async () => session.writeBinaryFile({ path: "target", content: new Uint8Array(MAX_BYTES + 1) }), + /File write exceeds 16 MiB/, + ); + await assert.rejects( + async () => session.writeTextFile({ path: "target", content: "😀".repeat(MAX_BYTES / 4 + 1) }), + /File write exceeds 16 MiB/, + ); + let cancelled = false; + const content = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(1024 * 1024)); + }, + cancel() { + cancelled = true; + }, + }); + await assert.rejects( + async () => session.writeFile({ path: "target", content }), + /File write exceeds 16 MiB/, + ); + assert.equal(cancelled, true); + assert.equal(await session.readTextFile({ path: "target" }), "unchanged"); + await session.writeBinaryFile({ path: "boundary", content: new Uint8Array(MAX_BYTES) }); + assert.equal((await session.readBinaryFile({ path: "boundary" }))?.byteLength, MAX_BYTES); + assert.equal((await session.run({ command: "mkfifo pipe" })).exitCode, 0); + await assert.rejects(async () => session.readBinaryFile({ path: "pipe" }), /regular files/); + await assert.rejects( + async () => session.writeTextFile({ path: "pipe", content: "no" }), + /ENXIO|regular files/, + ); + await assert.rejects( + async () => session.writeTextFile({ path: "/dev/null", content: "no" }), + /must stay inside/, + ); +}); + +test("aborted file operations and a blocked stream reject without writing", async (t) => { + const { session } = await fixture(t); + const controller = new AbortController(); + const reason = new Error("cancel file"); + const content = new ReadableStream(); + const pending = session.writeFile({ path: "never", content, abortSignal: controller.signal }); + controller.abort(reason); + await assert.rejects( + async () => pending, + (error) => error === reason, + ); + await assert.rejects( + async () => session.readFile({ path: "never", abortSignal: controller.signal }), + (error) => error === reason, + ); + await assert.rejects( + async () => + session.writeBinaryFile({ + path: "never", + content: new Uint8Array(), + abortSignal: controller.signal, + }), + (error) => error === reason, + ); + await assert.rejects( + session.removePath({ path: "never", abortSignal: controller.signal }), + (error) => error === reason, + ); + assert.equal(await session.readFile({ path: "never" }), null); +}); + +test("run and spawn preserve split UTF-8, stderr and native nonzero exits", async (t) => { + const { session } = await fixture(t); + const command = node( + "process.stdout.write(Buffer.from([0xf0, 0x9f])); setTimeout(() => { process.stdout.write(Buffer.from([0x98, 0x80])); process.stderr.write('err'); process.exitCode = 7; }, 20)", + ); + assert.deepEqual(await session.run({ command }), { stdout: "😀", stderr: "err", exitCode: 7 }); + const child = await session.spawn({ command }); + const [stdout, stderr, result] = await Promise.all([ + collect(child.stdout), + collect(child.stderr), + child.wait(), + ]); + assert.deepEqual({ stdout, stderr, ...result }, { stdout: "😀", stderr: "err", exitCode: 7 }); + await child.kill(); + await child.kill(); + await assert.rejects( + async () => session.run({ command: "true", workingDirectory: "absent" }), + /ENOENT/, + ); +}); + +for (const method of ["run", "spawn"] as const) { + test(`${method} timeout kills the entire process group`, async (t) => { + const { session, root } = await fixture(t, 200); + const operation = session[method]({ command: descendant }); + const pid = await ready(root); + if (method === "run") { + const result = (await operation) as Awaited>; + assert.equal(result.exitCode, 124); + assert.match(result.stderr, /process group was stopped/); + } else { + const child = (await operation) as Awaited>; + assert.equal((await child.wait()).exitCode, 124); + } + await stopped(pid); + await delay(550); + assert.equal(await session.readFile({ path: "escaped" }), null); + }); + test(`${method} abort kills descendants and rejects with the abort reason`, async (t) => { + const { session, root } = await fixture(t); + const controller = new AbortController(); + const reason = new Error("cancel command"); + const operation = session[method]({ command: descendant, abortSignal: controller.signal }); + const completion = + method === "run" + ? operation + : Promise.resolve(operation).then((child) => + (child as Awaited>).wait(), + ); + const rejection = assert.rejects( + async () => completion, + (error) => error === reason, + ); + const pid = await ready(root); + controller.abort(reason); + await rejection; + await stopped(pid); + await assert.rejects( + async () => session[method]({ command: "touch never", abortSignal: controller.signal }), + (error) => error === reason, + ); + assert.equal(await session.readFile({ path: "never" }), null); + }); + test(`${method} bounds combined stdout/stderr even when spawn output is not consumed`, async (t) => { + const { session } = await fixture(t); + const command = node( + "const chunk = Buffer.alloc(1024 * 1024, 'x'); setInterval(() => { process.stdout.write(chunk); process.stderr.write(chunk); }, 1)", + ); + if (method === "run") + await assert.rejects(async () => session.run({ command }), /Command output exceeds 16 MiB/); + else { + const child = await session.spawn({ command }); + await assert.rejects(async () => child.wait(), /Command output exceeds 16 MiB/); + await stopped(child.pid!); + } + }); +} + +test("stop/shutdown are idempotent, retain task files and allow reattachment; shell exit kills background work", async (t) => { + const { session, backend, handle, input, root } = await fixture(t); + await session.writeTextFile({ path: "keep", content: "task artifact" }); + for (const stop of [handle.stop, handle.shutdown]) { + const child = await session.spawn({ command: descendant }); + const pid = await ready(root); + await stop(); + await stop(); + await stopped(pid); + assert.equal((await child.wait()).exitCode, 137); + await session.removePath({ path: "descendant.pid" }); + } + assert.deepEqual(await handle.captureState(), { + backendName: backend.name, + metadata: {}, + sessionKey: input.sessionKey, + }); + const reopened = await backend.create(input); + assert.equal(reopened.session.id, session.id); + assert.equal((await reopened.useSessionFn()).id, session.id); + assert.equal(await reopened.session.readTextFile({ path: "keep" }), "task artifact"); + await session.run({ + command: "(sleep 0.5; printf leaked > escaped) >/dev/null 2>&1 & echo $! > descendant.pid", + }); + await stopped(await ready(root)); + await session.setNetworkPolicy("allow-all"); + await assert.rejects(session.setNetworkPolicy("deny-all"), /runner owns network policy/); +}); + +test("installed eve built-in bash/file tools work with workspace, actual root and home paths", async (t) => { + const { session, dir, root } = await fixture(t); + env(t, { HOME: join(dir, "home") }); + const installed = new URL("../../node_modules/eve/dist/src/", import.meta.url); + const { ContextContainer, contextStorage } = await import( + new URL("context/container.js", installed).href + ); + const { executeBashOnSandbox } = await import( + new URL("execution/sandbox/bash-tool.js", installed).href + ); + const { executeReadFileOnSandbox } = await import( + new URL("execution/sandbox/read-file-tool.js", installed).href + ); + const { executeWriteFileOnSandbox } = await import( + new URL("execution/sandbox/write-file-tool.js", installed).href + ); + const { executeGlobOnSandbox } = await import( + new URL("execution/sandbox/glob-tool.js", installed).href + ); + const { executeGrepOnSandbox } = await import( + new URL("execution/sandbox/grep-tool.js", installed).href + ); + await contextStorage.run(new ContextContainer(), async () => { + for (const filePath of ["/workspace/builtin.txt", "$HOME/.agents/skills/example/SKILL.md"]) { + assert.equal( + (await executeWriteFileOnSandbox(session, { filePath, content: "first\n" })).existed, + false, + ); + assert.equal((await executeReadFileOnSandbox(session, { filePath })).content, "1: first"); + assert.equal( + (await executeWriteFileOnSandbox(session, { filePath, content: "second\n" })).existed, + true, + ); + } + const result = await executeBashOnSandbox(session, { command: "cat builtin.txt" }); + assert.equal(result.exitCode, 0); + assert.equal(result.stdout, "second\n"); + const glob = await executeGlobOnSandbox(session, { path: root, pattern: "*.txt" }); + assert.equal(glob.count, 1); + assert.equal(glob.content, join(root, "builtin.txt")); + const grep = await executeGrepOnSandbox(session, { path: root, pattern: "second" }); + assert.equal(grep.matchCount, 1); + assert.ok(grep.content.includes("builtin.txt:1:second")); + }); +}); + +// Optional real extension check: supply the eve-code package path from the e0 +// source checkout. Run on Linux (GNU realpath/stat), with no model or network. +test( + "real e0 code__apply_patch adds, updates, moves and deletes files in the actual task root", + { + skip: !process.env.EVE_BENCH_EVE_CODE_DIR + ? "Set EVE_BENCH_EVE_CODE_DIR to the eve-code package" + : false, + }, + async (t) => { + const { session, root } = await fixture(t); + const { default: tool } = await import( + pathToFileURL(join(process.env.EVE_BENCH_EVE_CODE_DIR!, "extension/tools/apply_patch.ts")) + .href + ); + const initialized = await session.run({ command: "git init -q" }); + assert.equal(initialized.exitCode, 0, initialized.stderr); + const ctx = { getSandbox: async () => session }; + const added = await tool.execute( + { root, patchText: "*** Begin Patch\n*** Add File: hello.txt\n+before\n*** End Patch" }, + ctx, + ); + assert.equal(added.files[0].operation, "add"); + assert.deepEqual(added.diagnostics, []); + const updated = await tool.execute( + { + root, + patchText: + "*** Begin Patch\n*** Update File: hello.txt\n@@\n-before\n+after\n*** End Patch", + }, + ctx, + ); + assert.equal(updated.files[0].operation, "update"); + const moved = await tool.execute( + { + root, + patchText: + "*** Begin Patch\n*** Update File: hello.txt\n*** Move to: nested/moved.txt\n*** End Patch", + }, + ctx, + ); + assert.equal(moved.files[0].operation, "move"); + assert.equal(await session.readTextFile({ path: "/workspace/nested/moved.txt" }), "after\n"); + assert.equal(await session.readTextFile({ path: "hello.txt" }), null); + const deleted = await tool.execute( + { root, patchText: "*** Begin Patch\n*** Delete File: nested/moved.txt\n*** End Patch" }, + ctx, + ); + assert.equal(deleted.files[0].operation, "delete"); + assert.equal(await session.readTextFile({ path: "nested/moved.txt" }), null); + await assert.rejects( + tool.execute( + { root, patchText: "*** Begin Patch\n*** Add File: ../escape\n+no\n*** End Patch" }, + ctx, + ), + /invalid patch path/, + ); + }, +); diff --git a/packages/eve-bench/test/scenarios/job-resume.test.ts b/packages/eve-bench/test/scenarios/job-resume.test.ts new file mode 100644 index 0000000000..42fd9a8e44 --- /dev/null +++ b/packages/eve-bench/test/scenarios/job-resume.test.ts @@ -0,0 +1,366 @@ +import assert from "node:assert/strict"; +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + symlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import { runJob, type JobInput } from "../../src/core/job.ts"; +import { hashTask, identityHash } from "../../src/core/job-identity.ts"; +import { loadTask } from "../../src/core/task.ts"; +import type { TrialResult } from "../../src/core/result.ts"; + +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), "eve-bench-resume-")); + const bin = join(root, "bin"); + const calls = join(root, "docker-calls.jsonl"); + const containers = join(root, "containers.json"); + const taskDir = join(root, "tasks", "a"); + await mkdir(bin); + for (const dir of ["environment", "tests", "solution"]) + await mkdir(join(taskDir, dir), { recursive: true }); + await writeFile(join(taskDir, "instruction.md"), "Do the task.\n"); + await writeFile(join(taskDir, "task.toml"), '[task]\nname = "a"\n'); + await writeFile(join(taskDir, "environment", "Dockerfile"), "FROM scratch\n"); + await writeFile(join(taskDir, "environment", "input.bin"), Buffer.from([0, 255, 1])); + await writeFile(join(taskDir, "tests", "test.sh"), "true\n"); + await writeFile(join(taskDir, "solution", "solve.sh"), "true\n"); + await writeFile(calls, ""); + await writeFile(containers, "[]"); + // No daemon or model is used: every Docker invocation terminates in this local stub. + await writeFile( + join(bin, "docker"), + `#!${process.execPath} +import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; +const args = process.argv.slice(2); +appendFileSync(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); +if (args[0] === 'version') console.log('fake-docker'); +else if (args[0] === 'ps') for (const row of JSON.parse(readFileSync(${JSON.stringify(containers)}, 'utf8'))) console.log(JSON.stringify(row)); +else if (args[0] === 'rm') writeFileSync(${JSON.stringify(containers)}, '[]'); +else if (args[0] === 'exec') console.log('/workspace'); +`, + { mode: 0o755 }, + ); + const previousPath = process.env.PATH; + process.env.PATH = `${bin}:${previousPath}`; + t.after(async () => { + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + await rm(root, { recursive: true, force: true }); + }); + const input: JobInput = { + name: basename(root), + dir: join(root, "job"), + cacheDir: join(root, "cache"), + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a"], + }, + tasks: [await loadTask(taskDir)], + harness: { + name: "fake", + prepare: async () => ({ + dir: bin, + provenance: { key: "v1", source: { sha: "abc", dirty: false } }, + }), + command: () => "true", + env: () => ({}), + }, + model: "no-model", + attempts: 1, + concurrency: 1, + forwardEnv: {}, + signal: new AbortController().signal, + }; + const trial: TrialResult = { + task: "a", + attempt: 1, + harness: "fake", + model: "no-model", + reward: 1, + agent: { status: "completed", exitCode: 0, durationMs: 1 }, + verifier: { status: "completed", exitCode: 0, durationMs: 1 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + }; + async function initialize() { + await runJob({ ...input, signal: AbortSignal.abort() }); + await mkdir(join(input.dir, "a", "1"), { recursive: true }); + await writeFile(join(input.dir, "a", "1", "trial.json"), JSON.stringify(trial)); + await writeFile(calls, ""); + } + async function leftovers() { + await writeFile( + containers, + JSON.stringify([ + { + ID: "leftover", + Names: "leftover", + Labels: `eve-bench=1,eve-bench.job=${input.name}`, + CreatedAt: "yesterday", + }, + ]), + ); + } + return { root, input, taskDir, calls, containers, trial, initialize, leftovers }; +} + +async function snapshot(dir: string) { + return Promise.all( + ["job.json", "result.json", "a/1/trial.json"].map((file) => readFile(join(dir, file), "utf8")), + ); +} + +test("every resume identity mismatch rejects before metadata/results overwrite or Docker cleanup", async (t) => { + const f = await fixture(t); + await f.initialize(); + await f.leftovers(); + const before = await snapshot(f.input.dir); + const changes: Partial[] = [ + { model: "different" }, + { attempts: 2 }, + { harness: { ...f.input.harness, name: "other" } }, + { + harness: { + ...f.input.harness, + prepare: async () => ({ dir: f.root, provenance: { key: "v2" } }), + }, + }, + { tasks: [{ ...f.input.tasks[0]!, name: "renamed" }] }, + { tasks: [...f.input.tasks, { ...f.input.tasks[0]!, name: "added" }] }, + { tasks: [{ ...f.input.tasks[0]!, instruction: "Loaded instruction override" }] }, + { tasks: [{ ...f.input.tasks[0]!, agentTimeoutMs: 123 }] }, + ...(["name", "version", "gitUrl", "commit"] as const).map((key) => ({ + dataset: { ...f.input.dataset, [key]: "changed" }, + })), + ]; + await assert.rejects(runJob({ ...f.input, tasks: [] }), /requires at least one task/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + for (const change of changes) { + await assert.rejects( + runJob({ ...f.input, ...change }), + /resume identity differs.*new job name/, + ); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + } + for (const path of [ + "instruction.md", + "task.toml", + "tests/test.sh", + "environment/Dockerfile", + "environment/input.bin", + "solution/solve.sh", + ]) { + const file = join(f.taskDir, path); + const content = await readFile(file); + await writeFile(file, Buffer.concat([content, Buffer.from("changed")])); + await assert.rejects(runJob(f.input), /differs \(tasks\).*new job name/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + await writeFile(file, content); + } +}); + +test("matching resume keeps original metadata and cached trials, and then cleans leftovers", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + await f.leftovers(); + const seen: boolean[] = []; + const result = await runJob({ + ...f.input, + concurrency: 4, + forwardEnv: { FAKE_KEY: "not-identity-or-metadata" }, + harness: { + ...f.input.harness, + prepare: async () => ({ + dir: f.root, + provenance: { source: { dirty: false, sha: "abc" }, key: "v1" }, + }), + }, + onTrial: (_trial, resumed) => seen.push(resumed), + }); + assert.deepEqual(result.trials, [f.trial]); + assert.deepEqual(seen, [true]); + const after = await snapshot(f.input.dir); + assert.equal(after[0], before[0]); + assert.equal(after[2], before[2]); + assert.match(await readFile(f.calls, "utf8"), /\["rm","--force","--volumes","leftover"\]/); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["run"/); + assert.equal(await readFile(f.containers, "utf8"), "[]"); +}); + +test("legacy, malformed, unsupported and orphaned jobs refuse without preparation or Docker calls", async (t) => { + const f = await fixture(t); + await mkdir(f.input.dir); + let prepared = false; + const input = { + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + prepared = true; + throw new Error("must not prepare"); + }, + }, + }; + for (const text of [ + JSON.stringify({ job: f.input.name, harness: "fake" }), + "{invalid", + "null", + JSON.stringify({ identity: { version: 2 } }), + ]) { + await writeFile(join(input.dir, "job.json"), text); + await assert.rejects(runJob(input), /new job name/); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), text); + } + await rm(join(input.dir, "job.json")); + await writeFile(join(input.dir, "result.json"), "orphaned"); + await assert.rejects(runJob(input), /no job.json.*new job name/); + assert.equal(await readFile(join(input.dir, "result.json"), "utf8"), "orphaned"); + assert.equal(prepared, false); + assert.equal(await readFile(f.calls, "utf8"), ""); +}); + +test("task hashes are relocatable and ignore unselected trees and mtimes, but include paths and executable bits", async (t) => { + const f = await fixture(t); + const original = await hashTask(f.input.tasks[0]!); + const relocated = join(f.root, "relocated"); + await cp(f.taskDir, relocated, { recursive: true }); + const task = await loadTask(relocated); + assert.equal(await hashTask(task), original); + await utimes(join(relocated, "instruction.md"), new Date(0), new Date(0)); + await writeFile(join(f.root, "tasks", "unselected"), "does not affect a"); + assert.equal(await hashTask(task), original); + await rename( + join(relocated, "environment", "input.bin"), + join(relocated, "environment", "renamed.bin"), + ); + assert.notEqual(await hashTask(task), original); + await rename( + join(relocated, "environment", "renamed.bin"), + join(relocated, "environment", "input.bin"), + ); + await chmod(join(relocated, "solution", "solve.sh"), 0o755); + assert.notEqual(await hashTask(task), original); + await chmod(join(relocated, "solution", "solve.sh"), 0o644); + assert.equal(await hashTask(task), original); + await rm(join(relocated, "environment", "input.bin")); + assert.notEqual(await hashTask(task), original); +}); + +test("task hashing refuses symlinks and environments outside the selected tree", async (t) => { + const f = await fixture(t); + await symlink(f.calls, join(f.taskDir, "external")); + await assert.rejects(hashTask(f.input.tasks[0]!), /regular files\/directories/); + await rm(join(f.taskDir, "external")); + const task = f.input.tasks[0]!; + await assert.rejects( + hashTask({ ...task, environment: { ...task.environment, dockerfileDir: f.root } }), + /inside the selected task tree/, + ); +}); + +test("an aborted job retains its identity, releases the lock, and resumes the same remaining work", async (t) => { + const f = await fixture(t); + const controller = new AbortController(); + const input = { ...f.input, attempts: 2 }; + const partial = await runJob({ + ...input, + signal: controller.signal, + onTrial: () => controller.abort(), + }); + assert.equal(partial.trials.length, 1); + const metadata = await readFile(join(input.dir, "job.json"), "utf8"); + const seen: boolean[] = []; + const result = await runJob({ ...input, onTrial: (_trial, resumed) => seen.push(resumed) }); + assert.deepEqual(seen, [true, false]); + assert.equal(result.trials.length, 2); + assert.deepEqual(result.trials[0], partial.trials[0]); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), metadata); +}); + +test("same-job lock rejects concurrent runs even in another output directory, and releases on failure", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + let enter!: () => void; + let release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + const active = runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + enter(); + await released; + return f.input.harness.prepare({ model: f.input.model, cacheDir: f.input.cacheDir }); + }, + }, + }); + try { + await entered; + await assert.rejects(runJob(f.input), /is locked.*active run/); + const otherDir = join(f.root, "other-job-dir"); + await assert.rejects(runJob({ ...f.input, dir: otherDir }), /is locked/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + assert.ok(!(await readdir(f.root)).includes("other-job-dir")); + } finally { + release(); + await active; + } + await assert.rejects( + runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + throw new Error("prepare failed"); + }, + }, + }), + /prepare failed/, + ); + await runJob(f.input); +}); + +test("stale locks fail closed with explicit recovery, and unknown containers are not removed", async (t) => { + const f = await fixture(t); + const lock = join(tmpdir(), `eve-bench-job-${identityHash(f.input.name)}.lock`); + await writeFile(lock, "crashed owner", { flag: "wx" }); + try { + await assert.rejects(runJob(f.input), /confirm it has stopped before removing/); + assert.equal(await readFile(lock, "utf8"), "crashed owner"); + assert.equal(await readFile(f.calls, "utf8"), ""); + } finally { + await rm(lock); + } + await f.leftovers(); + const containers = await readFile(f.containers, "utf8"); + await assert.rejects(runJob(f.input), /containers but no resume identity.*new job name/); + assert.equal(await readFile(f.containers, "utf8"), containers); + assert.deepEqual(await readdir(f.input.dir), []); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["rm"/); +}); diff --git a/packages/eve-bench/test/stress/stress.test.ts b/packages/eve-bench/test/stress/stress.test.ts index 85977542bd..fee1b92a73 100644 --- a/packages/eve-bench/test/stress/stress.test.ts +++ b/packages/eve-bench/test/stress/stress.test.ts @@ -200,7 +200,7 @@ test( ); test( - "abort: exits 130, writes a partial result, cleans up, and resumes with a new task set", + "abort: exits 130, cleans up, and rejects a different resume identity", { timeout }, async () => { const job = "stress-abort"; @@ -228,11 +228,9 @@ test( await waitFor(async () => (await dockerNames(job)).length === 0, 15_000, "abort cleanup"); const resumed = await runCli(job, [taskDir("abort-resume")], [], false); - assertPassed(resumed); - const result = await readResult(job); - assert.equal(result.trials.length, 1); - assert.equal(result.trials[0]?.task, "stress-abort-resume"); - assert.equal(result.trials[0]?.reward, 1); + assert.equal(resumed.code, 1); + assert.match(resumed.stderr, /resume identity differs/); + assert.deepEqual(await dockerNames(job), []); } finally { if (child.exitCode === null) child.kill("SIGKILL"); await cleanupContainers(job); diff --git a/research/eve-bench-runner.md b/research/eve-bench-runner.md index e290609aae..ea48a163ad 100644 --- a/research/eve-bench-runner.md +++ b/research/eve-bench-runner.md @@ -1,7 +1,7 @@ --- issue: TODO status: approved -last_updated: "2026-09-09" +last_updated: "2026-09-14" --- # eve-bench: a zero-dependency benchmark runner @@ -99,6 +99,8 @@ an instruction. It is a spawned shell, nothing more: ```ts interface Harness { readonly name: string; + /** Host environment names allowed to cross into this harness process. */ + readonly credentials?: readonly string[]; /** Runs once per job on the host; returns a directory to place in the container. */ prepare(ctx: { model: string; cacheDir: string }): Promise<{ dir: string; provenance?: object }>; /** Optional per-task host paths uploaded into the install directory. */ @@ -126,8 +128,10 @@ and no bundler is involved. It writes `/logs/agent/events.ndjson` and An `oracle` harness runs each task's `solution/solve.sh` via `stage`. It validates the runner without a model and is the reward ceiling a real harness -is measured against. A harness wrapping another CLI agent is the same -interface and enables side-by-side comparison. +is measured against. The same contract supports a source snapshot of an e0 +application and explicitly versioned pi, OpenCode, and Codex CLI bundles. +Hermes remains unsupported until its Python environment and provider routing +can be pinned. ### Trial lifecycle @@ -150,7 +154,9 @@ removal is unconditional. Each trial writes to its own directory: container.log docker output for the trial ``` -Resume skips trial directories that already contain a `trial.json`. +Resume accepts existing trials only when the job identity matches the harness, +model, bundle provenance, dataset, task content, and attempt count. Malformed or +mismatched records fail closed instead of being overwritten. ### Result schema @@ -198,8 +204,10 @@ and write it; nothing else is stable. ``` eve-bench tasks sync fetch a pinned dataset eve-bench tasks list --cohort smoke print task names +eve-bench prepare --harness --model build a harness without model calls eve-bench run --cohort smoke --model zai/glm-5.2 [--attempts 3] [--concurrency 8] [--task fix-git ...] - [--harness eve|oracle] [--eve local|] [--job ] # same --job resumes + [--harness eve|e0|oracle|pi|opencode|codex] [--eve local|] + [--agent ] [--version ] [--native-build] [--job ] eve-bench report [--format console|json|junit] eve-bench diff per-task reward deltas ``` @@ -214,8 +222,8 @@ container only for the harness exec, never written to disk. directory as an artifact, and posts `eve-bench diff` against the base branch's artifact. - Humans: the same CLI with the console reporter. -- Agents: `src/tools/` exposes `run_trial`, `get_result`, and `diff` as eve - tools over `core`; they return the schema above. +- Agents: `src/tools/` exposes run, result, task-list, and diff tools over + `core`; they return the same schemas as the CLI. ## Invariants @@ -234,5 +242,4 @@ container only for the harness exec, never written to disk. - Reusing eve's `SandboxBackend` for the environment. The benchmark must not depend on the surface it measures. - Uploading results to a hosted dashboard. -- `src/tools/` (eve tools over `core`) and the CI workflow. Both consume the - result schema above and follow once the CLI has run a full dataset. +- A hosted results dashboard. Job directories remain local or CI artifacts.