From d449b72146b67a3db7f5c0fe1016667e69f97f57 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Wed, 9 Sep 2026 14:11:33 -0400 Subject: [PATCH 1/9] feat(eve-bench): replace Harbor with a zero-dependency benchmark runner Terminal-Bench evaluation of the eve harness no longer goes through Harbor and a Python adapter. packages/eve-bench now owns a small Node runner (task loader, docker lifecycle, trial/job scheduler with resume, reporters), an eve harness whose in-container runner speaks plain HTTP to the built server, an oracle harness for reference solutions, pinned dataset locks and cohorts, eve tools over the same core, and a CI workflow. Design in research/eve-bench-runner.md. Signed-off-by: Rui Conti --- .github/workflows/bench.yml | 157 +++++++++++ .gitignore | 3 + packages/eve-bench/README.md | 105 ++++++++ packages/eve-bench/agent/agent.ts | 10 + packages/eve-bench/agent/instructions.md | 11 + packages/eve-bench/agent/task-path.ts | 13 + .../eve-bench/agent/tools/ask_question.ts | 3 + packages/eve-bench/agent/tools/bash.ts | 37 +++ packages/eve-bench/agent/tools/glob.ts | 22 ++ packages/eve-bench/agent/tools/grep.ts | 45 ++++ packages/eve-bench/agent/tools/read_file.ts | 27 ++ packages/eve-bench/agent/tools/todo.ts | 3 + packages/eve-bench/agent/tools/web_fetch.ts | 3 + packages/eve-bench/agent/tools/web_search.ts | 3 + packages/eve-bench/agent/tools/write_file.ts | 24 ++ .../eve-bench/datasets/cohorts/smoke.json | 11 + .../datasets/terminal-bench-2.0.json | 97 +++++++ packages/eve-bench/package.json | 26 ++ packages/eve-bench/src/cli.ts | 175 +++++++++++++ packages/eve-bench/src/core/dataset.test.ts | 103 ++++++++ packages/eve-bench/src/core/dataset.ts | 77 ++++++ packages/eve-bench/src/core/docker.ts | 177 +++++++++++++ packages/eve-bench/src/core/harness.ts | 34 +++ packages/eve-bench/src/core/job.ts | 108 ++++++++ packages/eve-bench/src/core/report.test.ts | 63 +++++ packages/eve-bench/src/core/report.ts | 118 +++++++++ packages/eve-bench/src/core/result.ts | 52 ++++ packages/eve-bench/src/core/task.test.ts | 156 +++++++++++ packages/eve-bench/src/core/task.ts | 82 ++++++ packages/eve-bench/src/core/toml.test.ts | 68 +++++ packages/eve-bench/src/core/toml.ts | 159 +++++++++++ packages/eve-bench/src/core/trial.ts | 188 +++++++++++++ packages/eve-bench/src/harnesses/eve/index.ts | 247 ++++++++++++++++++ .../eve-bench/src/harnesses/eve/runner.mjs | 242 +++++++++++++++++ packages/eve-bench/src/harnesses/oracle.ts | 23 ++ packages/eve-bench/src/options.ts | 98 +++++++ packages/eve-bench/src/tools/diff_bench.ts | 25 ++ .../eve-bench/src/tools/get_bench_result.ts | 24 ++ .../eve-bench/src/tools/list_bench_tasks.ts | 19 ++ packages/eve-bench/src/tools/run_bench.ts | 55 ++++ packages/eve-bench/src/tools/tools.test.ts | 71 +++++ packages/eve-bench/tsconfig.json | 12 + pnpm-lock.yaml | 16 ++ research/eve-bench-runner.md | 238 +++++++++++++++++ 44 files changed, 3230 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 packages/eve-bench/README.md create mode 100644 packages/eve-bench/agent/agent.ts create mode 100644 packages/eve-bench/agent/instructions.md create mode 100644 packages/eve-bench/agent/task-path.ts create mode 100644 packages/eve-bench/agent/tools/ask_question.ts create mode 100644 packages/eve-bench/agent/tools/bash.ts create mode 100644 packages/eve-bench/agent/tools/glob.ts create mode 100644 packages/eve-bench/agent/tools/grep.ts create mode 100644 packages/eve-bench/agent/tools/read_file.ts create mode 100644 packages/eve-bench/agent/tools/todo.ts create mode 100644 packages/eve-bench/agent/tools/web_fetch.ts create mode 100644 packages/eve-bench/agent/tools/web_search.ts create mode 100644 packages/eve-bench/agent/tools/write_file.ts create mode 100644 packages/eve-bench/datasets/cohorts/smoke.json create mode 100644 packages/eve-bench/datasets/terminal-bench-2.0.json create mode 100644 packages/eve-bench/package.json create mode 100644 packages/eve-bench/src/cli.ts create mode 100644 packages/eve-bench/src/core/dataset.test.ts create mode 100644 packages/eve-bench/src/core/dataset.ts create mode 100644 packages/eve-bench/src/core/docker.ts create mode 100644 packages/eve-bench/src/core/harness.ts create mode 100644 packages/eve-bench/src/core/job.ts create mode 100644 packages/eve-bench/src/core/report.test.ts create mode 100644 packages/eve-bench/src/core/report.ts create mode 100644 packages/eve-bench/src/core/result.ts create mode 100644 packages/eve-bench/src/core/task.test.ts create mode 100644 packages/eve-bench/src/core/task.ts create mode 100644 packages/eve-bench/src/core/toml.test.ts create mode 100644 packages/eve-bench/src/core/toml.ts create mode 100644 packages/eve-bench/src/core/trial.ts create mode 100644 packages/eve-bench/src/harnesses/eve/index.ts create mode 100644 packages/eve-bench/src/harnesses/eve/runner.mjs create mode 100644 packages/eve-bench/src/harnesses/oracle.ts create mode 100644 packages/eve-bench/src/options.ts create mode 100644 packages/eve-bench/src/tools/diff_bench.ts create mode 100644 packages/eve-bench/src/tools/get_bench_result.ts create mode 100644 packages/eve-bench/src/tools/list_bench_tasks.ts create mode 100644 packages/eve-bench/src/tools/run_bench.ts create mode 100644 packages/eve-bench/src/tools/tools.test.ts create mode 100644 packages/eve-bench/tsconfig.json create mode 100644 research/eve-bench-runner.md diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..a759cdf859 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,157 @@ +name: eve Benchmarks + +on: + pull_request: + paths: + - "packages/eve-bench/**" + - research/eve-bench-runner.md + workflow_dispatch: + inputs: + model: + description: Model to benchmark + required: false + default: zai/glm-5.2 + type: string + cohort: + description: Task cohort to run + required: false + default: smoke + type: string + base_job_artifact: + description: Optional artifact containing a base job to compare + required: false + type: string + +concurrency: + group: eve-bench-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + oracle-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + - name: Run tests + run: pnpm run test + + - name: Sync benchmark tasks + run: pnpm bench tasks sync --json + + - name: Run oracle smoke benchmark + run: pnpm bench run --harness oracle --cohort smoke --concurrency 5 --job oracle-smoke --format junit > junit.xml + + - name: Print oracle smoke report + run: pnpm bench report oracle-smoke + + - name: Upload oracle smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-oracle-smoke + path: | + packages/eve-bench/.generated/jobs/oracle-smoke + packages/eve-bench/junit.xml + include-hidden-files: true + if-no-files-found: warn + + - name: Require every oracle attempt to resolve + run: | + node -e ' + const result = require("./.generated/jobs/oracle-smoke/result.json"); + const { attempts, resolved } = result.summary; + if (resolved < attempts) { + console.error(`Oracle resolved ${resolved} of ${attempts} attempts`); + process.exit(1); + } + ' + + eve-smoke: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + defaults: + run: + working-directory: packages/eve-bench + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: ${{ github.workspace }} + + - name: Build eve package + run: pnpm --filter eve run build + working-directory: ${{ github.workspace }} + + - name: Run eve smoke benchmark + env: + MODEL: ${{ inputs.model }} + COHORT: ${{ inputs.cohort }} + run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke + + - name: Download base job artifact + if: ${{ inputs.base_job_artifact != '' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.base_job_artifact }} + path: packages/eve-bench/.generated/base-job + + - name: Compare with base job + if: ${{ inputs.base_job_artifact != '' }} + run: | + pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt + { + echo "## Benchmark diff" + echo '```text' + cat diff.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload eve smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eve-bench-eve-smoke + path: packages/eve-bench/.generated/jobs/eve-smoke + include-hidden-files: true + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 8b2830ffc5..adebbfbd91 100644 --- a/.gitignore +++ b/.gitignore @@ -13,12 +13,14 @@ coverage/ .pnpm-store/ .bun/ .npm/ +__pycache__/ # Test artifacts .vitest/ .extension-contracts-cache/ packages/eve/.workflow-vitest/ packages/eve/.generated/ +packages/eve-bench/.generated/ playwright-report/ test-results/ @@ -31,6 +33,7 @@ packages/eve-buzz-acp-adapter/NOTICE # Logs *.log +/jobs/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md new file mode 100644 index 0000000000..149054b23b --- /dev/null +++ b/packages/eve-bench/README.md @@ -0,0 +1,105 @@ +# @eve/bench + +Terminal-Bench harness benchmarks for eve, run by a zero-dependency Node runner. + +The package owns the harness under evaluation (`agent/`), pinned dataset locks +and task cohorts (`datasets/`), and the runner (`src/`). Tasks keep the +Terminal-Bench layout and reward contract, so results stay comparable with +published runs. See `research/eve-bench-runner.md` for the design. + +## Prerequisites + +- Node.js 24 and pnpm +- Docker +- model-provider credentials in the environment, such as `AI_GATEWAY_API_KEY` +- a built local eve package when using `--eve local`; run `pnpm build` from the + repository root first + +Everything below runs from this directory with `pnpm bench` (alias for +`node src/cli.ts`). + +## Run + +```bash +pnpm bench tasks sync # fetch pinned datasets into .generated/datasets +pnpm bench tasks list --cohort smoke # print a cohort's task names + +AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8 +pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model +pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release + +pnpm bench report --format junit --out junit.xml +pnpm bench diff +``` + +Every command accepts `--json` and never prompts. Jobs live in +`.generated/jobs//`; re-running with the same `--job` name resumes it, +skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight +trials and removes their containers. + +## How a trial runs + +1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). +2. Start one container, upload the harness bundle to `/installed-agent`, and + run the harness command under the task's agent timeout. +3. Upload `tests/` and run `tests/test.sh` under the verifier timeout. The + verifier always runs, so the reward reflects what the agent left behind. +4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` + to the trial directory, remove the container, and write `trial.json`. + +## Harnesses + +- `eve` (default): builds `agent/` against the local checkout (`--eve local`) + or an exact published version, then packages the Nitro output and a small + runner that drives the server over HTTP. Build eve from the repository root + with `pnpm build` before using `--eve local`. For cache lookup, `prepare()` + derives the local eve identity from `packages/eve/dist` instead of packing + first. The cache lives in `.generated/cache/`. Each trial uploads the bundle + and only the Linux Node binary matching the task container's architecture, + for about 128 MB total. + + The agent's tools operate on the task working directory through + `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass + eve's nested sandbox on purpose. + +- `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner + and as the reward ceiling. + +Adding a harness means implementing `src/core/harness.ts` (prepare a +directory, return a command and env) and registering it in `src/cli.ts`. + +## Agent tools + +`src/tools/` exposes `run_bench`, `get_bench_result`, `diff_bench`, and +`list_bench_tasks` as eve `defineTool` definitions over the same core used by +the CLI. Their results use the same `JobResult`, `TaskDelta`, and task-list JSON +as the CLI. Add a file to an agent's `tools/` directory that re-exports a +definition by relative path, or copy the definition there. + +## Datasets and cohorts + +`datasets/-.json` pins a git URL, commit, and the task paths in +it. `datasets/cohorts/.json` names a reviewed task subset and the reason +it exists. Compare base and candidate runs with the same cohort, model, +attempt count, and eve source. + +## CI + +`.github/workflows/bench.yml` runs the oracle smoke cohort on pull requests that +touch this package or its design document and requires every task attempt to +resolve. It uploads the job directory and `junit.xml` as artifacts. + +The `eve-smoke` job runs only through `workflow_dispatch`, accepts `model` and +`cohort` inputs, and requires `AI_GATEWAY_API_KEY`. It uploads its job directory +and can optionally diff the result against a downloaded base-job artifact. + +## Layout + +- `agent/`: harness behavior under evaluation (model binding, instructions, tools) +- `src/core/`: runner library with no dependency on eve or third parties +- `src/harnesses/`: `eve` and `oracle` +- `src/tools/`: eve tool definitions over the runner core +- `src/options.ts`: run options shared by the CLI and tools +- `src/cli.ts`: `eve-bench` CLI +- `datasets/`: locks and cohorts +- `.generated/`: datasets, bundle cache, and job output; never committed diff --git a/packages/eve-bench/agent/agent.ts b/packages/eve-bench/agent/agent.ts new file mode 100644 index 0000000000..595843cf27 --- /dev/null +++ b/packages/eve-bench/agent/agent.ts @@ -0,0 +1,10 @@ +import { defineAgent } from "eve"; + +const model = process.env.EVE_BENCH_MODEL; +if (model === undefined || model.length === 0) { + throw new Error("EVE_BENCH_MODEL must be set by the eve-bench harness."); +} + +export default defineAgent({ + model, +}); diff --git a/packages/eve-bench/agent/instructions.md b/packages/eve-bench/agent/instructions.md new file mode 100644 index 0000000000..75ea47d355 --- /dev/null +++ b/packages/eve-bench/agent/instructions.md @@ -0,0 +1,11 @@ +You are an autonomous terminal task agent. + +Work until the requested task is complete or conclusively cannot be completed. + +- Inspect relevant files before modifying them. +- Use the available terminal and file tools to work directly in the task environment. +- Make the smallest correct changes that satisfy the task. +- Run relevant validation after making changes. +- Do not ask the user questions. +- Do not use network access unless the task explicitly requires it. +- Finish with a concise description of the completed work and validation. diff --git a/packages/eve-bench/agent/task-path.ts b/packages/eve-bench/agent/task-path.ts new file mode 100644 index 0000000000..8303fe355f --- /dev/null +++ b/packages/eve-bench/agent/task-path.ts @@ -0,0 +1,13 @@ +import { isAbsolute, resolve } from "node:path"; + +export function taskRoot(): string { + const root = process.env.EVE_BENCH_TASK_WORKDIR; + if (root === undefined || root.length === 0) { + throw new Error("EVE_BENCH_TASK_WORKDIR must be set by the eve-bench harness."); + } + return root; +} + +export function resolveTaskPath(path = "."): string { + return isAbsolute(path) ? path : resolve(taskRoot(), path); +} diff --git a/packages/eve-bench/agent/tools/ask_question.ts b/packages/eve-bench/agent/tools/ask_question.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/ask_question.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/bash.ts b/packages/eve-bench/agent/tools/bash.ts new file mode 100644 index 0000000000..2d8121c64d --- /dev/null +++ b/packages/eve-bench/agent/tools/bash.ts @@ -0,0 +1,37 @@ +import { exec } from "node:child_process"; +import { promisify } from "node:util"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { taskRoot } from "../task-path.js"; + +const execAsync = promisify(exec); + +export default defineTool({ + approval: never(), + description: + "Execute a shell command in the task environment. Commands start in the task working directory.", + inputSchema: z.object({ + command: z.string().describe("The shell command to execute."), + }), + async execute({ command }, ctx) { + try { + const { stdout, stderr } = await execAsync(command, { + cwd: taskRoot(), + env: process.env, + maxBuffer: 10 * 1024 * 1024, + signal: ctx.abortSignal, + }); + return { exitCode: 0, stderr, stdout }; + } catch (error) { + const result = error as Error & { code?: number; stderr?: string; stdout?: string }; + return { + exitCode: typeof result.code === "number" ? result.code : 1, + stderr: result.stderr ?? result.message, + stdout: result.stdout ?? "", + }; + } + }, +}); diff --git a/packages/eve-bench/agent/tools/glob.ts b/packages/eve-bench/agent/tools/glob.ts new file mode 100644 index 0000000000..764ac99ab9 --- /dev/null +++ b/packages/eve-bench/agent/tools/glob.ts @@ -0,0 +1,22 @@ +import { glob } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskRoot } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: "Find task-environment files whose paths match a glob pattern.", + inputSchema: z.object({ + pattern: z.string(), + path: z.string().optional().describe("Search root relative to the task working directory."), + }), + async execute({ pattern, path }) { + const cwd = path === undefined ? taskRoot() : resolveTaskPath(path); + const matches: string[] = []; + for await (const match of glob(pattern, { cwd })) matches.push(match); + return { matches: matches.sort() }; + }, +}); diff --git a/packages/eve-bench/agent/tools/grep.ts b/packages/eve-bench/agent/tools/grep.ts new file mode 100644 index 0000000000..5ce048f7de --- /dev/null +++ b/packages/eve-bench/agent/tools/grep.ts @@ -0,0 +1,45 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath, taskRoot } from "../task-path.js"; + +const execFileAsync = promisify(execFile); + +export default defineTool({ + approval: never(), + description: + "Search task-environment text files with ripgrep and return matching lines with file names and line numbers.", + inputSchema: z.object({ + pattern: z.string().describe("Regular expression accepted by ripgrep."), + path: z + .string() + .optional() + .describe("File or directory relative to the task working directory."), + include: z.string().optional().describe("Optional glob restricting searched files."), + }), + async execute({ pattern, path, include }, ctx) { + const args = ["--line-number", "--no-heading", "--color=never"]; + if (include !== undefined) args.push("--glob", include); + args.push("--", pattern, path === undefined ? "." : resolveTaskPath(path)); + try { + const { stdout, stderr } = await execFileAsync("rg", args, { + cwd: taskRoot(), + maxBuffer: 10 * 1024 * 1024, + signal: ctx.abortSignal, + }); + return { exitCode: 0, stderr, stdout }; + } catch (error) { + const result = error as Error & { code?: number; stderr?: string; stdout?: string }; + const exitCode = typeof result.code === "number" ? result.code : 1; + return { + exitCode, + stderr: result.stderr ?? (exitCode === 1 ? "" : result.message), + stdout: result.stdout ?? "", + }; + } + }, +}); diff --git a/packages/eve-bench/agent/tools/read_file.ts b/packages/eve-bench/agent/tools/read_file.ts new file mode 100644 index 0000000000..c64d4fe2da --- /dev/null +++ b/packages/eve-bench/agent/tools/read_file.ts @@ -0,0 +1,27 @@ +import { readFile } from "node:fs/promises"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: + "Read a UTF-8 text file from the task environment, optionally selecting an inclusive 1-based line range.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + startLine: z.number().int().positive().optional(), + endLine: z.number().int().positive().optional(), + }), + async execute({ path, startLine, endLine }) { + if (startLine !== undefined && endLine !== undefined && startLine > endLine) { + throw new Error("startLine must not exceed endLine."); + } + const content = await readFile(resolveTaskPath(path), "utf8"); + if (startLine === undefined && endLine === undefined) return { content }; + const lines = content.split(/(?<=\n)/u); + return { content: lines.slice((startLine ?? 1) - 1, endLine).join("") }; + }, +}); diff --git a/packages/eve-bench/agent/tools/todo.ts b/packages/eve-bench/agent/tools/todo.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/todo.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_fetch.ts b/packages/eve-bench/agent/tools/web_fetch.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_fetch.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/web_search.ts b/packages/eve-bench/agent/tools/web_search.ts new file mode 100644 index 0000000000..04bd054473 --- /dev/null +++ b/packages/eve-bench/agent/tools/web_search.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/packages/eve-bench/agent/tools/write_file.ts b/packages/eve-bench/agent/tools/write_file.ts new file mode 100644 index 0000000000..65d547a02d --- /dev/null +++ b/packages/eve-bench/agent/tools/write_file.ts @@ -0,0 +1,24 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { resolveTaskPath } from "../task-path.js"; + +export default defineTool({ + approval: never(), + description: + "Create or replace a UTF-8 text file in the task environment. Parent directories are created automatically.", + inputSchema: z.object({ + path: z.string().describe("Absolute path or path relative to the task working directory."), + content: z.string(), + }), + async execute({ path, content }) { + const resolved = resolveTaskPath(path); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile(resolved, content, "utf8"); + return { bytesWritten: Buffer.byteLength(content), path: resolved }; + }, +}); diff --git a/packages/eve-bench/datasets/cohorts/smoke.json b/packages/eve-bench/datasets/cohorts/smoke.json new file mode 100644 index 0000000000..fe2d0a37f9 --- /dev/null +++ b/packages/eve-bench/datasets/cohorts/smoke.json @@ -0,0 +1,11 @@ +{ + "dataset": "terminal-bench-2.0", + "description": "Short, deterministic tasks with small images that exercise shell, file editing, and git tooling. Use for harness changes before running the full dataset.", + "tasks": [ + "fix-git", + "openssl-selfsigned-cert", + "regex-log", + "sqlite-db-truncate", + "log-summary-date-ranges" + ] +} diff --git a/packages/eve-bench/datasets/terminal-bench-2.0.json b/packages/eve-bench/datasets/terminal-bench-2.0.json new file mode 100644 index 0000000000..c13b5eaf35 --- /dev/null +++ b/packages/eve-bench/datasets/terminal-bench-2.0.json @@ -0,0 +1,97 @@ +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": [ + "adaptive-rejection-sampler", + "bn-fit-modify", + "break-filter-js-from-html", + "build-cython-ext", + "build-pmars", + "build-pov-ray", + "caffe-cifar-10", + "cancel-async-tasks", + "chess-best-move", + "circuit-fibsqrt", + "cobol-modernization", + "code-from-image", + "compile-compcert", + "configure-git-webserver", + "constraints-scheduling", + "count-dataset-tokens", + "crack-7z-hash", + "custom-memory-heap-crash", + "db-wal-recovery", + "distribution-search", + "dna-assembly", + "dna-insert", + "extract-elf", + "extract-moves-from-video", + "feal-differential-cryptanalysis", + "feal-linear-cryptanalysis", + "filter-js-from-html", + "financial-document-processor", + "fix-code-vulnerability", + "fix-git", + "fix-ocaml-gc", + "gcode-to-text", + "git-leak-recovery", + "git-multibranch", + "gpt2-codegolf", + "headless-terminal", + "hf-model-inference", + "install-windows-3.11", + "kv-store-grpc", + "large-scale-text-editing", + "largest-eigenval", + "llm-inference-batching-scheduler", + "log-summary-date-ranges", + "mailman", + "make-doom-for-mips", + "make-mips-interpreter", + "mcmc-sampling-stan", + "merge-diff-arc-agi-task", + "model-extraction-relu-logits", + "modernize-scientific-stack", + "mteb-leaderboard", + "mteb-retrieve", + "multi-source-data-merger", + "nginx-request-logging", + "openssl-selfsigned-cert", + "overfull-hbox", + "password-recovery", + "path-tracing", + "path-tracing-reverse", + "polyglot-c-py", + "polyglot-rust-c", + "portfolio-optimization", + "protein-assembly", + "prove-plus-comm", + "pypi-server", + "pytorch-model-cli", + "pytorch-model-recovery", + "qemu-alpine-ssh", + "qemu-startup", + "query-optimize", + "raman-fitting", + "regex-chess", + "regex-log", + "reshard-c4-data", + "rstan-to-pystan", + "sam-cell-seg", + "sanitize-git-repo", + "schemelike-metacircular-eval", + "sparql-university", + "sqlite-db-truncate", + "sqlite-with-gcov", + "torch-pipeline-parallelism", + "torch-tensor-parallelism", + "train-fasttext", + "tune-mjcf", + "video-processing", + "vulnerable-secret", + "winning-avg-corewars", + "write-compressor" + ] +} diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json new file mode 100644 index 0000000000..c06941682a --- /dev/null +++ b/packages/eve-bench/package.json @@ -0,0 +1,26 @@ +{ + "name": "@eve/bench", + "version": "0.0.0", + "private": true, + "description": "Terminal-Bench harness benchmarks for eve.", + "bin": { + "eve-bench": "./src/cli.ts" + }, + "type": "module", + "scripts": { + "bench": "node src/cli.ts", + "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "eve": "workspace:*", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + }, + "engines": { + "node": ">=24" + } +} diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts new file mode 100644 index 0000000000..4c63c3d549 --- /dev/null +++ b/packages/eve-bench/src/cli.ts @@ -0,0 +1,175 @@ +#!/usr/bin/env node +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; + +import { syncDataset, taskDirs } from "./core/dataset.ts"; +import { readJobResult, runJob } from "./core/job.ts"; +import { diffJobs, formatDiff, formatReport, type ReportFormat } from "./core/report.ts"; +import { loadTask } from "./core/task.ts"; +import { + allLocks, + defaultJobName, + forwardedEnv, + jobDir, + lock, + paths, + selectHarness, + selectTasks, +} from "./options.ts"; + +const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve + + eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets + eve-bench tasks list [--cohort ] print task names + eve-bench run --model [--cohort ] [--task ...] [--attempts N] [--concurrency N] + [--harness eve|oracle] [--eve local|] [--job ] [--format console|json|junit] + eve-bench report [--format console|json|junit] + eve-bench diff [--json] + +Jobs live in .generated/jobs/; re-running with the same --job resumes it. +Provider credentials are read from the environment and forwarded into the task container. +`; + +const [command, subcommand, ...rest] = process.argv.slice(2); +try { + switch (command) { + case "tasks": + await tasks(subcommand, rest); + break; + case "run": + await run(subcommand === undefined ? [] : [subcommand, ...rest]); + break; + case "report": + await report(subcommand, rest); + break; + case "diff": + await diff(subcommand, rest); + break; + default: + process.stdout.write(USAGE); + process.exitCode = command === undefined || command === "--help" || command === "-h" ? 0 : 2; + } +} catch (error) { + process.stderr.write(`eve-bench: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} + +async function tasks(sub: string | undefined, args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { dataset: { type: "string" }, cohort: { type: "string" }, json: { type: "boolean" } }, + }); + if (sub === "sync") { + const locks = values.dataset ? [await lock(values.dataset)] : await allLocks(); + for (const entry of locks) { + const dir = await syncDataset(join(paths.generatedRoot, "datasets"), entry); + process.stdout.write( + values.json + ? `${JSON.stringify({ dataset: entry.name, version: entry.version, dir })}\n` + : `${entry.name}@${entry.version} -> ${dir}\n`, + ); + } + return; + } + if (sub === "list") { + const selection = await selectTasks(values); + process.stdout.write( + values.json ? `${JSON.stringify(selection.tasks)}\n` : `${selection.tasks.join("\n")}\n`, + ); + return; + } + throw new Error(`unknown tasks subcommand: ${sub ?? "(none)"}\n${USAGE}`); +} + +async function run(args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { + model: { type: "string" }, + cohort: { type: "string" }, + dataset: { type: "string" }, + task: { type: "string", multiple: true }, + attempts: { type: "string", default: "1" }, + concurrency: { type: "string", default: "4" }, + eve: { type: "string", default: "local" }, + harness: { type: "string", default: "eve" }, + job: { type: "string" }, + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + }, + }); + const harness = selectHarness(values.harness, values.eve); + const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); + if (!model) throw new Error("--model is required"); + const selection = await selectTasks(values); + const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const loaded = await Promise.all( + (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + ); + const name = values.job ?? defaultJobName(`${harness.name}-${model}`); + const controller = new AbortController(); + const stop = () => { + process.stderr.write("\neve-bench: aborting, cleaning up containers...\n"); + controller.abort(); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + const format = reportFormat(values.json ? "json" : values.format); + const result = await runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks: loaded, + harness, + model, + attempts: Number(values.attempts), + concurrency: Number(values.concurrency), + forwardEnv: forwardedEnv(), + signal: controller.signal, + onTrial: (trial, resumed) => { + if (format !== "console") return; + const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); + process.stderr.write( + `${resumed ? "resume" : "trial "} ${trial.task}#${trial.attempt} reward=${reward} agent=${trial.agent.status} verifier=${trial.verifier.status}${trial.error ? ` error=${trial.error}` : ""}\n`, + ); + }, + }); + process.stdout.write(formatReport(result, format)); + if (controller.signal.aborted) process.exitCode = 130; +} + +async function report(job: string | undefined, args: string[]): Promise { + if (!job) throw new Error("report requires a job name"); + const { values } = parseArgs({ + args, + options: { + format: { type: "string", default: "console" }, + json: { type: "boolean" }, + out: { type: "string" }, + }, + }); + const output = formatReport( + await readJobResult(join(paths.generatedRoot, "jobs", job)), + reportFormat(values.json ? "json" : values.format), + ); + if (values.out) await writeFile(values.out, output); + else process.stdout.write(output); +} + +async function diff(base: string | undefined, args: string[]): Promise { + const [candidate, ...restArgs] = args; + if (!base || !candidate) throw new Error("diff requires "); + const { values } = parseArgs({ args: restArgs, options: { json: { type: "boolean" } } }); + const deltas = diffJobs( + await readJobResult(jobDir(base)), + await readJobResult(jobDir(candidate)), + ); + process.stdout.write(values.json ? `${JSON.stringify(deltas, null, 2)}\n` : formatDiff(deltas)); +} + +function reportFormat(value: string | undefined): ReportFormat { + if (value === "console" || value === "json" || value === "junit") return value; + throw new Error(`unknown format: ${value}`); +} diff --git a/packages/eve-bench/src/core/dataset.test.ts b/packages/eve-bench/src/core/dataset.test.ts new file mode 100644 index 0000000000..c199c11519 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.test.ts @@ -0,0 +1,103 @@ +// syncDataset is intentionally untested because it performs network and git operations. + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { datasetDir, readCohort, readDatasetLock, taskDirs } from "./dataset.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-dataset-")); +after(() => rm(root, { recursive: true, force: true })); + +const VALID_LOCK = { + name: "terminal-bench", + version: "2.0", + gitUrl: "https://github.com/laude-institute/terminal-bench-2.git", + commit: "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + tasks: ["adaptive-rejection-sampler", "fix-git"], +}; + +test("parses a valid dataset lock", async () => { + const path = join(root, "valid-lock.json"); + await writeFile(path, JSON.stringify(VALID_LOCK)); + + const lock = await readDatasetLock(path); + + assert.deepEqual(lock, VALID_LOCK); +}); + +for (const field of ["name", "version", "gitUrl", "commit"] as const) { + test(`rejects a dataset lock missing ${field}`, async () => { + const path = join(root, `missing-${field}.json`); + const lock: Record = { ...VALID_LOCK }; + delete lock[field]; + await writeFile(path, JSON.stringify(lock)); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); + + test(`rejects a dataset lock with an empty ${field}`, async () => { + const path = join(root, `empty-${field}.json`); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, [field]: "" })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: missing "${field}"`, + }); + }); +} + +test("rejects a dataset lock with an empty tasks array", async () => { + const path = join(root, "empty-lock-tasks.json"); + await writeFile(path, JSON.stringify({ ...VALID_LOCK, tasks: [] })); + + await assert.rejects(readDatasetLock(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("rejects a cohort missing dataset", async () => { + const path = join(root, "missing-cohort-dataset.json"); + await writeFile(path, JSON.stringify({ tasks: ["fix-git"] })); + + await assert.rejects(readCohort(path), { + message: `${path}: missing "dataset"`, + }); +}); + +test("rejects a cohort with an empty tasks array", async () => { + const path = join(root, "empty-cohort-tasks.json"); + await writeFile(path, JSON.stringify({ dataset: "terminal-bench-2.0", tasks: [] })); + + await assert.rejects(readCohort(path), { + message: `${path}: "tasks" must be a non-empty array`, + }); +}); + +test("builds the versioned dataset directory path", () => { + assert.equal(datasetDir(root, VALID_LOCK), join(root, "terminal-bench-2.0")); +}); + +test("resolves task directories containing task.toml", async () => { + const dir = join(root, "tasks-valid"); + await mkdir(join(dir, "task-a"), { recursive: true }); + await mkdir(join(dir, "nested", "task-b"), { recursive: true }); + await writeFile(join(dir, "task-a", "task.toml"), ""); + await writeFile(join(dir, "nested", "task-b", "task.toml"), ""); + + const dirs = await taskDirs(dir, ["task-a", "nested/task-b"]); + + assert.deepEqual(dirs, [join(dir, "task-a"), join(dir, "nested", "task-b")]); +}); + +test("rejects when any task directory lacks task.toml", async () => { + const dir = join(root, "tasks-missing-toml"); + await mkdir(join(dir, "present"), { recursive: true }); + await mkdir(join(dir, "missing"), { recursive: true }); + await writeFile(join(dir, "present", "task.toml"), ""); + + await assert.rejects(taskDirs(dir, ["present", "missing"]), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/dataset.ts b/packages/eve-bench/src/core/dataset.ts new file mode 100644 index 0000000000..beaa588dd5 --- /dev/null +++ b/packages/eve-bench/src/core/dataset.ts @@ -0,0 +1,77 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface DatasetLock { + readonly name: string; + readonly version: string; + readonly gitUrl: string; + readonly commit: string; + readonly tasks: readonly string[]; +} + +export interface Cohort { + readonly dataset: string; + readonly description?: string; + readonly tasks: readonly string[]; +} + +export async function readDatasetLock(path: string): Promise { + const lock = JSON.parse(await readFile(path, "utf8")) as DatasetLock; + for (const key of ["name", "version", "gitUrl", "commit"] as const) { + if (typeof lock[key] !== "string" || lock[key].length === 0) { + throw new Error(`${path}: missing "${key}"`); + } + } + if (!Array.isArray(lock.tasks) || lock.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return lock; +} + +export async function readCohort(path: string): Promise { + const cohort = JSON.parse(await readFile(path, "utf8")) as Cohort; + if (typeof cohort.dataset !== "string") throw new Error(`${path}: missing "dataset"`); + if (!Array.isArray(cohort.tasks) || cohort.tasks.length === 0) + throw new Error(`${path}: "tasks" must be a non-empty array`); + return cohort; +} + +export function datasetDir(root: string, lock: DatasetLock): string { + return join(root, `${lock.name}-${lock.version}`); +} + +/** Fetches the pinned commit with a sparse, blobless checkout of the locked tasks. Idempotent. */ +export async function syncDataset(root: string, lock: DatasetLock): Promise { + const dir = datasetDir(root, lock); + if (await isSynced(dir, lock)) return dir; + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + const git = (...args: string[]) => + execFileAsync("git", ["-C", dir, ...args], { maxBuffer: 64 * 1024 * 1024 }); + await git("init", "--quiet"); + await git("remote", "add", "origin", lock.gitUrl); + await git("sparse-checkout", "set", "--cone", ...lock.tasks); + await git("fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", lock.commit); + await git("checkout", "--quiet", lock.commit); + return dir; +} + +async function isSynced(dir: string, lock: DatasetLock): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", dir, "rev-parse", "HEAD"]); + if (stdout.trim() !== lock.commit) return false; + const present = new Set(await readdir(dir)); + return lock.tasks.every((task) => present.has(task)); + } catch { + return false; + } +} + +export async function taskDirs(dir: string, tasks: readonly string[]): Promise { + const dirs = tasks.map((task) => join(dir, task)); + await Promise.all(dirs.map((task) => access(join(task, "task.toml")))); + return dirs; +} diff --git a/packages/eve-bench/src/core/docker.ts b/packages/eve-bench/src/core/docker.ts new file mode 100644 index 0000000000..039ce37451 --- /dev/null +++ b/packages/eve-bench/src/core/docker.ts @@ -0,0 +1,177 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import type { Writable } from "node:stream"; + +export interface ProcessResult { + readonly exitCode: number | null; + readonly timedOut: boolean; + readonly aborted: boolean; + readonly stdout: string; + readonly stderr: string; +} + +export interface ExecOptions { + readonly cwd?: string; + readonly env?: Readonly>; + readonly timeoutMs?: number; + readonly signal?: AbortSignal; + readonly log?: Writable; +} + +export interface ContainerOptions { + readonly image: string; + readonly name: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly network: boolean; +} + +export class Container { + readonly name: string; + + private constructor(name: string) { + this.name = name; + } + + static async start(options: ContainerOptions, signal?: AbortSignal): Promise { + const args = ["run", "--detach", "--name", options.name, "--entrypoint", "sh"]; + if (options.cpus !== undefined) args.push("--cpus", String(options.cpus)); + if (options.memoryMb !== undefined) args.push("--memory", `${options.memoryMb}m`); + if (!options.network) args.push("--network", "none"); + args.push(options.image, "-c", "sleep infinity"); + await dockerOk(args, { signal }); + return new Container(options.name); + } + + /** + * Runs `command` through `sh -c` inside the container. A `docker exec` + * process is its own session and process-group leader, so recording its + * pid lets timeouts and aborts kill the whole group in-container; killing + * the host-side client alone would leave the workload running. + */ + async exec(command: string, options: ExecOptions = {}): Promise { + const pidFile = `/tmp/.eve-bench-${randomUUID()}.pid`; + const wrapped = `echo $$ > ${pidFile}; exec sh -c "$0"`; + const args = ["exec"]; + if (options.cwd) args.push("--workdir", options.cwd); + for (const [key, value] of Object.entries(options.env ?? {})) + args.push("--env", `${key}=${value}`); + args.push(this.name, "sh", "-c", wrapped, command); + const result = await docker(args, options); + if (result.timedOut || result.aborted) { + await docker( + [ + "exec", + this.name, + "sh", + "-c", + `pid=$(cat ${pidFile} 2>/dev/null); [ -n "$pid" ] && { kill -KILL -- -$pid 2>/dev/null; kill -KILL $pid 2>/dev/null; }; true`, + ], + { timeoutMs: 10_000 }, + ); + } + return result; + } + + async upload(hostPath: string, containerPath: string, signal?: AbortSignal): Promise { + await dockerOk(["cp", hostPath, `${this.name}:${containerPath}`], { signal }); + } + + async download(containerPath: string, hostPath: string, signal?: AbortSignal): Promise { + const result = await docker(["cp", `${this.name}:${containerPath}`, hostPath], { signal }); + return result.exitCode === 0; + } + + async remove(): Promise { + await docker(["rm", "--force", "--volumes", this.name], { timeoutMs: 60_000 }); + } +} + +export async function imageExists(tag: string): Promise { + const result = await docker(["image", "inspect", tag], { timeoutMs: 30_000 }); + return result.exitCode === 0; +} + +export async function pullImage(image: string, options: ExecOptions = {}): Promise { + if (await imageExists(image)) return; + await dockerOk(["pull", image], options); +} + +export async function buildImage( + tag: string, + contextDir: string, + options: ExecOptions = {}, +): Promise { + if (await imageExists(tag)) return; + await dockerOk(["build", "--tag", tag, contextDir], options); +} + +export async function containerArch(container: Container): Promise<"x64" | "arm64"> { + const result = await container.exec("uname -m", { timeoutMs: 10_000 }); + const machine = result.stdout.trim(); + if (machine === "x86_64") return "x64"; + if (machine === "aarch64") return "arm64"; + throw new Error(`Unsupported container architecture: ${machine || result.stderr.trim()}`); +} + +async function dockerOk(args: readonly string[], options: ExecOptions): Promise { + const result = await docker(args, options); + if (result.exitCode !== 0) { + const reason = result.timedOut + ? "timed out" + : result.aborted + ? "aborted" + : `exit ${result.exitCode}`; + throw new Error(`docker ${args[0]} ${reason}: ${result.stderr.trim() || result.stdout.trim()}`); + } + return result; +} + +function docker(args: readonly string[], options: ExecOptions): Promise { + return new Promise((resolve, reject) => { + const child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + let aborted = false; + const timer = + options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.timeoutMs); + const onAbort = () => { + aborted = true; + child.kill("SIGKILL"); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + child.stdout.on("data", (chunk: Buffer) => { + stdout.push(chunk); + options.log?.write(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr.push(chunk); + options.log?.write(chunk); + }); + child.once("error", (error) => { + cleanup(); + reject(error); + }); + child.once("close", (code) => { + cleanup(); + resolve({ + exitCode: code, + timedOut, + aborted, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + function cleanup() { + if (timer) clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + } + }); +} diff --git a/packages/eve-bench/src/core/harness.ts b/packages/eve-bench/src/core/harness.ts new file mode 100644 index 0000000000..797d481d27 --- /dev/null +++ b/packages/eve-bench/src/core/harness.ts @@ -0,0 +1,34 @@ +/** + * A harness is anything that can be placed in a task container and told to + * solve an instruction. The runner uploads `bundleDir` to `/installed-agent`, + * then runs `command` through `sh -c` in the task working directory. + */ +export interface Harness { + readonly name: string; + prepare(ctx: HarnessPrepareContext): Promise; + /** Optional per-task host paths uploaded into the install directory before `command` runs. */ + stage?(ctx: { readonly taskDir: string }): readonly string[]; + command(ctx: HarnessRunContext): string; + env(ctx: HarnessRunContext): Record; +} + +export interface HarnessPrepareContext { + readonly model: string; + readonly cacheDir: string; +} + +export interface HarnessBundle { + readonly dir: string; + /** Architecture-specific host directories uploaded to `${installDir}/arch`. */ + readonly arch?: Readonly>; + /** Recorded in results so a run can be traced back to the exact harness build. */ + readonly provenance?: Record; +} + +export interface HarnessRunContext { + readonly model: string; + readonly installDir: string; + readonly instructionPath: string; + readonly taskWorkdir: string; + readonly logsDir: string; +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts new file mode 100644 index 0000000000..d16feead4e --- /dev/null +++ b/packages/eve-bench/src/core/job.ts @@ -0,0 +1,108 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import type { Harness } from "./harness.ts"; +import { summarize, type JobResult, type TrialResult } from "./result.ts"; +import type { Task } from "./task.ts"; +import { runTrial } from "./trial.ts"; + +export interface JobInput { + readonly name: string; + readonly dir: string; + readonly cacheDir: string; + readonly dataset: DatasetLock; + readonly tasks: readonly Task[]; + readonly harness: Harness; + readonly model: string; + readonly attempts: number; + readonly concurrency: number; + readonly forwardEnv: Readonly>; + readonly signal: AbortSignal; + readonly onTrial?: (trial: TrialResult, resumed: boolean) => void; +} + +export async function runJob(input: JobInput): Promise { + await mkdir(input.dir, { recursive: true }); + const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); + await writeFile( + join(input.dir, "job.json"), + `${JSON.stringify( + { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + attempts: input.attempts, + tasks: input.tasks.map((task) => task.name), + bundle: bundle.provenance ?? {}, + startedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + ); + + const queue = input.tasks.flatMap((task) => + Array.from({ length: input.attempts }, (_, index) => ({ task, attempt: index + 1 })), + ); + const trials: TrialResult[] = []; + const workers = Array.from({ length: Math.max(1, input.concurrency) }, async () => { + for (;;) { + const next = queue.shift(); + if (!next || input.signal.aborted) return; + const dir = join(input.dir, next.task.name, String(next.attempt)); + const existing = await readTrial(dir); + if (existing) { + trials.push(existing); + input.onTrial?.(existing, true); + continue; + } + const trial = await runTrial({ + task: next.task, + attempt: next.attempt, + harness: input.harness, + bundle, + model: input.model, + forwardEnv: input.forwardEnv, + dir, + signal: input.signal, + }); + trials.push(trial); + input.onTrial?.(trial, false); + } + }); + await Promise.all(workers); + + trials.sort((a, b) => a.task.localeCompare(b.task) || a.attempt - b.attempt); + const result: JobResult = { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + trials, + summary: summarize(trials), + }; + await writeFile(join(input.dir, "result.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +export async function readJobResult(dir: string): Promise { + return JSON.parse(await readFile(join(dir, "result.json"), "utf8")) as JobResult; +} + +async function readTrial(dir: string): Promise { + try { + return JSON.parse(await readFile(join(dir, "trial.json"), "utf8")) as TrialResult; + } catch { + return undefined; + } +} diff --git a/packages/eve-bench/src/core/report.test.ts b/packages/eve-bench/src/core/report.test.ts new file mode 100644 index 0000000000..70ff0e7390 --- /dev/null +++ b/packages/eve-bench/src/core/report.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { diffJobs, formatDiff, formatReport } from "./report.ts"; +import { summarize, type JobResult, type TrialResult } from "./result.ts"; + +function trial( + task: string, + attempt: number, + reward: number | null, + extra: Partial = {}, +): TrialResult { + return { + task, + attempt, + harness: "eve@local", + model: "test/model", + reward, + agent: { status: "completed", exitCode: 0, durationMs: 1000 }, + verifier: { status: "completed", exitCode: 0, durationMs: 500 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.500Z", + ...extra, + }; +} + +function job(name: string, trials: TrialResult[]): JobResult { + return { + job: name, + dataset: { name: "terminal-bench", version: "2.0", commit: "abc" }, + harness: "eve@local", + model: "test/model", + trials, + summary: summarize(trials), + }; +} + +test("summarize counts resolved attempts and mean reward", () => { + const summary = summarize([trial("a", 1, 1), trial("a", 2, 0), trial("b", 1, null)]); + assert.deepEqual(summary, { tasks: 2, attempts: 3, resolved: 1, meanReward: 1 / 3 }); +}); + +test("junit report marks unresolved trials as failures", () => { + const output = formatReport( + job("j", [trial("a", 1, 1), trial("b", 1, 0, { error: "boom " })]), + "junit", + ); + assert.match(output, /tests="2" failures="1"/u); + assert.match(output, //u); + assert.match(output, //u); +}); + +test("diff reports per-task mean reward deltas", () => { + const base = job("base", [trial("a", 1, 0), trial("a", 2, 1), trial("b", 1, 1)]); + const candidate = job("cand", [trial("a", 1, 1), trial("a", 2, 1), trial("c", 1, 0)]); + const deltas = diffJobs(base, candidate); + assert.deepEqual(deltas, [ + { task: "a", base: 0.5, candidate: 1, delta: 0.5 }, + { task: "b", base: 1, candidate: null, delta: null }, + { task: "c", base: null, candidate: 0, delta: null }, + ]); + assert.match(formatDiff(deltas), /1 task\(s\) changed, net \+0.500/u); +}); diff --git a/packages/eve-bench/src/core/report.ts b/packages/eve-bench/src/core/report.ts new file mode 100644 index 0000000000..5a931878da --- /dev/null +++ b/packages/eve-bench/src/core/report.ts @@ -0,0 +1,118 @@ +import type { JobResult, TrialResult } from "./result.ts"; + +export type ReportFormat = "console" | "json" | "junit"; + +export function formatReport(result: JobResult, format: ReportFormat): string { + switch (format) { + case "json": + return `${JSON.stringify(result, null, 2)}\n`; + case "junit": + return junit(result); + case "console": + return console_(result); + } +} + +function console_(result: JobResult): string { + const lines = [ + `${result.job} ${result.harness} / ${result.model} ${result.dataset.name}@${result.dataset.version}`, + "", + ]; + const byTask = groupByTask(result.trials); + for (const [task, trials] of byTask) { + const rewards = trials.map((trial) => (trial.reward === null ? "-" : trial.reward.toFixed(2))); + const errors = trials.filter((trial) => trial.error || trial.agent.status === "timeout").length; + lines.push( + `${task.padEnd(44)} ${rewards.join(" ").padEnd(16)} ${errors > 0 ? `${errors} error(s)` : ""}`.trimEnd(), + ); + } + const { summary } = result; + lines.push( + "", + `tasks ${summary.tasks} attempts ${summary.attempts} resolved ${summary.resolved} mean reward ${summary.meanReward.toFixed(3)}`, + ); + return `${lines.join("\n")}\n`; +} + +function junit(result: JobResult): string { + const failures = result.trials.filter((trial) => (trial.reward ?? 0) < 1).length; + const cases = result.trials.map((trial) => { + const time = ((trial.agent.durationMs + trial.verifier.durationMs) / 1000).toFixed(3); + const name = escape(`${trial.task} #${trial.attempt}`); + if ((trial.reward ?? 0) >= 1) return ` `; + const message = escape( + trial.error ?? + `reward ${trial.reward ?? "none"}; agent ${trial.agent.status}; verifier ${trial.verifier.status}`, + ); + return ` \n \n `; + }); + return [ + ``, + ``, + ` `, + ...cases, + ` `, + ``, + "", + ].join("\n"); +} + +export interface TaskDelta { + readonly task: string; + readonly base: number | null; + readonly candidate: number | null; + readonly delta: number | null; +} + +export function diffJobs(base: JobResult, candidate: JobResult): TaskDelta[] { + const baseMeans = meanByTask(base.trials); + const candidateMeans = meanByTask(candidate.trials); + const tasks = [...new Set([...baseMeans.keys(), ...candidateMeans.keys()])].sort(); + return tasks.map((task) => { + const b = baseMeans.get(task) ?? null; + const c = candidateMeans.get(task) ?? null; + return { task, base: b, candidate: c, delta: b === null || c === null ? null : c - b }; + }); +} + +export function formatDiff(deltas: readonly TaskDelta[]): string { + const lines = deltas.map((row) => { + const fmt = (value: number | null) => + value === null ? " - " : value.toFixed(3).padStart(6); + const delta = + row.delta === null + ? "" + : row.delta === 0 + ? "" + : row.delta > 0 + ? `+${row.delta.toFixed(3)}` + : row.delta.toFixed(3); + return `${row.task.padEnd(44)} ${fmt(row.base)} -> ${fmt(row.candidate)} ${delta}`.trimEnd(); + }); + const changed = deltas.filter((row) => row.delta !== null && row.delta !== 0); + const net = changed.reduce((sum, row) => sum + (row.delta ?? 0), 0); + lines.push("", `${changed.length} task(s) changed, net ${net >= 0 ? "+" : ""}${net.toFixed(3)}`); + return `${lines.join("\n")}\n`; +} + +function groupByTask(trials: readonly TrialResult[]): Map { + const groups = new Map(); + for (const trial of trials) { + const list = groups.get(trial.task) ?? []; + list.push(trial); + groups.set(trial.task, list); + } + return groups; +} + +function meanByTask(trials: readonly TrialResult[]): Map { + const means = new Map(); + for (const [task, group] of groupByTask(trials)) { + means.set(task, group.reduce((sum, trial) => sum + (trial.reward ?? 0), 0) / group.length); + } + return means; +} + +function escape(text: string): string { + return text.replaceAll("&", "&").replaceAll("<", "<").replaceAll('"', """); +} diff --git a/packages/eve-bench/src/core/result.ts b/packages/eve-bench/src/core/result.ts new file mode 100644 index 0000000000..d42e58a798 --- /dev/null +++ b/packages/eve-bench/src/core/result.ts @@ -0,0 +1,52 @@ +export type StepStatus = "completed" | "timeout" | "failed" | "skipped"; + +export interface StepResult { + readonly status: StepStatus; + readonly exitCode: number | null; + readonly durationMs: number; +} + +export interface Usage { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cachedTokens: number; + readonly costUsd: number; +} + +export interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: Usage; + readonly error?: string; + readonly startedAt: string; + readonly finishedAt: string; +} + +export interface JobResult { + readonly job: string; + readonly dataset: { readonly name: string; readonly version: string; readonly commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: JobSummary; +} + +export interface JobSummary { + readonly tasks: number; + readonly attempts: number; + readonly resolved: number; + readonly meanReward: number; +} + +export function summarize(trials: readonly TrialResult[]): JobSummary { + const tasks = new Set(trials.map((trial) => trial.task)).size; + const rewards = trials.map((trial) => trial.reward ?? 0); + const resolved = rewards.filter((reward) => reward >= 1).length; + const meanReward = rewards.length === 0 ? 0 : rewards.reduce((a, b) => a + b, 0) / rewards.length; + return { tasks, attempts: trials.length, resolved, meanReward }; +} diff --git a/packages/eve-bench/src/core/task.test.ts b/packages/eve-bench/src/core/task.test.ts new file mode 100644 index 0000000000..7767b05c41 --- /dev/null +++ b/packages/eve-bench/src/core/task.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import { loadTask } from "./task.ts"; + +const root = await mkdtemp(join(tmpdir(), "eve-bench-task-")); +after(() => rm(root, { recursive: true, force: true })); + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[agent] +timeout_sec = 1.234 + +[verifier] +timeout_sec = 2.789 + +[environment] +build_timeout_sec = 3.456 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1.5 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +async function writeTask(dir: string, toml = TASK_TOML): Promise { + await mkdir(join(dir, "tests"), { recursive: true }); + await writeFile(join(dir, "task.toml"), toml); + await writeFile(join(dir, "instruction.md"), "Solve the task.\n"); + await writeFile(join(dir, "tests", "test.sh"), "#!/bin/sh\n"); +} + +test("loads Terminal-Bench task fields and converts timeout seconds to milliseconds", async () => { + const dir = join(root, "complete"); + await writeTask(dir); + + const task = await loadTask(dir); + + assert.equal(task.name, "adaptive-rejection-sampler"); + assert.equal(task.dir, dir); + assert.equal(task.instruction, "Solve the task.\n"); + assert.equal(task.agentTimeoutMs, 1234); + assert.equal(task.verifierTimeoutMs, 2789); + assert.equal(task.buildTimeoutMs, 3456); + assert.equal(task.environment.dockerImage, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(task.environment.cpus, 1.5); + assert.equal(task.environment.memoryMb, 2048); + assert.equal(task.environment.allowInternet, true); +}); + +test('converts older-schema memory = "2G" to 2048 MB', async () => { + const dir = join(root, "memory-2g"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 2048); +}); + +test('converts older-schema memory = "512M" to 512 MB', async () => { + const dir = join(root, "memory-512m"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory = "512M"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 512); +}); + +test("prefers memory_mb when both memory fields are present", async () => { + const dir = join(root, "memory-precedence"); + await writeTask(dir, TASK_TOML.replace("memory_mb = 2048", 'memory_mb = 768\nmemory = "2G"')); + + const task = await loadTask(dir); + + assert.equal(task.environment.memoryMb, 768); +}); + +test("uses 15-minute defaults when agent and verifier timeouts are missing", async () => { + const dir = join(root, "default-timeouts"); + const toml = TASK_TOML.replace("[agent]\ntimeout_sec = 1.234\n\n", "").replace( + "[verifier]\ntimeout_sec = 2.789\n\n", + "", + ); + await writeTask(dir, toml); + + const task = await loadTask(dir); + + assert.equal(task.agentTimeoutMs, 900_000); + assert.equal(task.verifierTimeoutMs, 900_000); +}); + +test("allows internet when allow_internet is absent", async () => { + const dir = join(root, "internet-default"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true\n", "")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, true); +}); + +test("disallows internet when allow_internet is false", async () => { + const dir = join(root, "internet-false"); + await writeTask(dir, TASK_TOML.replace("allow_internet = true", "allow_internet = false")); + + const task = await loadTask(dir); + + assert.equal(task.environment.allowInternet, false); +}); + +test("rejects a task with neither docker_image nor environment/Dockerfile", async () => { + const dir = join(root, "missing-image"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); + +test("loads environment/Dockerfile when docker_image is absent", async () => { + const dir = join(root, "dockerfile"); + await writeTask( + dir, + TASK_TOML.replace('docker_image = "alexgshaw/adaptive-rejection-sampler:20251031"\n', ""), + ); + await mkdir(join(dir, "environment")); + await writeFile(join(dir, "environment", "Dockerfile"), "FROM scratch\n"); + + const task = await loadTask(dir); + + assert.equal(task.environment.dockerImage, undefined); + assert.equal(task.environment.dockerfileDir, join(dir, "environment")); +}); + +test("rejects a task missing tests/test.sh", async () => { + const dir = join(root, "missing-test"); + await writeTask(dir); + await rm(join(dir, "tests", "test.sh")); + + await assert.rejects(loadTask(dir), { code: "ENOENT" }); +}); diff --git a/packages/eve-bench/src/core/task.ts b/packages/eve-bench/src/core/task.ts new file mode 100644 index 0000000000..8788b8e74b --- /dev/null +++ b/packages/eve-bench/src/core/task.ts @@ -0,0 +1,82 @@ +import { access, readFile } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import { parseToml, type TomlTable } from "./toml.ts"; + +export interface Task { + readonly name: string; + readonly dir: string; + readonly instruction: string; + readonly agentTimeoutMs: number; + readonly verifierTimeoutMs: number; + readonly buildTimeoutMs: number; + readonly environment: { + readonly dockerImage?: string; + readonly dockerfileDir: string; + readonly cpus?: number; + readonly memoryMb?: number; + readonly allowInternet: boolean; + }; +} + +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; + +export async function loadTask(dir: string): Promise { + const toml = parseToml(await readFile(join(dir, "task.toml"), "utf8")); + const instruction = await readFile(join(dir, "instruction.md"), "utf8"); + await access(join(dir, "tests", "test.sh")); + const task = table(toml, "task"); + const agent = table(toml, "agent"); + const verifier = table(toml, "verifier"); + const environment = table(toml, "environment"); + const dockerImage = str(environment, "docker_image"); + const dockerfileDir = join(dir, "environment"); + if (dockerImage === undefined) await access(join(dockerfileDir, "Dockerfile")); + return { + name: shortName(str(task, "name") ?? basename(dir)), + dir, + instruction, + agentTimeoutMs: seconds(agent, "timeout_sec"), + verifierTimeoutMs: seconds(verifier, "timeout_sec"), + buildTimeoutMs: seconds(environment, "build_timeout_sec"), + environment: { + dockerImage, + dockerfileDir, + cpus: num(environment, "cpus"), + memoryMb: num(environment, "memory_mb") ?? memoryMb(str(environment, "memory")), + allowInternet: environment.allow_internet !== false, + }, + }; +} + +function shortName(name: string): string { + return name.includes("/") ? name.slice(name.lastIndexOf("/") + 1) : name; +} + +function table(root: TomlTable, key: string): TomlTable { + const value = root[key]; + return typeof value === "object" && value !== null && !Array.isArray(value) ? value : {}; +} + +function str(table: TomlTable, key: string): string | undefined { + const value = table[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function num(table: TomlTable, key: string): number | undefined { + const value = table[key]; + return typeof value === "number" ? value : undefined; +} + +/** Older tasks declare `memory = "2G"` instead of `memory_mb`. */ +function memoryMb(value: string | undefined): number | undefined { + const match = value?.match(/^(\d+(?:\.\d+)?)\s*([GgMm])/u); + if (!match) return undefined; + const amount = Number(match[1]); + return Math.round(match[2]!.toLowerCase() === "g" ? amount * 1024 : amount); +} + +function seconds(table: TomlTable, key: string): number { + const value = num(table, key); + return value === undefined ? DEFAULT_TIMEOUT_MS : Math.round(value * 1000); +} diff --git a/packages/eve-bench/src/core/toml.test.ts b/packages/eve-bench/src/core/toml.test.ts new file mode 100644 index 0000000000..a2d31ddc6b --- /dev/null +++ b/packages/eve-bench/src/core/toml.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { parseToml } from "./toml.ts"; + +const TASK_TOML = `schema_version = "1.1" +artifacts = [] + +[task] +name = "terminal-bench/adaptive-rejection-sampler" +keywords = ["applied-statistics", "simulation"] # trailing comment +[[task.authors]] +name = "jvpoulos" +email = "poulos@berkeley.edu" + +[metadata] +expert_time_estimate_min = 180.0 + +[verifier] +timeout_sec = 900.0 + +[environment] +build_timeout_sec = 600.0 +docker_image = "alexgshaw/adaptive-rejection-sampler:20251031" +cpus = 1 +memory_mb = 2048 +allow_internet = true +mcp_servers = [] + +[verifier.env] + +[environment.env] +`; + +test("parses the Terminal-Bench task.toml subset", () => { + const parsed = parseToml(TASK_TOML); + assert.equal(parsed.schema_version, "1.1"); + assert.deepEqual(parsed.artifacts, []); + const task = parsed.task as Record; + assert.equal(task.name, "terminal-bench/adaptive-rejection-sampler"); + assert.deepEqual(task.keywords, ["applied-statistics", "simulation"]); + assert.deepEqual(task.authors, [{ name: "jvpoulos", email: "poulos@berkeley.edu" }]); + const environment = parsed.environment as Record; + assert.equal(environment.docker_image, "alexgshaw/adaptive-rejection-sampler:20251031"); + assert.equal(environment.cpus, 1); + assert.equal(environment.allow_internet, true); + assert.deepEqual(environment.env, {}); + assert.deepEqual((parsed.verifier as Record).timeout_sec, 900); +}); + +test("handles dotted tables, escapes, and nested arrays", () => { + const parsed = parseToml(` +[a.b] +text = "line\\nbreak \\"quoted\\" # not a comment" +nested = [[1, 2], ["x, y"], []] +literal = 'C:\\path' +`); + const table = (parsed.a as Record).b as Record; + assert.equal(table.text, 'line\nbreak "quoted" # not a comment'); + assert.deepEqual(table.nested, [[1, 2], ["x, y"], []]); + assert.equal(table.literal, "C:\\path"); +}); + +test("rejects unsupported syntax with the line number", () => { + assert.throws(() => parseToml(`ok = 1\nbad = { inline = true }`), /line 2: inline tables/u); + assert.throws(() => parseToml(`ok = 1\nbad = 2024-01-01`), /line 2: unsupported value/u); + assert.throws(() => parseToml(`no equals sign`), /line 1: expected key = value/u); +}); diff --git a/packages/eve-bench/src/core/toml.ts b/packages/eve-bench/src/core/toml.ts new file mode 100644 index 0000000000..a17f39f06d --- /dev/null +++ b/packages/eve-bench/src/core/toml.ts @@ -0,0 +1,159 @@ +/** + * Minimal TOML reader for Terminal-Bench `task.toml` files. Supports tables, + * array-of-tables headers, bare/quoted keys, strings, numbers, booleans, and + * arrays. Anything else fails with the offending line so a dataset change + * surfaces immediately instead of being silently misread. + */ +export type TomlValue = string | number | boolean | TomlValue[] | TomlTable; +export interface TomlTable { + [key: string]: TomlValue; +} + +export function parseToml(source: string): TomlTable { + const root: TomlTable = {}; + let current = root; + const lines = source.split(/\r?\n/u); + for (let index = 0; index < lines.length; index++) { + const line = stripComment(lines[index] ?? "").trim(); + if (line.length === 0) continue; + const lineNumber = index + 1; + if (line.startsWith("[[") && line.endsWith("]]")) { + const path = parseKeyPath(line.slice(2, -2), lineNumber); + const parent = descend(root, path.slice(0, -1)); + const last = path.at(-1)!; + const existing = parent[last]; + const list = Array.isArray(existing) ? existing : []; + parent[last] = list; + current = {}; + list.push(current); + continue; + } + if (line.startsWith("[") && line.endsWith("]")) { + current = descend(root, parseKeyPath(line.slice(1, -1), lineNumber)); + continue; + } + const eq = findUnquoted(line, "="); + if (eq === -1) throw new TomlError(`expected key = value`, lineNumber); + const path = parseKeyPath(line.slice(0, eq), lineNumber); + const raw = line.slice(eq + 1).trim(); + const target = descend(current, path.slice(0, -1)); + target[path.at(-1)!] = parseValue(raw, lineNumber); + } + return root; +} + +export class TomlError extends Error { + constructor(message: string, line: number) { + super(`task.toml line ${line}: ${message}`); + } +} + +function descend(table: TomlTable, path: readonly string[]): TomlTable { + let node = table; + for (const segment of path) { + const next = node[segment]; + if (next === undefined) { + const created: TomlTable = {}; + node[segment] = created; + node = created; + } else if (Array.isArray(next)) { + const last = next.at(-1); + if (!isTable(last)) throw new Error(`cannot descend into array at ${segment}`); + node = last; + } else if (isTable(next)) { + node = next; + } else { + throw new Error(`key ${segment} is not a table`); + } + } + return node; +} + +function isTable(value: unknown): value is TomlTable { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseKeyPath(text: string, line: number): string[] { + const parts = text + .split(".") + .map((part) => part.trim()) + .map((part) => (part.startsWith('"') ? parseString(part, line) : part)); + if (parts.some((part) => part.length === 0)) throw new TomlError(`invalid key ${text}`, line); + return parts; +} + +function parseValue(raw: string, line: number): TomlValue { + if (raw.length === 0) throw new TomlError("missing value", line); + if (raw.startsWith('"') || raw.startsWith("'")) return parseString(raw, line); + if (raw === "true") return true; + if (raw === "false") return false; + if (raw.startsWith("[")) return parseArray(raw, line); + if (raw.startsWith("{")) throw new TomlError("inline tables are not supported", line); + const number = Number(raw.replaceAll("_", "")); + if (Number.isFinite(number)) return number; + throw new TomlError(`unsupported value ${raw}`, line); +} + +function parseString(raw: string, line: number): string { + const quote = raw[0]; + if (quote !== '"' && quote !== "'") throw new TomlError(`expected string`, line); + if (raw.length < 2 || raw.at(-1) !== quote) throw new TomlError(`unterminated string`, line); + const body = raw.slice(1, -1); + if (quote === "'") return body; + return body.replaceAll(/\\(["\\nrt])/gu, (_, code: string) => + code === "n" ? "\n" : code === "r" ? "\r" : code === "t" ? "\t" : code, + ); +} + +function parseArray(raw: string, line: number): TomlValue[] { + if (!raw.endsWith("]")) throw new TomlError("unterminated array", line); + const inner = raw.slice(1, -1).trim(); + if (inner.length === 0) return []; + return splitTopLevel(inner, line).map((item) => parseValue(item.trim(), line)); +} + +function splitTopLevel(text: string, line: number): string[] { + const items: string[] = []; + let depth = 0; + let quote: string | null = null; + let start = 0; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === "[") depth++; + else if (char === "]") depth--; + else if (char === "," && depth === 0) { + items.push(text.slice(start, index)); + start = index + 1; + } + } + if (quote !== null || depth !== 0) throw new TomlError("unbalanced array", line); + const tail = text.slice(start).trim(); + if (tail.length > 0) items.push(tail); + return items; +} + +function findUnquoted(text: string, needle: string): number { + let quote: string | null = null; + for (let index = 0; index < text.length; index++) { + const char = text[index]!; + if (quote !== null) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === needle) return index; + } + return -1; +} + +function stripComment(line: string): string { + const hash = findUnquoted(line, "#"); + return hash === -1 ? line : line.slice(0, hash); +} diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts new file mode 100644 index 0000000000..372ad58969 --- /dev/null +++ b/packages/eve-bench/src/core/trial.ts @@ -0,0 +1,188 @@ +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Writable } from "node:stream"; + +import { buildImage, Container, containerArch, pullImage, type ProcessResult } from "./docker.ts"; +import type { Harness, HarnessBundle } from "./harness.ts"; +import type { StepResult, TrialResult, Usage } from "./result.ts"; +import type { Task } from "./task.ts"; + +export interface TrialInput { + readonly task: Task; + readonly attempt: number; + readonly harness: Harness; + readonly bundle: HarnessBundle; + readonly model: string; + /** Host env forwarded to the harness exec only (provider credentials). */ + readonly forwardEnv: Readonly>; + readonly dir: string; + readonly signal: AbortSignal; +} + +const INSTALL_DIR = "/installed-agent"; +const LOGS_DIR = "/logs"; +const TESTS_DIR = "/tests"; + +export async function runTrial(input: TrialInput): Promise { + const { task, signal } = input; + const startedAt = new Date().toISOString(); + await mkdir(input.dir, { recursive: true }); + const containerLog = createWriteStream(join(input.dir, "container.log"), { flags: "a" }); + let container: Container | undefined; + let agent: StepResult = skipped(); + let verifier: StepResult = skipped(); + let reward: number | null = null; + let usage: Usage | undefined; + let error: string | undefined; + try { + const image = await resolveImage(task, containerLog, signal); + container = await Container.start( + { + image, + name: `eve-bench-${task.name}-${input.attempt}-${Date.now().toString(36)}`, + cpus: task.environment.cpus, + memoryMb: task.environment.memoryMb, + network: task.environment.allowInternet, + }, + signal, + ); + const workdir = await resolveWorkdir(container); + const bundleArch = input.bundle.arch; + const arch = bundleArch ? await containerArch(container) : undefined; + await container.exec( + `mkdir -p ${INSTALL_DIR} ${LOGS_DIR}/agent ${LOGS_DIR}/verifier ${TESTS_DIR}${arch ? ` ${INSTALL_DIR}/arch` : ""}`, + { timeoutMs: 30_000, signal }, + ); + await container.upload(`${input.bundle.dir}/.`, INSTALL_DIR, signal); + if (arch && bundleArch) { + await container.upload(`${bundleArch[arch]}/.`, `${INSTALL_DIR}/arch`, signal); + } + for (const path of input.harness.stage?.({ taskDir: task.dir }) ?? []) { + await container.upload(path, INSTALL_DIR, signal); + } + const instructionPath = join(input.dir, "instruction.md"); + await writeFile(instructionPath, task.instruction); + await container.upload(instructionPath, `${INSTALL_DIR}/instruction.md`, signal); + + const runContext = { + model: input.model, + installDir: INSTALL_DIR, + instructionPath: `${INSTALL_DIR}/instruction.md`, + taskWorkdir: workdir, + logsDir: `${LOGS_DIR}/agent`, + }; + agent = await step(() => + container!.exec(input.harness.command(runContext), { + cwd: workdir, + env: { ...input.forwardEnv, ...input.harness.env(runContext) }, + timeoutMs: task.agentTimeoutMs, + signal, + log: createWriteStream(join(input.dir, "agent.log")), + }), + ); + + await container.upload(`${join(task.dir, "tests")}/.`, TESTS_DIR, signal); + verifier = await step(() => + container!.exec(`bash ${TESTS_DIR}/test.sh`, { + cwd: workdir, + timeoutMs: task.verifierTimeoutMs, + signal, + log: createWriteStream(join(input.dir, "verifier.log")), + }), + ); + + for (const logs of ["agent", "verifier"]) { + if (!(await container.download(`${LOGS_DIR}/${logs}`, input.dir))) { + containerLog.write(`[eve-bench] could not download ${LOGS_DIR}/${logs}\n`); + } + } + reward = await readReward(join(input.dir, "verifier", "reward.txt")); + usage = await readUsage(join(input.dir, "agent", "result.json")); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } finally { + if (container) await container.remove(); + containerLog.end(); + } + const result: TrialResult = { + task: task.name, + attempt: input.attempt, + harness: input.harness.name, + model: input.model, + reward, + agent, + verifier, + ...(usage ? { usage } : {}), + ...(error ? { error } : {}), + startedAt, + finishedAt: new Date().toISOString(), + }; + await writeFile(join(input.dir, "trial.json"), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +async function resolveImage(task: Task, log: Writable, signal: AbortSignal): Promise { + const options = { timeoutMs: task.buildTimeoutMs, signal, log }; + if (task.environment.dockerImage) { + await pullImage(task.environment.dockerImage, options); + return task.environment.dockerImage; + } + const dockerfile = await readFile(join(task.environment.dockerfileDir, "Dockerfile")); + const tag = `eve-bench/${task.name}:${createHash("sha256").update(dockerfile).digest("hex").slice(0, 12)}`; + await buildImage(tag, task.environment.dockerfileDir, options); + return tag; +} + +async function resolveWorkdir(container: Container): Promise { + const result = await container.exec("pwd", { timeoutMs: 10_000 }); + const workdir = result.stdout.trim(); + if (result.exitCode !== 0 || workdir.length === 0) { + throw new Error(`Could not resolve the task working directory: ${result.stderr.trim()}`); + } + return workdir; +} + +async function step(run: () => Promise): Promise { + const started = Date.now(); + const result = await run(); + return { + status: result.timedOut + ? "timeout" + : result.aborted || result.exitCode !== 0 + ? "failed" + : "completed", + exitCode: result.exitCode, + durationMs: Date.now() - started, + }; +} + +function skipped(): StepResult { + return { status: "skipped", exitCode: null, durationMs: 0 }; +} + +async function readReward(path: string): Promise { + try { + const value = Number((await readFile(path, "utf8")).trim()); + return Number.isFinite(value) ? value : null; + } catch { + return null; + } +} + +async function readUsage(path: string): Promise { + try { + const parsed = JSON.parse(await readFile(path, "utf8")) as { usage?: Partial }; + const usage = parsed.usage; + if (!usage) return undefined; + return { + inputTokens: usage.inputTokens ?? 0, + outputTokens: usage.outputTokens ?? 0, + cachedTokens: usage.cachedTokens ?? 0, + costUsd: usage.costUsd ?? 0, + }; + } catch { + return undefined; + } +} diff --git a/packages/eve-bench/src/harnesses/eve/index.ts b/packages/eve-bench/src/harnesses/eve/index.ts new file mode 100644 index 0000000000..55dcc4dabf --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/index.ts @@ -0,0 +1,247 @@ +import { execFile } from "node:child_process"; +import { createHash, type Hash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { chmod, cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { + Harness, + HarnessBundle, + HarnessPrepareContext, + HarnessRunContext, +} from "../../core/harness.ts"; + +const execFileAsync = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const packageRoot = resolve(here, "../../.."); +const repoRoot = resolve(packageRoot, "../.."); +const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; + +export type EveSource = + | { readonly kind: "local" } + | { readonly kind: "release"; readonly version: string }; + +/** + * Builds the `agent/` app against a local or released eve, and packages the + * Nitro output and runner with architecture-specific Linux Node binaries. + * Cached by content hash. + */ +export function createEveHarness(source: EveSource): Harness { + return { + name: source.kind === "local" ? "eve@local" : `eve@${source.version}`, + prepare: (ctx) => prepare(source, ctx), + command: (ctx) => `sh ${ctx.installDir}/run.sh`, + env: (ctx: HarnessRunContext) => ({ + EVE_PROJECT_DIR: `${ctx.installDir}/app`, + EVE_INSTRUCTION_PATH: ctx.instructionPath, + EVE_AGENT_LOGS_DIR: ctx.logsDir, + EVE_BENCH_MODEL: ctx.model, + EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, + }), + }; +} + +async function prepare(source: EveSource, ctx: HarnessPrepareContext): Promise { + const eve: ResolvedEve = + source.kind === "local" + ? await resolveLocalEve() + : { + dependency: source.version, + identity: source.version, + provenance: { kind: "release", version: source.version }, + }; + + const hash = createHash("sha256"); + hash.update(eve.identity).update(ctx.model).update(NODE_VERSION); + for (const file of await listFiles(join(packageRoot, "agent"))) + hash.update(file).update(await readFile(join(packageRoot, "agent", file))); + hash.update(await readFile(join(here, "runner.mjs"))).update(RUN_SH); + const key = hash.digest("hex").slice(0, 16); + const bundleDir = join(ctx.cacheDir, "bundles", key); + const provenance = { key, model: ctx.model, eve: eve.provenance }; + const nodeDir = join(ctx.cacheDir, `node-v${NODE_VERSION}`); + await Promise.all(ARCHES.map((arch) => ensureNode(nodeDir, arch))); + const arch = Object.fromEntries(ARCHES.map((name) => [name, join(nodeDir, name)])) as Record< + (typeof ARCHES)[number], + string + >; + if (await exists(join(bundleDir, "run.sh"))) return { dir: bundleDir, arch, provenance }; + + const building = `${bundleDir}.building`; + await rm(building, { recursive: true, force: true }); + const app = join(building, "app"); + await mkdir(app, { recursive: true }); + await cp(join(packageRoot, "agent"), join(app, "agent"), { recursive: true }); + await writeFile( + join(app, "package.json"), + `${JSON.stringify({ name: "eve-bench-app", private: true, type: "module", dependencies: { eve: eve.dependency, zod: "4.4.3" } }, null, 2)}\n`, + ); + await writeFile(join(app, "tsconfig.json"), APP_TSCONFIG); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + if (source.kind === "local") { + const artifacts = join(ctx.cacheDir, "artifacts"); + await mkdir(artifacts, { recursive: true }); + await cp(await packLocalEve(artifacts), join(app, "eve.tgz")); + } + const npm = (...args: string[]) => + execFileAsync("npm", [...args, "--ignore-scripts", "--userconfig", join(app, ".npmrc")], { + cwd: app, + maxBuffer: 64 * 1024 * 1024, + }); + await npm("install", "--package-lock-only"); + await npm("ci"); + await execFileAsync(join(app, "node_modules", ".bin", "eve"), ["build"], { + cwd: app, + env: { ...process.env, EVE_BENCH_MODEL: ctx.model, EVE_BENCH_TASK_WORKDIR: "/workspace" }, + maxBuffer: 64 * 1024 * 1024, + }); + await rm(join(app, "agent"), { recursive: true, force: true }); + await rm(join(app, "package-lock.json"), { force: true }); + await rm(join(app, ".npmrc"), { force: true }); + await rm(join(app, "eve.tgz"), { force: true }); + await rm(join(app, "tsconfig.json"), { force: true }); + await rm(join(app, ".eve"), { recursive: true, force: true }); + // Nitro's `.output` is self-contained and the runner speaks plain HTTP. + await rm(join(app, "node_modules"), { recursive: true, force: true }); + await cp(join(here, "runner.mjs"), join(app, "runner.mjs")); + await writeFile(join(building, "run.sh"), RUN_SH, { mode: 0o755 }); + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await rm(bundleDir, { recursive: true, force: true }); + await rename(building, bundleDir); + return { dir: bundleDir, arch, provenance }; +} + +interface ResolvedEve { + readonly dependency: string; + readonly identity: string; + readonly provenance: Record; +} + +async function resolveLocalEve(): Promise { + const evePackage = join(repoRoot, "packages/eve"); + const dist = join(evePackage, "dist"); + if (!(await isDirectory(dist))) { + throw new Error( + "Local eve dist is missing. Run `pnpm build` or `pnpm --filter eve build` first.", + ); + } + + const hash = createHash("sha256"); + hash.update("package.json\0").update(await readFile(join(evePackage, "package.json"))); + for (const file of await listFiles(dist)) { + hash.update("\0").update(file).update("\0"); + await hashFile(hash, join(dist, file)); + } + const distSha256 = hash.digest("hex"); + const [gitSha, status] = await Promise.all([ + execFileAsync("git", ["rev-parse", "HEAD"], { cwd: repoRoot }), + execFileAsync("git", ["status", "--porcelain", "packages/eve"], { cwd: repoRoot }), + ]); + return { + dependency: "file:eve.tgz", + identity: distSha256, + provenance: { + kind: "local", + gitSha: gitSha.stdout.trim(), + dirty: status.stdout.trim().length > 0, + distSha256, + }, + }; +} + +async function packLocalEve(artifacts: string): Promise { + const { stdout } = await execFileAsync( + "pnpm", + ["--filter", "eve", "pack", "--pack-destination", artifacts, "--json"], + { cwd: repoRoot, maxBuffer: 16 * 1024 * 1024 }, + ); + const jsonStart = stdout.indexOf("{"); + const packed = JSON.parse(stdout.slice(jsonStart)) as { filename: string }; + return resolve(artifacts, packed.filename); +} + +async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { + const archDir = join(nodeDir, arch); + const target = join(archDir, "node"); + if (await exists(target)) return; + await mkdir(archDir, { recursive: true }); + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const url = `https://nodejs.org/dist/v${NODE_VERSION}/${name}.tar.xz`; + const response = await fetch(url); + if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status}`); + const archive = join(nodeDir, `${name}.tar.xz`); + await writeFile(archive, Buffer.from(await response.arrayBuffer())); + const extractDir = join(nodeDir, `${name}.extract`); + await mkdir(extractDir, { recursive: true }); + await execFileAsync("tar", [ + "-xJf", + archive, + "--strip-components=2", + "-C", + extractDir, + `${name}/bin/node`, + ]); + await rename(join(extractDir, "node"), target); + await chmod(target, 0o755); + await rm(extractDir, { recursive: true, force: true }); + await rm(archive, { force: true }); +} + +async function listFiles(dir: string, prefix = ""): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) files.push(...(await listFiles(join(dir, entry.name), rel))); + else files.push(rel); + } + return files; +} + +async function hashFile(hash: Hash, path: string): Promise { + for await (const chunk of createReadStream(path)) hash.update(chunk); +} + +async function isDirectory(path: string): Promise { + return stat(path).then( + (entry) => entry.isDirectory(), + () => false, + ); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} + +const RUN_SH = `#!/bin/sh +set -eu +dir=$(cd "$(dirname "$0")" && pwd) +exec "$dir/arch/node" "$dir/app/runner.mjs" +`; + +const APP_TSCONFIG = `${JSON.stringify( + { + compilerOptions: { + target: "ES2024", + lib: ["ES2024"], + module: "NodeNext", + moduleResolution: "NodeNext", + moduleDetection: "force", + verbatimModuleSyntax: true, + noEmit: true, + strict: true, + isolatedModules: true, + skipLibCheck: true, + types: ["node"], + }, + include: ["agent/**/*.ts"], + }, + null, + 2, +)}\n`; diff --git a/packages/eve-bench/src/harnesses/eve/runner.mjs b/packages/eve-bench/src/harnesses/eve/runner.mjs new file mode 100644 index 0000000000..d2e593afca --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/runner.mjs @@ -0,0 +1,242 @@ +// Runs inside the task container: boots the built eve server, sends the task +// instruction as one session, follows the event stream to its boundary, and +// writes /logs/agent/{events.ndjson,result.json,server.log}. Talks to the +// server over plain HTTP so the bundle ships no node_modules. +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const projectDir = requireEnv("EVE_PROJECT_DIR"); +const instructionPath = requireEnv("EVE_INSTRUCTION_PATH"); +const logsDir = process.env.EVE_AGENT_LOGS_DIR ?? "/logs/agent"; +const host = process.env.EVE_HOST ?? "127.0.0.1"; +const port = Number(process.env.EVE_PORT ?? "3024"); +const serverUrl = `http://${host}:${port}`; +const startupTimeoutMs = Number(process.env.EVE_STARTUP_TIMEOUT_MS ?? "60000"); + +await mkdir(logsDir, { recursive: true }); +const eventsPath = join(logsDir, "events.ndjson"); +const resultPath = join(logsDir, "result.json"); +const serverLogPath = join(logsDir, "server.log"); + +let server; +let wroteResult = false; +try { + server = startServer(); + await waitForHealth(server); + + const instruction = await readFile(instructionPath, "utf8"); + const sessionId = await createSession(instruction); + const events = await followStream(sessionId); + + await writeFile(eventsPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); + const status = statusFromEvents(events); + const summary = { + status, + sessionId, + message: finalMessage(events), + inputRequests: events + .filter((event) => event.type === "input.requested") + .flatMap((event) => event.data?.requests ?? []), + usage: summarizeUsage(events), + eventCount: events.length, + failure: failureFromEvents(events) ?? failureFromStatus(status), + }; + await writeFile(resultPath, `${JSON.stringify(summary, null, 2)}\n`); + wroteResult = true; + if (summary.failure) throw new Error(summary.failure.message); +} catch (error) { + if (!wroteResult) { + await writeFile( + resultPath, + `${JSON.stringify( + { + status: "failed", + inputRequests: [], + usage: {}, + eventCount: 0, + failure: { type: "runner.error", message: error?.message ?? String(error) }, + }, + null, + 2, + )}\n`, + ); + } + throw error; +} finally { + if (server) await stopServer(server); +} + +function requireEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} must be set`); + return value; +} + +function startServer() { + const log = createWriteStream(serverLogPath, { flags: "a" }); + const entry = join(projectDir, ".output", "server", "index.mjs"); + const child = spawn(process.execPath, [entry], { + cwd: projectDir, + env: { + ...process.env, + EVE_DEV: "1", + HOST: host, + NITRO_HOST: host, + NITRO_PORT: String(port), + PORT: String(port), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + child.once("exit", (code, signal) => { + log.write(`\n[eve-server exited code=${code ?? ""} signal=${signal ?? ""}]\n`); + }); + return child; +} + +async function waitForHealth(child) { + const deadline = Date.now() + startupTimeoutMs; + let lastError; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`eve server exited before becoming healthy (code ${child.exitCode})`); + } + try { + const response = await fetch(`${serverUrl}/eve/v1/health`); + if (response.ok) return; + lastError = new Error(`health returned ${response.status}`); + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for eve health: ${lastError?.message ?? "unknown"}`); +} + +async function createSession(message) { + const response = await fetch(`${serverUrl}/eve/v1/session`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message }), + }); + if (!response.ok) + throw new Error(`session create failed: ${response.status} ${await response.text()}`); + const payload = await response.json(); + const sessionId = payload?.sessionId ?? response.headers.get("x-eve-session-id"); + if (!sessionId) throw new Error("session create returned no session id"); + return sessionId; +} + +/** Reads the durable NDJSON stream until a session boundary, reconnecting from the cursor if the transport ends first. */ +async function followStream(sessionId) { + const events = []; + for (;;) { + const url = new URL(`${serverUrl}/eve/v1/session/${encodeURIComponent(sessionId)}/stream`); + if (events.length > 0) url.searchParams.set("startIndex", String(events.length)); + const response = await fetch(url); + if (!response.ok || !response.body) + throw new Error(`stream failed: ${response.status} ${await response.text()}`); + for await (const event of ndjson(response.body)) { + events.push(event); + if (isBoundary(event)) return events; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } +} + +async function* ndjson(body) { + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of body) { + buffer += decoder.decode(chunk, { stream: true }); + let newline; + while ((newline = buffer.indexOf("\n")) !== -1) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) yield JSON.parse(line); + } + } + if (buffer.trim()) yield JSON.parse(buffer); +} + +function isBoundary(event) { + return ( + event.type === "session.completed" || + event.type === "session.failed" || + event.type === "session.waiting" + ); +} + +function statusFromEvents(events) { + const boundary = events.findLast(isBoundary); + if (boundary?.type === "session.completed") return "completed"; + if (boundary?.type === "session.waiting") return "waiting"; + if (boundary?.type === "session.failed") return "failed"; + return "unknown"; +} + +function finalMessage(events) { + return events.findLast( + (event) => event.type === "message.completed" && event.data?.finishReason !== "tool-calls", + )?.data?.message; +} + +function failureFromEvents(events) { + for (const event of events) { + if (event.type === "input.requested") { + return { + type: event.type, + message: "eve requested human input during a non-interactive benchmark run", + data: event.data, + }; + } + if (event.type === "turn.failed" || event.type === "session.failed") { + const message = + event.data?.message ?? event.data?.error?.message ?? `${event.type} emitted during eve run`; + return { type: event.type, message, data: event.data }; + } + } + return null; +} + +function failureFromStatus(status) { + if (status === "waiting" || status === "completed") return null; + return { type: "session.unsettled", message: `eve run ended with status: ${status}` }; +} + +function summarizeUsage(events) { + const usage = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: 0 }; + for (const event of events) { + if (event.type !== "step.completed") continue; + const source = event.data?.usage ?? event.data ?? {}; + usage.inputTokens += numberValue(source.inputTokens, source.input_tokens); + usage.outputTokens += numberValue(source.outputTokens, source.output_tokens); + usage.cachedTokens += numberValue( + source.cacheReadTokens, + source.cachedTokens, + source.cached_tokens, + ); + usage.costUsd += numberValue(source.costUsd, source.cost_usd); + } + return usage; +} + +function numberValue(...values) { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return 0; +} + +async function stopServer(child) { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", resolve)), + new Promise((resolve) => setTimeout(resolve, 5000)), + ]); + if (child.exitCode === null) child.kill("SIGKILL"); +} diff --git a/packages/eve-bench/src/harnesses/oracle.ts b/packages/eve-bench/src/harnesses/oracle.ts new file mode 100644 index 0000000000..1389f91a22 --- /dev/null +++ b/packages/eve-bench/src/harnesses/oracle.ts @@ -0,0 +1,23 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import type { Harness } from "../core/harness.ts"; + +/** + * Runs each task's reference `solution/solve.sh`. Validates the runner and + * verifier end to end without a model, and gives the reward ceiling a real + * harness is measured against. + */ +export function createOracleHarness(): Harness { + return { + name: "oracle", + async prepare(ctx) { + const dir = join(ctx.cacheDir, "bundles", "oracle"); + await mkdir(dir, { recursive: true }); + return { dir }; + }, + stage: (ctx) => [join(ctx.taskDir, "solution")], + command: (ctx) => `bash ${ctx.installDir}/solution/solve.sh`, + env: () => ({}), + }; +} diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts new file mode 100644 index 0000000000..7f028379f8 --- /dev/null +++ b/packages/eve-bench/src/options.ts @@ -0,0 +1,98 @@ +import { readdir } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts"; +import type { Harness } from "./core/harness.ts"; +import { createEveHarness } from "./harnesses/eve/index.ts"; +import { createOracleHarness } from "./harnesses/oracle.ts"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +export const paths = { + packageRoot, + datasetsRoot: join(packageRoot, "datasets"), + generatedRoot: join(packageRoot, ".generated"), +} as const; + +export interface TaskSelection { + readonly lock: DatasetLock; + readonly tasks: readonly string[]; +} + +export async function selectTasks(input: { + cohort?: string; + dataset?: string; + task?: readonly string[]; +}): Promise { + if (input.cohort) { + const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); + const entry = await lock(cohort.dataset); + const tasks = input.task?.length + ? cohort.tasks.filter((task) => input.task!.includes(task)) + : cohort.tasks; + return { lock: entry, tasks }; + } + + const entry = await lock(input.dataset ?? (await defaultDataset())); + const tasks = input.task?.length + ? entry.tasks.filter((task) => input.task!.includes(task)) + : entry.tasks; + if (input.task?.length && tasks.length !== input.task.length) { + const missing = input.task.filter((task) => !tasks.includes(task)); + throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); + } + return { lock: entry, tasks }; +} + +export function jobDir(nameOrPath: string): string { + return nameOrPath.includes("/") + ? resolve(nameOrPath) + : join(paths.generatedRoot, "jobs", nameOrPath); +} + +export async function lock(name: string): Promise { + return readDatasetLock(join(paths.datasetsRoot, `${name}.json`)); +} + +export async function allLocks(): Promise { + const files = (await readdir(paths.datasetsRoot)).filter((file) => file.endsWith(".json")); + return Promise.all(files.map((file) => readDatasetLock(join(paths.datasetsRoot, file)))); +} + +export function defaultJobName(label: string): string { + const stamp = new Date().toISOString().replaceAll(/[:.]/gu, "-").slice(0, 19); + return `${stamp}-${label.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}`; +} + +export function selectHarness(name: string, eve: string): Harness { + if (name === "oracle") return createOracleHarness(); + if (name === "eve") { + return createEveHarness( + eve === "local" ? { kind: "local" } : { kind: "release", version: eve }, + ); + } + throw new Error(`unknown harness: ${name}`); +} + +export function forwardedEnv(): Record { + const env: Record = {}; + for (const key of [ + "AI_GATEWAY_API_KEY", + "VERCEL_OIDC_TOKEN", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + ]) { + const value = process.env[key]; + if (value) env[key] = value; + } + return env; +} + +async function defaultDataset(): Promise { + const locks = await allLocks(); + if (locks.length !== 1) { + throw new Error("pass --dataset or --cohort; multiple datasets are available"); + } + return `${locks[0]!.name}-${locks[0]!.version}`; +} diff --git a/packages/eve-bench/src/tools/diff_bench.ts b/packages/eve-bench/src/tools/diff_bench.ts new file mode 100644 index 0000000000..ff59f82988 --- /dev/null +++ b/packages/eve-bench/src/tools/diff_bench.ts @@ -0,0 +1,25 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { diffJobs } from "../core/report.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ + base: z.string().min(1), + candidate: z.string().min(1), +}); + +export default defineTool({ + description: "Compare task rewards between two eve benchmark jobs.", + inputSchema, + approval: never(), + async execute(input) { + const [base, candidate] = await Promise.all([ + readJobResult(jobDir(input.base)), + readJobResult(jobDir(input.candidate)), + ]); + return diffJobs(base, candidate); + }, +}); diff --git a/packages/eve-bench/src/tools/get_bench_result.ts b/packages/eve-bench/src/tools/get_bench_result.ts new file mode 100644 index 0000000000..a3ff347ffc --- /dev/null +++ b/packages/eve-bench/src/tools/get_bench_result.ts @@ -0,0 +1,24 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { readJobResult } from "../core/job.ts"; +import { jobDir } from "../options.ts"; + +export const inputSchema = z.object({ job: z.string().min(1) }); + +export default defineTool({ + description: "Read a completed eve benchmark job result.", + inputSchema, + approval: never(), + async execute(input) { + try { + return await readJobResult(jobDir(input.job)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`benchmark result not found for job "${input.job}"`); + } + throw error; + } + }, +}); diff --git a/packages/eve-bench/src/tools/list_bench_tasks.ts b/packages/eve-bench/src/tools/list_bench_tasks.ts new file mode 100644 index 0000000000..ad2460a546 --- /dev/null +++ b/packages/eve-bench/src/tools/list_bench_tasks.ts @@ -0,0 +1,19 @@ +import { defineTool } from "eve/tools"; +import { never } from "eve/tools/approval"; +import { z } from "zod"; + +import { selectTasks } from "../options.ts"; + +export const inputSchema = z.object({ + cohort: z.string().min(1).optional(), + dataset: z.string().min(1).optional(), +}); + +export default defineTool({ + description: "List task names in an eve benchmark dataset or cohort.", + inputSchema, + approval: never(), + async execute(input) { + return (await selectTasks(input)).tasks; + }, +}); diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts new file mode 100644 index 0000000000..09f410af5f --- /dev/null +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { syncDataset, taskDirs } from "../core/dataset.ts"; +import { runJob } from "../core/job.ts"; +import { loadTask } from "../core/task.ts"; +import { defaultJobName, forwardedEnv, paths, selectHarness, selectTasks } from "../options.ts"; + +export const inputSchema = z + .object({ + cohort: z.string().min(1).optional(), + tasks: z.array(z.string().min(1)).min(1).optional(), + model: z.string().min(1).optional(), + harness: z.enum(["eve", "oracle"]).default("eve"), + eve: z.string().min(1).default("local"), + attempts: z.number().int().positive().default(1), + concurrency: z.number().int().positive().default(4), + job: z.string().min(1).optional(), + }) + .superRefine((input, ctx) => { + if (input.harness !== "oracle" && !input.model) { + ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); + } + }); + +export default defineTool({ + description: "Run an eve benchmark job and return its result.", + inputSchema, + async execute(input, ctx) { + const harness = selectHarness(input.harness, input.eve); + const model = input.model ?? "none"; + const selection = await selectTasks({ cohort: input.cohort, task: input.tasks }); + const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const tasks = await Promise.all( + (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + ); + const name = input.job ?? defaultJobName(`${harness.name}-${model}`); + + return runJob({ + name, + dir: join(paths.generatedRoot, "jobs", name), + cacheDir: join(paths.generatedRoot, "cache"), + dataset: selection.lock, + tasks, + harness, + model, + attempts: input.attempts, + concurrency: input.concurrency, + forwardEnv: forwardedEnv(), + signal: ctx.abortSignal, + }); + }, +}); diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts new file mode 100644 index 0000000000..41515610e8 --- /dev/null +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { inputSchema as diffBenchInput } from "./diff_bench.ts"; +import { inputSchema as getBenchResultInput } from "./get_bench_result.ts"; +import { inputSchema as listBenchTasksInput } from "./list_bench_tasks.ts"; +import { inputSchema as runBenchInput } from "./run_bench.ts"; + +describe("run_bench input", () => { + it("accepts valid eve and oracle inputs", () => { + assert.equal( + runBenchInput.safeParse({ + cohort: "smoke", + tasks: ["fix-git"], + model: "zai/glm-5.2", + harness: "eve", + eve: "local", + attempts: 2, + concurrency: 1, + job: "t1", + }).success, + true, + ); + assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); + }); + + it("rejects invalid inputs", () => { + assert.equal(runBenchInput.safeParse({}).success, false); + assert.equal(runBenchInput.safeParse({ harness: "other", model: "model" }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", attempts: 0 }).success, false); + assert.equal(runBenchInput.safeParse({ harness: "oracle", tasks: "fix-git" }).success, false); + }); +}); + +describe("get_bench_result input", () => { + it("accepts a job name", () => { + assert.equal(getBenchResultInput.safeParse({ job: "smoke-run" }).success, true); + }); + + it("rejects a missing or empty job name", () => { + assert.equal(getBenchResultInput.safeParse({}).success, false); + assert.equal(getBenchResultInput.safeParse({ job: "" }).success, false); + }); +}); + +describe("diff_bench input", () => { + it("accepts base and candidate jobs", () => { + assert.equal( + diffBenchInput.safeParse({ base: "base-run", candidate: "candidate-run" }).success, + true, + ); + }); + + it("rejects incomplete input", () => { + assert.equal(diffBenchInput.safeParse({ base: "base-run" }).success, false); + assert.equal(diffBenchInput.safeParse({ base: 1, candidate: "candidate-run" }).success, false); + }); +}); + +describe("list_bench_tasks input", () => { + it("accepts empty, cohort, and dataset selections", () => { + assert.equal(listBenchTasksInput.safeParse({}).success, true); + assert.equal(listBenchTasksInput.safeParse({ cohort: "smoke" }).success, true); + assert.equal(listBenchTasksInput.safeParse({ dataset: "terminal-bench-2.0" }).success, true); + }); + + it("rejects invalid selections", () => { + assert.equal(listBenchTasksInput.safeParse({ cohort: "" }).success, false); + assert.equal(listBenchTasksInput.safeParse({ dataset: ["terminal-bench-2.0"] }).success, false); + }); +}); diff --git a/packages/eve-bench/tsconfig.json b/packages/eve-bench/tsconfig.json new file mode 100644 index 0000000000..807ccbdd2d --- /dev/null +++ b/packages/eve-bench/tsconfig.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": ["ES2024"], + "rootDir": ".", + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": false + }, + "include": ["agent/**/*.ts", "src/**/*.ts"], + "exclude": [] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61cc596628..804d5fd3c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1260,6 +1260,22 @@ importers: specifier: 5.0.0 version: 5.0.0(zod@4.4.3) + packages/eve-bench: + dependencies: + eve: + specifier: workspace:* + version: link:../eve + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + typescript: + specifier: 'catalog:' + version: 7.0.2 + packages/eve-buzz-acp-adapter: dependencies: eve: diff --git a/research/eve-bench-runner.md b/research/eve-bench-runner.md new file mode 100644 index 0000000000..e290609aae --- /dev/null +++ b/research/eve-bench-runner.md @@ -0,0 +1,238 @@ +--- +issue: TODO +status: approved +last_updated: "2026-09-09" +--- + +# eve-bench: a zero-dependency benchmark runner + +`packages/eve-bench` currently benchmarks the eve harness on Terminal-Bench +through Harbor: a Python adapter subclasses Harbor's private `Eve` agent, a +Node runner drives the built app, and a materialization script produces a +self-contained app directory (two Node binaries, a rolldown bundle, `npm ci`) +because Harbor copies the agent directory into every task container. + +That stack needs three runtimes (Python, Harbor at a pinned revision, Node), +leaks Harbor-specific env (`HARBOR_MODEL`, `EVE_HARBOR_TASK_WORKDIR`) into the +harness under test, breaks on any upstream refactor of Harbor's private +methods, and has no path to a CLI, an agent tool, or a GitHub Action that +does not drag Python along. + +Per trial, Harbor does five things: resolve a task directory, run a Docker +container, exec the agent under a timeout, exec `tests/test.sh` under a +timeout and read `/logs/verifier/reward.txt`, and write a result record. A +job is that loop over tasks × attempts with concurrency and resume. All of it +fits in a small amount of Node with no third-party dependencies. + +## Proposal + +Replace Harbor with an eve-owned runner. Keep the Terminal-Bench task format +and reward contract unchanged so datasets stay interchangeable and results +remain comparable with published Harbor runs. + +``` +packages/eve-bench/ + src/core/ zero-dependency library; does not import eve + task.ts load task.toml + instruction.md + docker.ts container lifecycle over the docker CLI + trial.ts run one attempt: start → agent → verify → record + job.ts matrix, concurrency, attempts, resume + result.ts trial/job JSON schema + report.ts console, json, junit, diff + src/harnesses/eve/ the eve harness: build once, exec in container + src/harnesses/ oracle (reference solutions) and future harnesses + src/cli.ts eve-bench run | tasks | report | diff + src/tools/ eve tools wrapping core for agent consumers (follow-up) + datasets/ pinned dataset locks and cohorts (JSON) + agent/ the harness under test (unchanged) +``` + +The runner is plain `.ts` executed by Node 24 with type stripping; the root +tsconfig already enforces `erasableSyntaxOnly`. No build step, no bundler. + +### Datasets + +A dataset lock pins a git remote, commit, and the task paths within it: + +```json +{ + "name": "terminal-bench", + "version": "2.0", + "gitUrl": "https://github.com/laude-institute/terminal-bench-2.git", + "commit": "69671fbaac6d67a7ef0dfec016cc38a64ef7a77c", + "tasks": ["adaptive-rejection-sampler", "fix-git", "..."] +} +``` + +`eve-bench tasks sync ` clones the pinned commit (shallow, sparse) into +`.generated/datasets/-/`. Cohorts are named task subsets in +`datasets/cohorts/*.json` and reference a lock by name; membership changes +are reviewed like code. + +### Tasks + +A task is a Terminal-Bench directory: `task.toml`, `instruction.md`, +`environment/Dockerfile` (optionally with a prebuilt `docker_image`), +`tests/test.sh`, `solution/`. The loader reads only what the runner needs: + +- `task.name` +- `agent.timeout_sec`, `verifier.timeout_sec`, `environment.build_timeout_sec` +- `environment.docker_image`, `cpus`, `memory_mb`, `allow_internet` + +`task.toml` is parsed by a minimal TOML reader covering the subset the +dataset uses (tables, strings, numbers, booleans, arrays). Anything outside +that subset is a load error naming the line. + +### Environment + +One container per trial, always via the `docker` CLI. Prebuilt +`environment.docker_image` is preferred; otherwise the Dockerfile is built +once per task and cached by image tag. Every operation (`build`, `run`, +`exec`, `cp`, `rm`) accepts an `AbortSignal`; SIGINT to the runner aborts +in-flight trials and removes their containers before exiting. + +### Harness contract + +A harness is something that can be placed in a container and told to solve +an instruction. It is a spawned shell, nothing more: + +```ts +interface Harness { + readonly name: string; + /** Runs once per job on the host; returns a directory to place in the container. */ + prepare(ctx: { model: string; cacheDir: string }): Promise<{ dir: string; provenance?: object }>; + /** Optional per-task host paths uploaded into the install directory. */ + stage?(ctx: { taskDir: string }): readonly string[]; + /** Shell command executed inside the container in the task working directory. */ + command(ctx: { + installDir: string; + instructionPath: string; + taskWorkdir: string; + logsDir: string; + }): string; + /** Env visible to the command. Provider credentials are forwarded by the runner. */ + env(ctx: { model: string }): Record; +} +``` + +The bundle directory is uploaded to `/installed-agent`. The eve harness +bundles the Nitro output built from `agent/`, a runner, and Linux Node +binaries, cached by content hash of `agent/`, the eve source, and the model. +The runner drives the server over plain HTTP (`POST /eve/v1/session`, then +`GET /eve/v1/session/:id/stream`), so the bundle carries no `node_modules` +and no bundler is involved. It writes `/logs/agent/events.ndjson` and +`/logs/agent/result.json`. The env the harness reads is `EVE_BENCH_MODEL` and +`EVE_BENCH_TASK_WORKDIR`. + +An `oracle` harness runs each task's `solution/solve.sh` via `stage`. It +validates the runner without a model and is the reward ceiling a real harness +is measured against. A harness wrapping another CLI agent is the same +interface and enables side-by-side comparison. + +### Trial lifecycle + +``` +resolve task ─▶ start container ─▶ upload harness ─▶ exec harness (agent timeout) + │ + record ◀── read reward.txt ◀── exec tests/test.sh ◀─────┘ + (verifier timeout) +``` + +The verifier always runs, including after an agent timeout or failure, so a +trial's reward reflects the filesystem the agent left behind. Container +removal is unconditional. Each trial writes to its own directory: + +``` +.generated/jobs//// + trial.json the result record below + agent/ copied /logs/agent + verifier/ copied /logs/verifier + container.log docker output for the trial +``` + +Resume skips trial directories that already contain a `trial.json`. + +### Result schema + +```ts +interface TrialResult { + readonly task: string; + readonly attempt: number; + readonly harness: string; + readonly model: string; + readonly reward: number | null; // null when the verifier produced none + readonly agent: StepResult; + readonly verifier: StepResult; + readonly usage?: { + inputTokens: number; + outputTokens: number; + cachedTokens: number; + costUsd: number; + }; + readonly error?: string; + readonly startedAt: string; + readonly finishedAt: string; +} + +interface StepResult { + readonly status: "completed" | "timeout" | "failed" | "skipped"; + readonly exitCode: number | null; + readonly durationMs: number; +} + +interface JobResult { + readonly job: string; + readonly dataset: { name: string; version: string; commit: string }; + readonly harness: string; + readonly model: string; + readonly trials: readonly TrialResult[]; + readonly summary: { tasks: number; attempts: number; resolved: number; meanReward: number }; +} +``` + +This schema is the API. The CLI, the GitHub Action, and agent tools all read +and write it; nothing else is stable. + +### CLI + +``` +eve-bench tasks sync fetch a pinned dataset +eve-bench tasks list --cohort smoke print task names +eve-bench run --cohort smoke --model zai/glm-5.2 [--attempts 3] [--concurrency 8] [--task fix-git ...] + [--harness eve|oracle] [--eve local|] [--job ] # same --job resumes +eve-bench report [--format console|json|junit] +eve-bench diff per-task reward deltas +``` + +Every command accepts `--json` and never prompts. Provider credentials come +from the environment (`AI_GATEWAY_API_KEY`) and are forwarded into the +container only for the harness exec, never written to disk. + +### Consumers + +- CI: a workflow runs `eve-bench run --cohort smoke --json`, uploads the job + directory as an artifact, and posts `eve-bench diff` against the base + branch's artifact. +- Humans: the same CLI with the console reporter. +- Agents: `src/tools/` exposes `run_trial`, `get_result`, and `diff` as eve + tools over `core`; they return the schema above. + +## Invariants + +- `src/core` has no dependency on `eve` or any third-party package; only the + eve harness imports eve. +- Task format, verifier contract, and reward semantics are Terminal-Bench's; + eve-bench adds nothing to a task directory. +- A harness never sees runner internals beyond the contract above. +- Trial directories are append-only; resume never rewrites a `trial.json`. + +## Out of scope + +- Non-Docker environments (Vercel Sandbox, remote runners). The `docker.ts` + surface is small enough to add a second implementation later; it is not + abstracted ahead of need. +- Reusing eve's `SandboxBackend` for the environment. The benchmark must not + depend on the surface it measures. +- Uploading results to a hosted dashboard. +- `src/tools/` (eve tools over `core`) and the CI workflow. Both consume the + result schema above and follow once the CLI has run a full dataset. From 67d0a4c84e3996b624923eeb50004d797930c735 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Wed, 9 Sep 2026 14:26:36 -0400 Subject: [PATCH 2/9] feat(eve-bench): container hygiene, local task dirs, and stress scenarios Containers now carry eve-bench labels; a job removes leftovers from a crashed previous run of the same name, cleans up by label after an abort, checks the Docker daemon up front, and caps per-trial logs at 64 MiB. The CLI accepts repeatable --task-dir for local Terminal-Bench task directories, which powers a Docker-backed stress suite (pnpm run test:stress) covering timeouts, bad images, SIGINT abort and resume, and concurrency. The eve-smoke CI job can run pre-merge on PRs labeled bench:eve. Signed-off-by: Rui Conti --- .github/workflows/bench.yml | 12 +- packages/eve-bench/README.md | 39 ++- packages/eve-bench/package.json | 1 + packages/eve-bench/src/cli.ts | 18 +- packages/eve-bench/src/core/docker.test.ts | 35 +++ packages/eve-bench/src/core/docker.ts | 129 ++++++++- packages/eve-bench/src/core/job.ts | 17 +- packages/eve-bench/src/core/trial.ts | 57 ++-- packages/eve-bench/src/options.ts | 15 +- packages/eve-bench/test/stress/stress.test.ts | 266 ++++++++++++++++++ .../tasks/abort-a/environment/Dockerfile | 3 + .../test/stress/tasks/abort-a/instruction.md | 1 + .../stress/tasks/abort-a/solution/solve.sh | 2 + .../test/stress/tasks/abort-a/task.toml | 12 + .../test/stress/tasks/abort-a/tests/test.sh | 3 + .../tasks/abort-b/environment/Dockerfile | 3 + .../test/stress/tasks/abort-b/instruction.md | 1 + .../stress/tasks/abort-b/solution/solve.sh | 2 + .../test/stress/tasks/abort-b/task.toml | 12 + .../test/stress/tasks/abort-b/tests/test.sh | 3 + .../tasks/abort-resume/environment/Dockerfile | 3 + .../stress/tasks/abort-resume/instruction.md | 1 + .../tasks/abort-resume/solution/solve.sh | 2 + .../test/stress/tasks/abort-resume/task.toml | 12 + .../stress/tasks/abort-resume/tests/test.sh | 3 + .../agent-timeout/environment/Dockerfile | 3 + .../stress/tasks/agent-timeout/instruction.md | 1 + .../tasks/agent-timeout/solution/solve.sh | 2 + .../test/stress/tasks/agent-timeout/task.toml | 12 + .../stress/tasks/agent-timeout/tests/test.sh | 3 + .../tasks/bad-image/environment/Dockerfile | 3 + .../stress/tasks/bad-image/instruction.md | 1 + .../stress/tasks/bad-image/solution/solve.sh | 2 + .../test/stress/tasks/bad-image/task.toml | 13 + .../test/stress/tasks/bad-image/tests/test.sh | 3 + .../concurrency-1/environment/Dockerfile | 3 + .../stress/tasks/concurrency-1/instruction.md | 1 + .../tasks/concurrency-1/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-1/task.toml | 12 + .../stress/tasks/concurrency-1/tests/test.sh | 3 + .../concurrency-2/environment/Dockerfile | 3 + .../stress/tasks/concurrency-2/instruction.md | 1 + .../tasks/concurrency-2/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-2/task.toml | 12 + .../stress/tasks/concurrency-2/tests/test.sh | 3 + .../concurrency-3/environment/Dockerfile | 3 + .../stress/tasks/concurrency-3/instruction.md | 1 + .../tasks/concurrency-3/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-3/task.toml | 12 + .../stress/tasks/concurrency-3/tests/test.sh | 3 + .../concurrency-4/environment/Dockerfile | 3 + .../stress/tasks/concurrency-4/instruction.md | 1 + .../tasks/concurrency-4/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-4/task.toml | 12 + .../stress/tasks/concurrency-4/tests/test.sh | 3 + .../concurrency-5/environment/Dockerfile | 3 + .../stress/tasks/concurrency-5/instruction.md | 1 + .../tasks/concurrency-5/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-5/task.toml | 12 + .../stress/tasks/concurrency-5/tests/test.sh | 3 + .../concurrency-6/environment/Dockerfile | 3 + .../stress/tasks/concurrency-6/instruction.md | 1 + .../tasks/concurrency-6/solution/solve.sh | 2 + .../test/stress/tasks/concurrency-6/task.toml | 12 + .../stress/tasks/concurrency-6/tests/test.sh | 3 + .../stress/tasks/happy/environment/Dockerfile | 3 + .../test/stress/tasks/happy/instruction.md | 1 + .../test/stress/tasks/happy/solution/solve.sh | 2 + .../test/stress/tasks/happy/task.toml | 12 + .../test/stress/tasks/happy/tests/test.sh | 3 + .../verifier-timeout/environment/Dockerfile | 3 + .../tasks/verifier-timeout/instruction.md | 1 + .../tasks/verifier-timeout/solution/solve.sh | 2 + .../stress/tasks/verifier-timeout/task.toml | 12 + .../tasks/verifier-timeout/tests/test.sh | 3 + 75 files changed, 827 insertions(+), 36 deletions(-) create mode 100644 packages/eve-bench/src/core/docker.test.ts create mode 100644 packages/eve-bench/test/stress/stress.test.ts create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/happy/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/happy/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/happy/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/happy/tests/test.sh create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml create mode 100644 packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a759cdf859..76f1780fb5 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,6 +2,7 @@ name: eve Benchmarks on: pull_request: + types: [opened, synchronize, reopened, labeled] paths: - "packages/eve-bench/**" - research/eve-bench-runner.md @@ -90,8 +91,9 @@ jobs: } ' + # AI_GATEWAY_API_KEY is available to pull requests from branches in this repository, not forks. eve-smoke: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bench:eve') }} runs-on: ubuntu-latest timeout-minutes: 90 env: @@ -125,19 +127,19 @@ jobs: - name: Run eve smoke benchmark env: - MODEL: ${{ inputs.model }} - COHORT: ${{ inputs.cohort }} + MODEL: ${{ inputs.model || 'zai/glm-5.2' }} + COHORT: ${{ inputs.cohort || 'smoke' }} run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke - name: Download base job artifact - if: ${{ inputs.base_job_artifact != '' }} + if: ${{ inputs.base_job_artifact }} uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ inputs.base_job_artifact }} path: packages/eve-bench/.generated/base-job - name: Compare with base job - if: ${{ inputs.base_job_artifact != '' }} + if: ${{ inputs.base_job_artifact }} run: | pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt { diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md index 149054b23b..b3a41bf612 100644 --- a/packages/eve-bench/README.md +++ b/packages/eve-bench/README.md @@ -27,6 +27,7 @@ pnpm bench tasks list --cohort smoke # print a cohort's task AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8 pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release +pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task pnpm bench report --format junit --out junit.xml pnpm bench diff @@ -37,6 +38,10 @@ Every command accepts `--json` and never prompts. Jobs live in skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight trials and removes their containers. +Pass `--task-dir ` more than once to run one or more local Terminal-Bench +task directories. A local-only run does not sync a dataset and records its +dataset as `local`. + ## How a trial runs 1. Pull the task's prebuilt image (or build `environment/Dockerfile` once). @@ -47,6 +52,32 @@ trials and removes their containers. 4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` to the trial directory, remove the container, and write `trial.json`. +Each runner-owned log (`container.log`, `agent.log`, and `verifier.log`) is +capped at 64 MiB. The runner appends a truncation notice when a log exceeds the +cap. + +Containers carry the labels `eve-bench=1`, `eve-bench.job=`, and +`eve-bench.task=`. Find any leftovers with: + +```bash +docker ps -a --filter label=eve-bench=1 +``` + +At the start of a job, the runner removes stale containers with the same +`eve-bench.job` label before starting new trials. + +## Stress tests + +Run the Docker-backed stress suite separately from the default test suite: + +```bash +pnpm run test:stress +``` + +The suite requires Docker and takes about 30 seconds. It covers the happy path, +agent and verifier timeouts, a bad image, SIGINT abort and resume, and concurrent +trials. + ## Harnesses - `eve` (default): builds `agent/` against the local checkout (`--eve local`) @@ -89,9 +120,11 @@ attempt count, and eve source. touch this package or its design document and requires every task attempt to resolve. It uploads the job directory and `junit.xml` as artifacts. -The `eve-smoke` job runs only through `workflow_dispatch`, accepts `model` and -`cohort` inputs, and requires `AI_GATEWAY_API_KEY`. It uploads its job directory -and can optionally diff the result against a downloaded base-job artifact. +The real-model `eve-smoke` job runs through `workflow_dispatch` or when the +`bench:eve` label is added to a pull request. It accepts `model` and `cohort` +inputs for manual runs and requires `AI_GATEWAY_API_KEY`, which is unavailable +to pull requests from forks. It uploads its job directory and can optionally +diff the result against a downloaded base-job artifact. ## Layout diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index c06941682a..4ad272b0eb 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -10,6 +10,7 @@ "scripts": { "bench": "node src/cli.ts", "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "dependencies": { diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts index 4c63c3d549..95636fad45 100644 --- a/packages/eve-bench/src/cli.ts +++ b/packages/eve-bench/src/cli.ts @@ -22,8 +22,9 @@ const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets eve-bench tasks list [--cohort ] print task names - eve-bench run --model [--cohort ] [--task ...] [--attempts N] [--concurrency N] - [--harness eve|oracle] [--eve local|] [--job ] [--format console|json|junit] + eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] + [--concurrency N] [--harness eve|oracle] [--eve local|] [--job ] + [--format console|json|junit] eve-bench report [--format console|json|junit] eve-bench diff [--json] @@ -90,6 +91,7 @@ async function run(args: string[]): Promise { cohort: { type: "string" }, dataset: { type: "string" }, task: { type: "string", multiple: true }, + "task-dir": { type: "string", multiple: true }, attempts: { type: "string", default: "1" }, concurrency: { type: "string", default: "4" }, eve: { type: "string", default: "local" }, @@ -102,10 +104,15 @@ async function run(args: string[]): Promise { const harness = selectHarness(values.harness, values.eve); const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); if (!model) throw new Error("--model is required"); - const selection = await selectTasks(values); - const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); + const selection = await selectTasks({ ...values, taskDir: values["task-dir"] }); + const datasetTaskDirs = selection.tasks.length + ? await taskDirs( + await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock), + selection.tasks, + ) + : []; const loaded = await Promise.all( - (await taskDirs(datasetDir, selection.tasks)).map((dir) => loadTask(dir)), + [...datasetTaskDirs, ...selection.taskDirs].map((dir) => loadTask(dir)), ); const name = values.job ?? defaultJobName(`${harness.name}-${model}`); const controller = new AbortController(); @@ -128,6 +135,7 @@ async function run(args: string[]): Promise { concurrency: Number(values.concurrency), forwardEnv: forwardedEnv(), signal: controller.signal, + onLog: (message) => process.stderr.write(`${message}\n`), onTrial: (trial, resumed) => { if (format !== "console") return; const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); diff --git a/packages/eve-bench/src/core/docker.test.ts b/packages/eve-bench/src/core/docker.test.ts new file mode 100644 index 0000000000..e5221dc165 --- /dev/null +++ b/packages/eve-bench/src/core/docker.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { PassThrough } from "node:stream"; +import { finished } from "node:stream/promises"; + +import { createBoundedWritable } from "./docker.ts"; + +const TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; + +test("bounded writable keeps the head and appends one truncation line", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.write("abc"); + log.write("def"); + log.write("ignored"); + log.end(); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), `abcde${TRUNCATED_LINE}`); +}); + +test("bounded writable does not mark a log within the limit", async () => { + const output = new PassThrough(); + const chunks: Buffer[] = []; + output.on("data", (chunk: Buffer) => chunks.push(chunk)); + const log = createBoundedWritable(output, 5); + + log.end("abcde"); + await finished(log); + + assert.equal(Buffer.concat(chunks).toString(), "abcde"); +}); diff --git a/packages/eve-bench/src/core/docker.ts b/packages/eve-bench/src/core/docker.ts index 039ce37451..bb5937b0c2 100644 --- a/packages/eve-bench/src/core/docker.ts +++ b/packages/eve-bench/src/core/docker.ts @@ -1,6 +1,11 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import type { Writable } from "node:stream"; +import { createWriteStream } from "node:fs"; +import { Writable } from "node:stream"; + +const CAPTURE_LIMIT_BYTES = 16 * 1024 * 1024; +const LOG_LIMIT_BYTES = 64 * 1024 * 1024; +const LOG_TRUNCATED_LINE = "[eve-bench] log truncated at 64 MiB\n"; export interface ProcessResult { readonly exitCode: number | null; @@ -21,11 +26,21 @@ export interface ExecOptions { export interface ContainerOptions { readonly image: string; readonly name: string; + readonly job: string; + readonly task: string; readonly cpus?: number; readonly memoryMb?: number; readonly network: boolean; } +export interface RunnerContainer { + readonly id: string; + readonly name: string; + readonly job: string; + readonly task: string; + readonly createdAt: string; +} + export class Container { readonly name: string; @@ -34,7 +49,20 @@ export class Container { } static async start(options: ContainerOptions, signal?: AbortSignal): Promise { - const args = ["run", "--detach", "--name", options.name, "--entrypoint", "sh"]; + const args = [ + "run", + "--detach", + "--name", + options.name, + "--label", + "eve-bench=1", + "--label", + `eve-bench.job=${options.job}`, + "--label", + `eve-bench.task=${options.task}`, + "--entrypoint", + "sh", + ]; if (options.cpus !== undefined) args.push("--cpus", String(options.cpus)); if (options.memoryMb !== undefined) args.push("--memory", `${options.memoryMb}m`); if (!options.network) args.push("--network", "none"); @@ -87,6 +115,81 @@ export class Container { } } +export async function assertDockerAvailable(): Promise { + try { + await dockerOk(["version", "--format", "{{.Server.Version}}"], { timeoutMs: 10_000 }); + } catch { + throw new Error("Docker is unavailable; start the Docker daemon and retry."); + } +} + +export async function listRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const args = [ + "ps", + "-a", + "--filter", + "label=eve-bench=1", + ...(filter.job ? ["--filter", `label=eve-bench.job=${filter.job}`] : []), + "--format", + "{{json .}}", + ]; + const result = await dockerOk(args, { timeoutMs: 30_000 }); + return result.stdout + .split("\n") + .filter(Boolean) + .map((line) => { + const row = JSON.parse(line) as Record; + return { + id: row.ID ?? "", + name: row.Names ?? "", + job: labelValue(row.Labels, "eve-bench.job"), + task: labelValue(row.Labels, "eve-bench.task"), + createdAt: row.CreatedAt ?? "", + }; + }); +} + +export async function removeRunnerContainers( + filter: { + job?: string; + } = {}, +): Promise { + const containers = await listRunnerContainers(filter); + if (containers.length > 0) { + await dockerOk(["rm", "--force", "--volumes", ...containers.map((container) => container.id)], { + timeoutMs: 60_000, + }); + } + return containers; +} + +export function createBoundedLog(path: string): Writable { + return createBoundedWritable(createWriteStream(path), LOG_LIMIT_BYTES); +} + +export function createBoundedWritable(output: Writable, limitBytes: number): Writable { + let written = 0; + let truncated = false; + return new Writable({ + write(chunk: Buffer | string, encoding, callback) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); + const remaining = Math.max(0, limitBytes - written); + const kept = bytes.subarray(0, remaining); + written += kept.length; + if (kept.length < bytes.length) truncated = true; + if (kept.length === 0) callback(); + else output.write(kept, callback); + }, + final(callback) { + output.end(truncated ? LOG_TRUNCATED_LINE : undefined, callback); + }, + }); +} + export async function imageExists(tag: string): Promise { const result = await docker(["image", "inspect", tag], { timeoutMs: 30_000 }); return result.exitCode === 0; @@ -132,6 +235,8 @@ function docker(args: readonly string[], options: ExecOptions): Promise { - stdout.push(chunk); + stdoutBytes = appendHead(stdout, stdoutBytes, chunk, CAPTURE_LIMIT_BYTES); options.log?.write(chunk); }); child.stderr.on("data", (chunk: Buffer) => { - stderr.push(chunk); + stderrBytes = appendHead(stderr, stderrBytes, chunk, CAPTURE_LIMIT_BYTES); options.log?.write(chunk); }); child.once("error", (error) => { @@ -175,3 +280,19 @@ function docker(args: readonly string[], options: ExecOptions): Promise 0) chunks.push(kept); + return size + kept.length; +} + +function labelValue(labels: string | undefined, key: string): string { + const prefix = `${key}=`; + return ( + labels + ?.split(",") + .find((label) => label.startsWith(prefix)) + ?.slice(prefix.length) ?? "" + ); +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts index d16feead4e..18f21de5ef 100644 --- a/packages/eve-bench/src/core/job.ts +++ b/packages/eve-bench/src/core/job.ts @@ -2,6 +2,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { DatasetLock } from "./dataset.ts"; +import { assertDockerAvailable, listRunnerContainers, removeRunnerContainers } from "./docker.ts"; import type { Harness } from "./harness.ts"; import { summarize, type JobResult, type TrialResult } from "./result.ts"; import type { Task } from "./task.ts"; @@ -19,10 +20,19 @@ export interface JobInput { readonly concurrency: number; readonly forwardEnv: Readonly>; readonly signal: AbortSignal; + readonly onLog?: (message: string) => void; readonly onTrial?: (trial: TrialResult, resumed: boolean) => void; } export async function runJob(input: JobInput): Promise { + await assertDockerAvailable(); + const leftovers = await listRunnerContainers({ job: input.name }); + if (leftovers.length > 0) { + await removeRunnerContainers({ job: input.name }); + for (const container of leftovers) { + input.onLog?.(`Removed leftover container ${container.name} from job ${input.name}`); + } + } await mkdir(input.dir, { recursive: true }); const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); await writeFile( @@ -69,6 +79,7 @@ export async function runJob(input: JobInput): Promise { bundle, model: input.model, forwardEnv: input.forwardEnv, + job: input.name, dir, signal: input.signal, }); @@ -76,7 +87,11 @@ export async function runJob(input: JobInput): Promise { input.onTrial?.(trial, false); } }); - await Promise.all(workers); + const outcomes = await Promise.allSettled(workers); + if (input.signal.aborted) await removeRunnerContainers({ job: input.name }); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } trials.sort((a, b) => a.task.localeCompare(b.task) || a.attempt - b.attempt); const result: JobResult = { diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts index 372ad58969..c91f71eeb1 100644 --- a/packages/eve-bench/src/core/trial.ts +++ b/packages/eve-bench/src/core/trial.ts @@ -1,10 +1,17 @@ import { createHash } from "node:crypto"; -import { createWriteStream } from "node:fs"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Writable } from "node:stream"; +import { finished } from "node:stream/promises"; -import { buildImage, Container, containerArch, pullImage, type ProcessResult } from "./docker.ts"; +import { + buildImage, + Container, + containerArch, + createBoundedLog, + pullImage, + type ProcessResult, +} from "./docker.ts"; import type { Harness, HarnessBundle } from "./harness.ts"; import type { StepResult, TrialResult, Usage } from "./result.ts"; import type { Task } from "./task.ts"; @@ -17,6 +24,7 @@ export interface TrialInput { readonly model: string; /** Host env forwarded to the harness exec only (provider credentials). */ readonly forwardEnv: Readonly>; + readonly job: string; readonly dir: string; readonly signal: AbortSignal; } @@ -29,7 +37,7 @@ export async function runTrial(input: TrialInput): Promise { const { task, signal } = input; const startedAt = new Date().toISOString(); await mkdir(input.dir, { recursive: true }); - const containerLog = createWriteStream(join(input.dir, "container.log"), { flags: "a" }); + const containerLog = createBoundedLog(join(input.dir, "container.log")); let container: Container | undefined; let agent: StepResult = skipped(); let verifier: StepResult = skipped(); @@ -42,6 +50,8 @@ export async function runTrial(input: TrialInput): Promise { { image, name: `eve-bench-${task.name}-${input.attempt}-${Date.now().toString(36)}`, + job: input.job, + task: task.name, cpus: task.environment.cpus, memoryMb: task.environment.memoryMb, network: task.environment.allowInternet, @@ -74,23 +84,27 @@ export async function runTrial(input: TrialInput): Promise { logsDir: `${LOGS_DIR}/agent`, }; agent = await step(() => - container!.exec(input.harness.command(runContext), { - cwd: workdir, - env: { ...input.forwardEnv, ...input.harness.env(runContext) }, - timeoutMs: task.agentTimeoutMs, - signal, - log: createWriteStream(join(input.dir, "agent.log")), - }), + withBoundedLog(join(input.dir, "agent.log"), (log) => + container!.exec(input.harness.command(runContext), { + cwd: workdir, + env: { ...input.forwardEnv, ...input.harness.env(runContext) }, + timeoutMs: task.agentTimeoutMs, + signal, + log, + }), + ), ); await container.upload(`${join(task.dir, "tests")}/.`, TESTS_DIR, signal); verifier = await step(() => - container!.exec(`bash ${TESTS_DIR}/test.sh`, { - cwd: workdir, - timeoutMs: task.verifierTimeoutMs, - signal, - log: createWriteStream(join(input.dir, "verifier.log")), - }), + withBoundedLog(join(input.dir, "verifier.log"), (log) => + container!.exec(`bash ${TESTS_DIR}/test.sh`, { + cwd: workdir, + timeoutMs: task.verifierTimeoutMs, + signal, + log, + }), + ), ); for (const logs of ["agent", "verifier"]) { @@ -105,6 +119,7 @@ export async function runTrial(input: TrialInput): Promise { } finally { if (container) await container.remove(); containerLog.end(); + await finished(containerLog); } const result: TrialResult = { task: task.name, @@ -123,6 +138,16 @@ export async function runTrial(input: TrialInput): Promise { return result; } +async function withBoundedLog(path: string, run: (log: Writable) => Promise): Promise { + const log = createBoundedLog(path); + try { + return await run(log); + } finally { + log.end(); + await finished(log); + } +} + async function resolveImage(task: Task, log: Writable, signal: AbortSignal): Promise { const options = { timeoutMs: task.buildTimeoutMs, signal, log }; if (task.environment.dockerImage) { diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts index 7f028379f8..cf35a1a996 100644 --- a/packages/eve-bench/src/options.ts +++ b/packages/eve-bench/src/options.ts @@ -18,20 +18,31 @@ export const paths = { export interface TaskSelection { readonly lock: DatasetLock; readonly tasks: readonly string[]; + readonly taskDirs: readonly string[]; } export async function selectTasks(input: { cohort?: string; dataset?: string; task?: readonly string[]; + taskDir?: readonly string[]; }): Promise { + const localTaskDirs = input.taskDir?.map((dir) => resolve(dir)) ?? []; + if (localTaskDirs.length > 0 && !input.cohort && !input.dataset && !input.task?.length) { + return { + lock: { name: "local", version: "0", commit: "", gitUrl: "", tasks: [] }, + tasks: [], + taskDirs: localTaskDirs, + }; + } + if (input.cohort) { const cohort = await readCohort(join(paths.datasetsRoot, "cohorts", `${input.cohort}.json`)); const entry = await lock(cohort.dataset); const tasks = input.task?.length ? cohort.tasks.filter((task) => input.task!.includes(task)) : cohort.tasks; - return { lock: entry, tasks }; + return { lock: entry, tasks, taskDirs: localTaskDirs }; } const entry = await lock(input.dataset ?? (await defaultDataset())); @@ -42,7 +53,7 @@ export async function selectTasks(input: { const missing = input.task.filter((task) => !tasks.includes(task)); throw new Error(`tasks not in ${entry.name}@${entry.version}: ${missing.join(", ")}`); } - return { lock: entry, tasks }; + return { lock: entry, tasks, taskDirs: localTaskDirs }; } export function jobDir(nameOrPath: string): string { diff --git a/packages/eve-bench/test/stress/stress.test.ts b/packages/eve-bench/test/stress/stress.test.ts new file mode 100644 index 0000000000..85977542bd --- /dev/null +++ b/packages/eve-bench/test/stress/stress.test.ts @@ -0,0 +1,266 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFile, rm } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const tasksRoot = join(packageRoot, "test", "stress", "tasks"); +const jobsRoot = join(packageRoot, ".generated", "jobs"); +const timeout = 300_000; + +interface StepResult { + status: "completed" | "timeout" | "failed" | "skipped"; +} + +interface TrialResult { + task: string; + reward: number | null; + agent: StepResult; + verifier: StepResult; + error?: string; +} + +interface JobResult { + dataset: { name: string; version: string; commit: string }; + trials: TrialResult[]; +} + +interface CliResult { + code: number | null; + stdout: string; + stderr: string; + durationMs: number; +} + +function taskDir(name: string): string { + return join(tasksRoot, name); +} + +function cliArgs( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], +): string[] { + return [ + "src/cli.ts", + "run", + "--harness", + "oracle", + "--job", + job, + "--json", + ...taskDirs.flatMap((dir) => ["--task-dir", dir]), + ...extra, + ]; +} + +async function runCli( + job: string, + taskDirs: readonly string[], + extra: readonly string[] = [], + fresh = true, +): Promise { + if (fresh) await rm(join(jobsRoot, job), { recursive: true, force: true }); + const started = Date.now(); + const child = spawn(process.execPath, cliArgs(job, taskDirs, extra), { + cwd: packageRoot, + stdio: ["ignore", "pipe", "pipe"], + }); + return collect(child, started); +} + +function collect(child: ReturnType, started = Date.now()): Promise { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout?.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk)); + return new Promise((resolvePromise, reject) => { + child.once("error", reject); + child.once("close", (code) => { + resolvePromise({ + code, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + durationMs: Date.now() - started, + }); + }); + }); +} + +async function readResult(job: string): Promise { + return JSON.parse(await readFile(join(jobsRoot, job, "result.json"), "utf8")) as JobResult; +} + +async function readTrial(job: string, task: string): Promise { + return JSON.parse( + await readFile(join(jobsRoot, job, task, "1", "trial.json"), "utf8"), + ) as TrialResult; +} + +async function dockerNames(filter: string): Promise { + const result = await runCommand("docker", [ + "ps", + "-a", + "--filter", + `name=eve-bench-${filter}`, + "--format", + "{{.Names}}", + ]); + assert.equal(result.code, 0, result.stderr); + return result.stdout.trim() ? result.stdout.trim().split("\n") : []; +} + +async function cleanupContainers(filter: string): Promise { + const names = await dockerNames(filter); + if (names.length === 0) return; + const result = await runCommand("docker", ["rm", "--force", ...names]); + assert.equal(result.code, 0, result.stderr); +} + +async function runCommand(command: string, args: readonly string[]): Promise { + const started = Date.now(); + return collect( + spawn(command, args, { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }), + started, + ); +} + +async function waitFor( + condition: () => Promise, + waitMs: number, + description: string, +): Promise { + const deadline = Date.now() + waitMs; + while (Date.now() < deadline) { + if (await condition()) return; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 250)); + } + assert.fail(`timed out waiting for ${description}`); +} + +function assertPassed(result: CliResult): void { + assert.equal(result.code, 0, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`); +} + +test("happy: records completed steps and reward 1", { timeout }, async () => { + const job = "stress-happy"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("happy")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-happy"); + assert.equal(trial.reward, 1); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "completed"); + assert.deepEqual((await readResult(job)).dataset, { name: "local", version: "0", commit: "" }); +}); + +test("agent-timeout: verifier still runs and the container is removed", { timeout }, async () => { + const job = "stress-agent-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("agent-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-agent-timeout"); + assert.equal(trial.agent.status, "timeout"); + assert.equal(trial.verifier.status, "completed"); + assert.equal(trial.reward, 0); + assert.deepEqual(await dockerNames(job), []); +}); + +test("verifier-timeout: records timeout and null reward", { timeout }, async () => { + const job = "stress-verifier-timeout"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("verifier-timeout")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-verifier-timeout"); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.verifier.status, "timeout"); + assert.equal(trial.reward, null); +}); + +test( + "bad-image: records the setup error, skips both steps, and leaves no container", + { timeout }, + async () => { + const job = "stress-bad-image"; + await cleanupContainers(job); + const result = await runCli(job, [taskDir("bad-image")]); + assertPassed(result); + + const trial = await readTrial(job, "stress-bad-image"); + assert.ok(trial.error); + assert.equal(trial.agent.status, "skipped"); + assert.equal(trial.verifier.status, "skipped"); + assert.deepEqual(await dockerNames(job), []); + }, +); + +test( + "abort: exits 130, writes a partial result, cleans up, and resumes with a new task set", + { timeout }, + async () => { + const job = "stress-abort"; + await cleanupContainers(job); + await rm(join(jobsRoot, job), { recursive: true, force: true }); + const child = spawn( + process.execPath, + cliArgs(job, [taskDir("abort-a"), taskDir("abort-b")], ["--concurrency", "2"]), + { cwd: packageRoot, stdio: ["ignore", "pipe", "pipe"] }, + ); + const completed = collect(child); + + try { + await waitFor( + async () => (await dockerNames(job)).length === 2, + 120_000, + "both abort containers", + ); + child.kill("SIGINT"); + const aborted = await completed; + assert.equal(aborted.code, 130, `stdout:\n${aborted.stdout}\nstderr:\n${aborted.stderr}`); + + const partial = await readResult(job); + assert.ok(partial.trials.length <= 2); + await waitFor(async () => (await dockerNames(job)).length === 0, 15_000, "abort cleanup"); + + const resumed = await runCli(job, [taskDir("abort-resume")], [], false); + assertPassed(resumed); + const result = await readResult(job); + assert.equal(result.trials.length, 1); + assert.equal(result.trials[0]?.task, "stress-abort-resume"); + assert.equal(result.trials[0]?.reward, 1); + } finally { + if (child.exitCode === null) child.kill("SIGKILL"); + await cleanupContainers(job); + } + }, +); + +test( + "concurrency: six trials finish substantially faster than serial execution", + { timeout }, + async () => { + const dirs = Array.from({ length: 6 }, (_, index) => taskDir(`concurrency-${index + 1}`)); + await cleanupContainers("stress-concurrency"); + + const single = await runCli("stress-concurrency-single", [dirs[0]!], ["--concurrency", "1"]); + assertPassed(single); + const concurrent = await runCli("stress-concurrency", dirs, ["--concurrency", "6"]); + assertPassed(concurrent); + + const result = await readResult("stress-concurrency"); + assert.equal(result.trials.length, 6); + assert.deepEqual( + result.trials.map((trial) => trial.reward), + [1, 1, 1, 1, 1, 1], + ); + assert.ok( + concurrent.durationMs < single.durationMs * 3, + `single=${single.durationMs}ms concurrent=${concurrent.durationMs}ms`, + ); + }, +); diff --git a/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-a/instruction.md b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-a/task.toml b/packages/eve-bench/test/stress/tasks/abort-a/task.toml new file mode 100644 index 0000000000..d0a9a638a5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-a" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-a/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-b/instruction.md b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh new file mode 100644 index 0000000000..f29ebee1a6 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 120; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-b/task.toml b/packages/eve-bench/test/stress/tasks/abort-b/task.toml new file mode 100644 index 0000000000..cf271a1f1a --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-b" + +[agent] +timeout_sec = 180 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-b/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/task.toml b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml new file mode 100644 index 0000000000..cb0de4aea3 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-abort-resume" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/abort-resume/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh new file mode 100644 index 0000000000..eee73eb76e --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 60; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml new file mode 100644 index 0000000000..58831360d4 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-agent-timeout" + +[agent] +timeout_sec = 3 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/agent-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/bad-image/instruction.md b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/bad-image/task.toml b/packages/eve-bench/test/stress/tasks/bad-image/task.toml new file mode 100644 index 0000000000..d74d09f84c --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/task.toml @@ -0,0 +1,13 @@ +[task] +name = "stress-bad-image" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false +docker_image = "eve-bench/does-not-exist:1" diff --git a/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh new file mode 100644 index 0000000000..169b2c15e2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/bad-image/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +echo 1 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml new file mode 100644 index 0000000000..257a3f2044 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-1" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-1/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml new file mode 100644 index 0000000000..21e6f19aeb --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-2" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-2/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml new file mode 100644 index 0000000000..c0faeb0b80 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-3" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-3/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml new file mode 100644 index 0000000000..0b2d2ba227 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-4" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-4/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml new file mode 100644 index 0000000000..52baba26c5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-5" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-5/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh new file mode 100644 index 0000000000..dcb080ade2 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 2; touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml new file mode 100644 index 0000000000..15ae72e375 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-concurrency-6" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/concurrency-6/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/happy/instruction.md b/packages/eve-bench/test/stress/tasks/happy/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/happy/task.toml b/packages/eve-bench/test/stress/tasks/happy/task.toml new file mode 100644 index 0000000000..eaf9709812 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-happy" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 30 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/happy/tests/test.sh b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh new file mode 100644 index 0000000000..b74a14c1a0 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/happy/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +[ -f /app/done ] && echo 1 > /logs/verifier/reward.txt || echo 0 > /logs/verifier/reward.txt diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile new file mode 100644 index 0000000000..c798018255 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.20 +RUN apk add --no-cache bash +WORKDIR /app diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md new file mode 100644 index 0000000000..f65ba49611 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/instruction.md @@ -0,0 +1 @@ +Create /app/done. diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh new file mode 100644 index 0000000000..fbb35e41b5 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +touch /app/done diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml new file mode 100644 index 0000000000..7672b068e9 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/task.toml @@ -0,0 +1,12 @@ +[task] +name = "stress-verifier-timeout" + +[agent] +timeout_sec = 30 + +[verifier] +timeout_sec = 3 + +[environment] +build_timeout_sec = 60 +allow_internet = false diff --git a/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh new file mode 100644 index 0000000000..fb0ab7d284 --- /dev/null +++ b/packages/eve-bench/test/stress/tasks/verifier-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p /logs/verifier +sleep 60; echo 1 > /logs/verifier/reward.txt From cb990c0030e5abcb2198891e94a13e221691e131 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:16:03 -0400 Subject: [PATCH 3/9] feat(eve-bench): add CLI and e0 harnesses and separate invalid trials from scores Add pinned pi, OpenCode, and Codex CLI harnesses and an e0 harness, with resumable job identities. Classify every trial as scored or invalid (environment, harness, or verifier). A harness that completes no model call is excluded from scores instead of counting as a task failure. Usage is read on the host from each harness's own event log, so it survives agent timeouts. Run every harness's tools with the task container's own HOME, PATH, and image environment. Give eve's bash tool a process-group timeout and restore the task's PORT and HOST; report a missing ripgrep explicitly. CI builds eve before the tests, runs the Docker stress suite, and fails smoke jobs that contain invalid attempts. Signed-off-by: Rui Conti --- .github/workflows/bench.yml | 17 +- packages/eve-bench/README.md | 70 ++- packages/eve-bench/agent/task-path.ts | 15 + packages/eve-bench/agent/tools/bash.ts | 94 ++- packages/eve-bench/agent/tools/glob.ts | 2 +- packages/eve-bench/agent/tools/grep.ts | 11 +- packages/eve-bench/agent/tools/read_file.ts | 2 +- packages/eve-bench/agent/tools/write_file.ts | 2 +- packages/eve-bench/package.json | 5 +- packages/eve-bench/src/cli.ts | 46 +- packages/eve-bench/src/core/harness.ts | 11 + packages/eve-bench/src/core/id.test.ts | 15 + packages/eve-bench/src/core/id.ts | 8 + .../eve-bench/src/core/job-identity.test.ts | 92 +++ packages/eve-bench/src/core/job-identity.ts | 129 +++++ packages/eve-bench/src/core/job.ts | 153 ++++- packages/eve-bench/src/core/report.test.ts | 104 +++- packages/eve-bench/src/core/report.ts | 92 ++- packages/eve-bench/src/core/result.ts | 121 +++- packages/eve-bench/src/core/task.ts | 3 +- packages/eve-bench/src/core/trial.ts | 52 +- packages/eve-bench/src/core/usage.test.ts | 123 ++++ packages/eve-bench/src/core/usage.ts | 63 ++ packages/eve-bench/src/harnesses/cli/index.ts | 50 ++ .../eve-bench/src/harnesses/cli/output.ts | 53 ++ .../eve-bench/src/harnesses/cli/plan.test.ts | 207 +++++++ packages/eve-bench/src/harnesses/cli/plan.ts | 205 +++++++ .../eve-bench/src/harnesses/cli/prepare.ts | 247 ++++++++ .../eve-bench/src/harnesses/cli/runner.ts | 237 ++++++++ packages/eve-bench/src/harnesses/cli/usage.ts | 55 ++ .../src/harnesses/e0/container-backend.ts | 394 +++++++++++++ packages/eve-bench/src/harnesses/e0/index.ts | 252 ++++++++ .../src/harnesses/e0/snapshot.test.ts | 46 ++ .../eve-bench/src/harnesses/e0/snapshot.ts | 53 ++ packages/eve-bench/src/harnesses/eve/index.ts | 7 +- .../eve-bench/src/harnesses/eve/runner.mjs | 54 +- packages/eve-bench/src/harnesses/eve/usage.ts | 22 + packages/eve-bench/src/harnesses/oracle.ts | 1 + packages/eve-bench/src/options.ts | 36 +- packages/eve-bench/src/tools/run_bench.ts | 17 +- packages/eve-bench/src/tools/tools.test.ts | 21 + .../test/scenarios/cli-adapters.test.ts | 320 +++++++++++ .../test/scenarios/cli-report.test.ts | 68 +++ .../test/scenarios/e0-container.test.ts | 539 ++++++++++++++++++ .../test/scenarios/eve-agent-tools.test.ts | 139 +++++ .../test/scenarios/job-resume.test.ts | 364 ++++++++++++ .../test/scenarios/trial-validity.test.ts | 208 +++++++ packages/eve-bench/test/stress/stress.test.ts | 24 +- 48 files changed, 4657 insertions(+), 192 deletions(-) create mode 100644 packages/eve-bench/src/core/id.test.ts create mode 100644 packages/eve-bench/src/core/id.ts create mode 100644 packages/eve-bench/src/core/job-identity.test.ts create mode 100644 packages/eve-bench/src/core/job-identity.ts create mode 100644 packages/eve-bench/src/core/usage.test.ts create mode 100644 packages/eve-bench/src/core/usage.ts create mode 100644 packages/eve-bench/src/harnesses/cli/index.ts create mode 100644 packages/eve-bench/src/harnesses/cli/output.ts create mode 100644 packages/eve-bench/src/harnesses/cli/plan.test.ts create mode 100644 packages/eve-bench/src/harnesses/cli/plan.ts create mode 100644 packages/eve-bench/src/harnesses/cli/prepare.ts create mode 100644 packages/eve-bench/src/harnesses/cli/runner.ts create mode 100644 packages/eve-bench/src/harnesses/cli/usage.ts create mode 100644 packages/eve-bench/src/harnesses/e0/container-backend.ts create mode 100644 packages/eve-bench/src/harnesses/e0/index.ts create mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.test.ts create mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.ts create mode 100644 packages/eve-bench/src/harnesses/eve/usage.ts create mode 100644 packages/eve-bench/test/scenarios/cli-adapters.test.ts create mode 100644 packages/eve-bench/test/scenarios/cli-report.test.ts create mode 100644 packages/eve-bench/test/scenarios/e0-container.test.ts create mode 100644 packages/eve-bench/test/scenarios/eve-agent-tools.test.ts create mode 100644 packages/eve-bench/test/scenarios/job-resume.test.ts create mode 100644 packages/eve-bench/test/scenarios/trial-validity.test.ts diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 76f1780fb5..213bbe3796 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -57,17 +57,26 @@ jobs: run: pnpm install --frozen-lockfile working-directory: ${{ github.workspace }} + # The typecheck and the tool and agent tests import eve's built exports. + - name: Build eve package + run: pnpm --filter eve run build + working-directory: ${{ github.workspace }} + - name: Run tests run: pnpm run test + # Real containers: timeouts, bad images, abort/resume, concurrency, and trial validity. + - name: Run Docker stress tests + run: pnpm run test:stress + - name: Sync benchmark tasks run: pnpm bench tasks sync --json - name: Run oracle smoke benchmark run: pnpm bench run --harness oracle --cohort smoke --concurrency 5 --job oracle-smoke --format junit > junit.xml - - name: Print oracle smoke report - run: pnpm bench report oracle-smoke + - name: Print oracle smoke report and require valid attempts + run: pnpm bench report oracle-smoke --fail-on-invalid - name: Upload oracle smoke artifacts if: always() @@ -131,6 +140,10 @@ jobs: COHORT: ${{ inputs.cohort || 'smoke' }} run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke + # Scores are only meaningful when every attempt reached the model and the verifier. + - name: Print eve smoke report and require valid attempts + run: pnpm bench report eve-smoke --fail-on-invalid + - name: Download base job artifact if: ${{ inputs.base_job_artifact }} uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md index b3a41bf612..6d1bec4eb5 100644 --- a/packages/eve-bench/README.md +++ b/packages/eve-bench/README.md @@ -29,14 +29,24 @@ pnpm bench run --harness oracle --cohort smoke # reference solutions; pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task +# Build and cache a local e0 snapshot without calling a model. +pnpm bench prepare --harness e0 --agent /path/to/internal-agents/agents/e0 \ + --model openai/gpt-5.6-terra + +# Run one e0 smoke task. The source app must have installed workspace dependencies. +AI_GATEWAY_API_KEY=... pnpm bench run --harness e0 \ + --agent /path/to/internal-agents/agents/e0 \ + --model deepseek/deepseek-v4-pro --cohort smoke --task fix-git --job e0-deepseek + pnpm bench report --format junit --out junit.xml pnpm bench diff ``` Every command accepts `--json` and never prompts. Jobs live in -`.generated/jobs//`; re-running with the same `--job` name resumes it, -skipping trials that already have a `trial.json`. Ctrl-C aborts in-flight -trials and removes their containers. +`.generated/jobs//`. Re-running the same job resumes only when the model, +harness, bundle provenance, dataset, tasks, and attempt count match. Invalid or +mismatched trial records fail closed instead of being overwritten. Ctrl-C +aborts in-flight trials and removes their containers. Pass `--task-dir ` more than once to run one or more local Terminal-Bench task directories. A local-only run does not sync a dataset and records its @@ -52,6 +62,42 @@ dataset as `local`. 4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier` to the trial directory, remove the container, and write `trial.json`. +## Reading results + +A trial is scored only when it measured the harness. Otherwise `trial.json` +records `invalid` with a phase and reason, and the trial is excluded from +`resolved`, `meanReward`, and `diff`: + +- `environment`: runner-owned work failed, such as the image pull or build, + container start, uploads, or log download. +- `harness`: the harness ran but completed no model call (zero or missing token + usage). This usually means broken credentials, adapter configuration, or + startup, not a weak harness. +- `verifier`: the verifier wrote no reward, including verifier timeouts. + +After one model call completes, crashes, agent timeouts, and wrong answers +count against the harness. Reports list every invalid attempt, JUnit marks them +as errors instead of failures, and `diff` shows invalid counts next to each +task. Resolve invalid attempts before comparing jobs; `pnpm bench report +--fail-on-invalid` exits 1 when any remain, and CI applies it to every smoke job. + +Usage is read on the host from each harness's own event log (`events.ndjson` +for eve and e0, `.jsonl` for pi, OpenCode, and Codex), so it survives agent +timeouts. Input tokens include cache reads and output tokens include reasoning. +`costUsd` is null when the harness reports no cost; the CLI competitors report +none for custom Gateway providers, so compare tokens rather than cost across +harnesses. + +## Environment parity + +Every harness's tool commands run with the task container's own environment: +the image's `HOME`, `PATH`, and `ENV`, starting in the task working directory. +Harness state lives only in harness-owned paths under `/installed-agent` +(`CODEX_HOME`, `PI_CODING_AGENT_DIR`, `OPENCODE_CONFIG`), so what an agent +installs or configures in the task survives into verification. The eve runner +restores the container's own `PORT`, `HOST`, and Nitro variables for agent +commands. + Each runner-owned log (`container.log`, `agent.log`, and `verifier.log`) is capped at 64 MiB. The runner appends a truncation notice when a log exceeds the cap. @@ -93,6 +139,22 @@ trials. `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass eve's nested sandbox on purpose. +- `e0`: snapshots an installed local e0 app and its source-backed `eve-code` + extension. The bundle records source, dependency, and installed eve hashes. + It preserves e0's instructions, coding capabilities, tools, skills, and + authored worker model. The adapter replaces production sandbox and transport + integrations with the task-container backend and a local eve channel. It + removes production Connect configuration, channels, connections, schedules, + instrumentation, and sandbox bootstrap. `--model` changes the root model, + not e0's authored worker model. + +- `pi`, `opencode`, and `codex`: prepare explicitly versioned CLI bundles on the + host and route full provider-qualified model IDs through an HTTPS API base. + Pass an exact `--version`. The default base is Vercel AI Gateway. Codex uses + the Responses protocol; pi and OpenCode use chat completions. Hermes fails + explicitly until a pinned Python/uv bundle and verified custom-provider + routing are implemented. + - `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner and as the reward ceiling. @@ -130,7 +192,7 @@ diff the result against a downloaded base-job artifact. - `agent/`: harness behavior under evaluation (model binding, instructions, tools) - `src/core/`: runner library with no dependency on eve or third parties -- `src/harnesses/`: `eve` and `oracle` +- `src/harnesses/`: `eve`, `e0`, CLI competitors, and `oracle` - `src/tools/`: eve tool definitions over the runner core - `src/options.ts`: run options shared by the CLI and tools - `src/cli.ts`: `eve-bench` CLI diff --git a/packages/eve-bench/agent/task-path.ts b/packages/eve-bench/agent/task-path.ts index 8303fe355f..b27ed33c6b 100644 --- a/packages/eve-bench/agent/task-path.ts +++ b/packages/eve-bench/agent/task-path.ts @@ -11,3 +11,18 @@ export function taskRoot(): string { export function resolveTaskPath(path = "."): string { return isAbsolute(path) ? path : resolve(taskRoot(), path); } + +/** + * The container environment as the task sees it: the runner's server-only + * overrides (such as PORT) are restored so task commands behave as they would + * for any other harness. + */ +export function taskEnv(): NodeJS.ProcessEnv { + const { EVE_BENCH_TASK_ENV_RESTORE: restore, ...env } = process.env; + const original = JSON.parse(restore ?? "{}") as Record; + for (const [key, value] of Object.entries(original)) { + if (value === null) delete env[key]; + else env[key] = value; + } + return env; +} diff --git a/packages/eve-bench/agent/tools/bash.ts b/packages/eve-bench/agent/tools/bash.ts index 2d8121c64d..9979744b41 100644 --- a/packages/eve-bench/agent/tools/bash.ts +++ b/packages/eve-bench/agent/tools/bash.ts @@ -1,37 +1,91 @@ -import { exec } from "node:child_process"; -import { promisify } from "node:util"; +import { spawn } from "node:child_process"; +import { constants } from "node:os"; import { defineTool } from "eve/tools"; import { never } from "eve/tools/approval"; import { z } from "zod"; -import { taskRoot } from "../task-path.js"; +import { taskEnv, taskRoot } from "../task-path.ts"; -const execAsync = promisify(exec); +const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; +const DEFAULT_TIMEOUT_SECONDS = 120; +const MAX_TIMEOUT_SECONDS = 1800; export default defineTool({ approval: never(), - description: - "Execute a shell command in the task environment. Commands start in the task working directory.", + description: `Execute a shell command in the task environment. Commands start in the task working directory. A command that runs longer than timeoutSeconds (default ${DEFAULT_TIMEOUT_SECONDS}) is stopped with its child processes and returns exit code 124 with the output captured so far; raise timeoutSeconds for long builds, and start servers in the background.`, inputSchema: z.object({ command: z.string().describe("The shell command to execute."), + timeoutSeconds: z + .number() + .int() + .positive() + .max(MAX_TIMEOUT_SECONDS) + .optional() + .describe(`Seconds before the command is stopped. Default ${DEFAULT_TIMEOUT_SECONDS}.`), }), - async execute({ command }, ctx) { - try { - const { stdout, stderr } = await execAsync(command, { + execute({ command, timeoutSeconds = DEFAULT_TIMEOUT_SECONDS }, ctx) { + return new Promise((resolve) => { + // A separate process group lets a timeout stop everything the command started. + const child = spawn("sh", ["-c", command], { cwd: taskRoot(), - env: process.env, - maxBuffer: 10 * 1024 * 1024, - signal: ctx.abortSignal, + env: taskEnv(), + detached: true, + stdio: ["ignore", "pipe", "pipe"], }); - return { exitCode: 0, stderr, stdout }; - } catch (error) { - const result = error as Error & { code?: number; stderr?: string; stdout?: string }; - return { - exitCode: typeof result.code === "number" ? result.code : 1, - stderr: result.stderr ?? result.message, - stdout: result.stdout ?? "", + const output = { stdout: "", stderr: "" }; + let bytes = 0; + let truncated = false; + let timedOut = false; + const collect = (stream: "stdout" | "stderr") => (chunk: Buffer) => { + const kept = chunk.subarray(0, Math.max(0, MAX_OUTPUT_BYTES - bytes)); + bytes += kept.length; + output[stream] += kept.toString("utf8"); + if (kept.length < chunk.length) truncated = true; }; - } + child.stdout.on("data", collect("stdout")); + child.stderr.on("data", collect("stderr")); + const stop = () => { + try { + process.kill(-child.pid!, "SIGKILL"); + } catch { + // The group already exited. + } + }; + const timer = setTimeout(() => { + timedOut = true; + stop(); + }, timeoutSeconds * 1000); + ctx.abortSignal?.addEventListener("abort", stop, { once: true }); + let done = false; + const finish = (exitCode: number, note?: string) => { + if (done) return; + done = true; + clearTimeout(timer); + child.stdout.destroy(); + child.stderr.destroy(); + ctx.abortSignal?.removeEventListener("abort", stop); + const notes = [ + note, + timedOut ? `Command stopped after ${timeoutSeconds} seconds.` : undefined, + truncated ? `Output truncated at ${MAX_OUTPUT_BYTES} bytes.` : undefined, + ].filter(Boolean); + resolve({ + exitCode: timedOut ? 124 : exitCode, + stdout: output.stdout, + stderr: + notes.length > 0 ? `${output.stderr}\n[${notes.join(" ")}]`.trimStart() : output.stderr, + }); + }; + const exitCode = (code: number | null, signal: NodeJS.Signals | null) => + code ?? 128 + (signal ? (constants.signals[signal] ?? 0) : 0); + child.on("error", (error) => finish(1, error.message)); + child.on("close", (code, signal) => finish(exitCode(code, signal))); + // Background jobs can keep the pipes open after the shell exits; return with + // what was captured instead of waiting for them. + child.on("exit", (code, signal) => { + setTimeout(() => finish(exitCode(code, signal)), 250); + }); + }); }, }); diff --git a/packages/eve-bench/agent/tools/glob.ts b/packages/eve-bench/agent/tools/glob.ts index 764ac99ab9..b9fc371db1 100644 --- a/packages/eve-bench/agent/tools/glob.ts +++ b/packages/eve-bench/agent/tools/glob.ts @@ -4,7 +4,7 @@ import { defineTool } from "eve/tools"; import { never } from "eve/tools/approval"; import { z } from "zod"; -import { resolveTaskPath, taskRoot } from "../task-path.js"; +import { resolveTaskPath, taskRoot } from "../task-path.ts"; export default defineTool({ approval: never(), diff --git a/packages/eve-bench/agent/tools/grep.ts b/packages/eve-bench/agent/tools/grep.ts index 5ce048f7de..6cbdc0bd74 100644 --- a/packages/eve-bench/agent/tools/grep.ts +++ b/packages/eve-bench/agent/tools/grep.ts @@ -5,7 +5,7 @@ import { defineTool } from "eve/tools"; import { never } from "eve/tools/approval"; import { z } from "zod"; -import { resolveTaskPath, taskRoot } from "../task-path.js"; +import { resolveTaskPath, taskEnv, taskRoot } from "../task-path.ts"; const execFileAsync = promisify(execFile); @@ -28,12 +28,19 @@ export default defineTool({ try { const { stdout, stderr } = await execFileAsync("rg", args, { cwd: taskRoot(), + env: taskEnv(), maxBuffer: 10 * 1024 * 1024, signal: ctx.abortSignal, }); return { exitCode: 0, stderr, stdout }; } catch (error) { - const result = error as Error & { code?: number; stderr?: string; stdout?: string }; + const result = error as Error & { code?: number | string; stderr?: string; stdout?: string }; + // Otherwise a missing binary reads exactly like "no matches". + if (result.code === "ENOENT") { + throw new Error( + "ripgrep (rg) is not installed in this task environment; search with the bash tool (for example grep -rn) instead.", + ); + } const exitCode = typeof result.code === "number" ? result.code : 1; return { exitCode, diff --git a/packages/eve-bench/agent/tools/read_file.ts b/packages/eve-bench/agent/tools/read_file.ts index c64d4fe2da..bcaa4ad7b2 100644 --- a/packages/eve-bench/agent/tools/read_file.ts +++ b/packages/eve-bench/agent/tools/read_file.ts @@ -4,7 +4,7 @@ import { defineTool } from "eve/tools"; import { never } from "eve/tools/approval"; import { z } from "zod"; -import { resolveTaskPath } from "../task-path.js"; +import { resolveTaskPath } from "../task-path.ts"; export default defineTool({ approval: never(), diff --git a/packages/eve-bench/agent/tools/write_file.ts b/packages/eve-bench/agent/tools/write_file.ts index 65d547a02d..a31e52a732 100644 --- a/packages/eve-bench/agent/tools/write_file.ts +++ b/packages/eve-bench/agent/tools/write_file.ts @@ -5,7 +5,7 @@ import { defineTool } from "eve/tools"; import { never } from "eve/tools/approval"; import { z } from "zod"; -import { resolveTaskPath } from "../task-path.js"; +import { resolveTaskPath } from "../task-path.ts"; export default defineTool({ approval: never(), diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index 4ad272b0eb..47e2ce2426 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -9,7 +9,10 @@ "type": "module", "scripts": { "bench": "node src/cli.ts", - "test": "pnpm run typecheck && node --test src/core/*.test.ts", + "test": "pnpm run test:unit && pnpm run test:integration", + "test:unit": "pnpm run typecheck && node --test src/core/*.test.ts src/harnesses/**/*.test.ts src/tools/*.test.ts", + "test:integration": "node --test test/scenarios/*.test.ts", + "test:scenario": "node --test test/scenarios/*.test.ts", "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts index 95636fad45..cfd1d5a178 100644 --- a/packages/eve-bench/src/cli.ts +++ b/packages/eve-bench/src/cli.ts @@ -23,9 +23,11 @@ const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets eve-bench tasks list [--cohort ] print task names eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] - [--concurrency N] [--harness eve|oracle] [--eve local|] [--job ] - [--format console|json|junit] - eve-bench report [--format console|json|junit] + [--concurrency N] [--harness eve|e0|oracle|pi|opencode|codex] [--job ] + [--eve local|] [--agent ] [--version ] + [--reasoning ] [--base-url ] [--format console|json|junit] + eve-bench prepare --harness --model [harness options] build/cache without model calls + eve-bench report [--format console|json|junit] [--fail-on-invalid] eve-bench diff [--json] Jobs live in .generated/jobs/; re-running with the same --job resumes it. @@ -38,8 +40,9 @@ try { case "tasks": await tasks(subcommand, rest); break; + case "prepare": case "run": - await run(subcommand === undefined ? [] : [subcommand, ...rest]); + await run(subcommand === undefined ? [] : [subcommand, ...rest], command === "prepare"); break; case "report": await report(subcommand, rest); @@ -83,7 +86,7 @@ async function tasks(sub: string | undefined, args: string[]): Promise { throw new Error(`unknown tasks subcommand: ${sub ?? "(none)"}\n${USAGE}`); } -async function run(args: string[]): Promise { +async function run(args: string[], prepareOnly = false): Promise { const { values } = parseArgs({ args, options: { @@ -96,14 +99,27 @@ async function run(args: string[]): Promise { concurrency: { type: "string", default: "4" }, eve: { type: "string", default: "local" }, harness: { type: "string", default: "eve" }, + agent: { type: "string" }, + version: { type: "string" }, + "base-url": { type: "string" }, + reasoning: { type: "string" }, job: { type: "string" }, format: { type: "string", default: "console" }, json: { type: "boolean" }, }, }); - const harness = selectHarness(values.harness, values.eve); + const harness = selectHarness(values.harness, values.eve, { + ...values, + baseUrl: values["base-url"], + }); const model = values.model ?? (harness.name === "oracle" ? "none" : undefined); if (!model) throw new Error("--model is required"); + if (prepareOnly) { + const bundle = await harness.prepare({ model, cacheDir: join(paths.generatedRoot, "cache") }); + process.stdout.write(`${JSON.stringify(bundle, null, 2)}\n`); + return; + } + const forwardEnv = forwardedEnv(harness); const selection = await selectTasks({ ...values, taskDir: values["task-dir"] }); const datasetTaskDirs = selection.tasks.length ? await taskDirs( @@ -133,14 +149,14 @@ async function run(args: string[]): Promise { model, attempts: Number(values.attempts), concurrency: Number(values.concurrency), - forwardEnv: forwardedEnv(), + forwardEnv, signal: controller.signal, onLog: (message) => process.stderr.write(`${message}\n`), onTrial: (trial, resumed) => { if (format !== "console") return; const reward = trial.reward === null ? "-" : trial.reward.toFixed(2); process.stderr.write( - `${resumed ? "resume" : "trial "} ${trial.task}#${trial.attempt} reward=${reward} agent=${trial.agent.status} verifier=${trial.verifier.status}${trial.error ? ` error=${trial.error}` : ""}\n`, + `${resumed ? "resume" : "trial "} ${trial.task}#${trial.attempt} reward=${reward} agent=${trial.agent.status} verifier=${trial.verifier.status}${trial.invalid ? ` INVALID(${trial.invalid.phase}): ${trial.invalid.reason}` : ""}\n`, ); }, }); @@ -156,14 +172,20 @@ async function report(job: string | undefined, args: string[]): Promise { format: { type: "string", default: "console" }, json: { type: "boolean" }, out: { type: "string" }, + "fail-on-invalid": { type: "boolean" }, }, }); - const output = formatReport( - await readJobResult(join(paths.generatedRoot, "jobs", job)), - reportFormat(values.json ? "json" : values.format), - ); + const result = await readJobResult(join(paths.generatedRoot, "jobs", job)); + const output = formatReport(result, reportFormat(values.json ? "json" : values.format)); if (values.out) await writeFile(values.out, output); else process.stdout.write(output); + const invalid = Object.values(result.summary.invalid).reduce((sum, count) => sum + count, 0); + if (values["fail-on-invalid"] && invalid > 0) { + process.stderr.write( + `${invalid} of ${result.summary.attempts} attempt(s) are invalid and did not measure the harness.\n`, + ); + process.exitCode = 1; + } } async function diff(base: string | undefined, args: string[]): Promise { diff --git a/packages/eve-bench/src/core/harness.ts b/packages/eve-bench/src/core/harness.ts index 797d481d27..d36bf6b260 100644 --- a/packages/eve-bench/src/core/harness.ts +++ b/packages/eve-bench/src/core/harness.ts @@ -1,3 +1,5 @@ +import type { Usage } from "./result.ts"; + /** * A harness is anything that can be placed in a task container and told to * solve an instruction. The runner uploads `bundleDir` to `/installed-agent`, @@ -5,11 +7,20 @@ */ export interface Harness { readonly name: string; + /** Host environment names allowed to cross into this harness process. */ + readonly credentials?: readonly string[]; + /** Set when the harness makes no model calls, so missing usage does not invalidate a trial. */ + readonly modelFree?: boolean; prepare(ctx: HarnessPrepareContext): Promise; /** Optional per-task host paths uploaded into the install directory before `command` runs. */ stage?(ctx: { readonly taskDir: string }): readonly string[]; command(ctx: HarnessRunContext): string; env(ctx: HarnessRunContext): Record; + /** + * Reads root-session usage from the downloaded agent logs on the host. It runs + * after timeouts too, so harnesses should log usage incrementally. + */ + readUsage?(agentLogsDir: string): Promise; } export interface HarnessPrepareContext { diff --git a/packages/eve-bench/src/core/id.test.ts b/packages/eve-bench/src/core/id.test.ts new file mode 100644 index 0000000000..cccf1ed792 --- /dev/null +++ b/packages/eve-bench/src/core/id.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertSafeId } from "./id.ts"; + +test("accepts bounded path- and Docker-safe identifiers", () => { + for (const value of ["job", "task-1", "a.b_c", `a${"x".repeat(127)}`]) { + assert.equal(assertSafeId(value, "job"), value); + } +}); + +test("rejects traversal, separators, labels, whitespace, and oversized identifiers", () => { + for (const value of ["", ".", "..", "../job", "a/b", "a:b", "two words", `a${"x".repeat(128)}`]) { + assert.throws(() => assertSafeId(value, "task"), /path- and Docker-safe/); + } +}); diff --git a/packages/eve-bench/src/core/id.ts b/packages/eve-bench/src/core/id.ts new file mode 100644 index 0000000000..54bc39ecc5 --- /dev/null +++ b/packages/eve-bench/src/core/id.ts @@ -0,0 +1,8 @@ +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; + +export function assertSafeId(value: string, kind: "job" | "task"): string { + if (!SAFE_ID.test(value) || value === "." || value === "..") { + throw new Error(`${kind} name must be 1-128 path- and Docker-safe characters`); + } + return value; +} diff --git a/packages/eve-bench/src/core/job-identity.test.ts b/packages/eve-bench/src/core/job-identity.test.ts new file mode 100644 index 0000000000..dd61bd45dd --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { assertJobIdentity, createJobIdentity, identityHash } from "./job-identity.ts"; + +const input = { + name: "resume", + harness: "eve@local", + model: "provider/model", + attempts: 2, + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a", "b"], + }, + tasks: [ + { name: "a", hash: "aaa" }, + { name: "b", hash: "bbb" }, + ], + provenance: { key: "bundle", build: { version: "1", sha: "abc" } }, +}; + +test("identity is stable across object key and selected task order, and JSON round trips", () => { + const expected = createJobIdentity(input); + const reordered = createJobIdentity({ + ...input, + tasks: [...input.tasks].reverse(), + provenance: { build: { sha: "abc", version: "1" }, key: "bundle" }, + }); + assert.deepEqual(reordered, expected); + assertJobIdentity(JSON.parse(JSON.stringify(expected)), reordered); + assert.equal( + identityHash({ b: [1, { z: 2, a: 3 }], a: true }), + identityHash({ a: true, b: [1, { a: 3, z: 2 }] }), + ); + assert.notEqual(identityHash([1, 2]), identityHash([2, 1])); + assert.notEqual(identityHash(["ab", "c"]), identityHash(["a", "bc"])); +}); + +for (const [field, changes] of [ + ["job", { name: "different" }], + ["harness", { harness: "oracle" }], + ["model", { model: "provider/other" }], + ["attempts", { attempts: 3 }], + ["tasks", { tasks: [{ name: "renamed", hash: "aaa" }, input.tasks[1]!] }], + ["tasks", { tasks: [{ name: "a", hash: "changed" }, input.tasks[1]!] }], + ["tasks", { tasks: input.tasks.slice(0, 1) }], + ["provenance", { provenance: { key: "bundle", build: { version: "1", sha: "changed" } } }], + ...(["name", "version", "gitUrl", "commit"] as const).map( + (key) => ["dataset", { dataset: { ...input.dataset, [key]: "changed" } }] as const, + ), +] as const) { + test(`changed ${field} refuses resume with an actionable error`, () => { + assert.throws( + () => + assertJobIdentity(createJobIdentity(input), createJobIdentity({ ...input, ...changes })), + new RegExp(`differs \\(.*${field}.*\\).*new job name`), + ); + }); +} + +test("unselected dataset task inventory does not change selected content identity", () => { + assert.deepEqual( + createJobIdentity({ ...input, dataset: { ...input.dataset, tasks: ["a", "b", "unselected"] } }), + createJobIdentity(input), + ); +}); + +test("legacy, unsupported, and incomplete identities fail closed", () => { + const expected = createJobIdentity(input); + for (const saved of [undefined, null, {}, "bad", { ...expected, version: 2 }, { version: 1 }]) { + assert.throws(() => assertJobIdentity(saved, expected), /new job name/); + } +}); + +test("duplicate task names cannot share trial directories", () => { + assert.throws( + () => createJobIdentity({ ...input, tasks: [input.tasks[0]!, input.tasks[0]!] }), + /unique/, + ); +}); + +test("absent provenance is stable and distinct from recorded provenance", () => { + const { provenance: _provenance, ...without } = input; + assert.deepEqual(createJobIdentity(without), createJobIdentity({ ...without, provenance: {} })); + assert.throws( + () => assertJobIdentity(createJobIdentity(without), createJobIdentity(input)), + /provenance/, + ); +}); diff --git a/packages/eve-bench/src/core/job-identity.ts b/packages/eve-bench/src/core/job-identity.ts new file mode 100644 index 0000000000..c4b0001dc0 --- /dev/null +++ b/packages/eve-bench/src/core/job-identity.ts @@ -0,0 +1,129 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, opendir } from "node:fs/promises"; +import { isAbsolute, join, relative, sep } from "node:path"; + +import type { DatasetLock } from "./dataset.ts"; +import type { Task } from "./task.ts"; + +export interface JobIdentity { + readonly version: 1; + readonly job: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: string; + readonly tasks: string; + readonly provenance: string; +} + +export function identityHash(value: unknown): string { + const json = JSON.stringify(value, (_key, item: unknown) => { + if (item !== null && typeof item === "object" && !Array.isArray(item)) { + return Object.fromEntries( + Object.entries(item).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + } + return item; + }); + if (json === undefined) throw new Error("Job identity must be JSON serializable."); + return createHash("sha256").update(json).digest("hex"); +} + +export function createJobIdentity(input: { + readonly name: string; + readonly harness: string; + readonly model: string; + readonly attempts: number; + readonly dataset: DatasetLock; + readonly tasks: readonly { readonly name: string; readonly hash: string }[]; + readonly provenance?: Record; +}): JobIdentity { + const { name, version, gitUrl, commit } = input.dataset; + const tasks = [...input.tasks].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + if (new Set(tasks.map((task) => task.name)).size !== tasks.length) { + throw new Error("Selected task names must be unique to resume safely."); + } + return { + version: 1, + job: input.name, + harness: input.harness, + model: input.model, + attempts: input.attempts, + dataset: identityHash({ name, version, gitUrl, commit }), + tasks: identityHash(tasks), + provenance: identityHash(input.provenance ?? {}), + }; +} + +export function assertJobIdentity(saved: unknown, expected: JobIdentity): void { + if (!saved || typeof saved !== "object" || !("version" in saved) || saved.version !== 1) { + throw new Error( + `Job ${expected.job} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + const fields = Object.keys(expected) as (keyof JobIdentity)[]; + const changed = fields.filter( + (key) => !(key in saved) || Reflect.get(saved, key) !== expected[key], + ); + if (changed.length > 0) { + throw new Error( + `Job ${expected.job} resume identity differs (${changed.join(", ")}). Use a new job name for changed inputs.`, + ); + } +} + +/** Hash only the selected tree, independent of its location, mtimes, and traversal order. */ +export async function hashTask(task: Task): Promise { + const dockerfileDir = relative(task.dir, task.environment.dockerfileDir); + if (isAbsolute(dockerfileDir) || dockerfileDir === ".." || dockerfileDir.startsWith(`..${sep}`)) { + throw new Error(`Task ${task.name}: environment must be inside the selected task tree.`); + } + const entries: { path: string; kind: string; executable: number; hash?: string }[] = []; + let bytes = 0; + let count = 0; + async function visit(path: string, depth: number): Promise { + if (depth > 64 || ++count > 100_000) { + throw new Error(`Task ${task.name}: identity scan exceeds 64 levels or 100,000 entries.`); + } + const absolute = join(task.dir, path); + const stat = await lstat(absolute); + // Do not follow links into unselected tasks (or hash a link but execute its changed target). + if (!stat.isDirectory() && !stat.isFile()) { + throw new Error( + `Task ${task.name}: identity requires regular files/directories, not ${path || "."}.`, + ); + } + const entry = { + path: path.split(sep).join("/"), + kind: stat.isDirectory() ? "directory" : "file", + executable: stat.mode & 0o111, + }; + if (stat.isDirectory()) { + entries.push(entry); + for await (const child of await opendir(absolute)) { + await visit(join(path, child.name), depth + 1); + } + } else { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(absolute)) { + bytes += chunk.length; + if (bytes > 1024 * 1024 * 1024) { + throw new Error(`Task ${task.name}: identity scan exceeds 1 GiB.`); + } + hash.update(chunk); + } + entries.push({ ...entry, hash: hash.digest("hex") }); + } + } + await visit("", 0); + entries.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + return identityHash({ + entries, + instruction: task.instruction, + agentTimeoutMs: task.agentTimeoutMs, + verifierTimeoutMs: task.verifierTimeoutMs, + buildTimeoutMs: task.buildTimeoutMs, + environment: { ...task.environment, dockerfileDir: dockerfileDir.split(sep).join("/") }, + }); +} diff --git a/packages/eve-bench/src/core/job.ts b/packages/eve-bench/src/core/job.ts index 18f21de5ef..ab35dd7f82 100644 --- a/packages/eve-bench/src/core/job.ts +++ b/packages/eve-bench/src/core/job.ts @@ -1,10 +1,13 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, open, readFile, readdir, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import type { DatasetLock } from "./dataset.ts"; import { assertDockerAvailable, listRunnerContainers, removeRunnerContainers } from "./docker.ts"; import type { Harness } from "./harness.ts"; -import { summarize, type JobResult, type TrialResult } from "./result.ts"; +import { assertSafeId } from "./id.ts"; +import { assertJobIdentity, createJobIdentity, hashTask, identityHash } from "./job-identity.ts"; +import { assertTrialResult, summarize, type JobResult, type TrialResult } from "./result.ts"; import type { Task } from "./task.ts"; import { runTrial } from "./trial.ts"; @@ -25,37 +28,77 @@ export interface JobInput { } export async function runJob(input: JobInput): Promise { + assertSafeId(input.name, "job"); + for (const task of input.tasks) assertSafeId(task.name, "task"); + // Containers are labeled by name, so different output directories must share the lock. + const lockPath = join(tmpdir(), `eve-bench-job-${identityHash(input.name)}.lock`); + const lock = await open(lockPath, "wx", 0o600).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + throw new Error( + `Job ${input.name} is locked. Wait for the active run or use a new job name. If the owner crashed, confirm it has stopped before removing ${lockPath}.`, + ); + }); + try { + await lock.writeFile(`${JSON.stringify({ pid: process.pid, job: input.name })}\n`); + return await runLockedJob(input); + } finally { + await lock.close(); + await unlink(lockPath); + } +} + +async function runLockedJob(input: JobInput): Promise { + await mkdir(input.dir, { recursive: true }); + const savedIdentity = await readSavedIdentity(input.dir, input.name); + const taskHashes = []; + for (const task of input.tasks) taskHashes.push({ name: task.name, hash: await hashTask(task) }); + const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); + const identity = createJobIdentity({ + ...input, + harness: input.harness.name, + tasks: taskHashes, + provenance: bundle.provenance, + }); + if (savedIdentity !== undefined) assertJobIdentity(savedIdentity, identity); + await assertDockerAvailable(); const leftovers = await listRunnerContainers({ job: input.name }); + if (savedIdentity === undefined && leftovers.length > 0) { + throw new Error( + `Job ${input.name} has containers but no resume identity here. Use a new job name or resume from the original job directory.`, + ); + } + if (savedIdentity === undefined) { + await writeFile( + join(input.dir, "job.json"), + `${JSON.stringify( + { + job: input.name, + dataset: { + name: input.dataset.name, + version: input.dataset.version, + commit: input.dataset.commit, + }, + harness: input.harness.name, + model: input.model, + attempts: input.attempts, + tasks: input.tasks.map((task) => task.name), + bundle: bundle.provenance ?? {}, + identity, + startedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + { flag: "wx" }, + ); + } if (leftovers.length > 0) { await removeRunnerContainers({ job: input.name }); for (const container of leftovers) { input.onLog?.(`Removed leftover container ${container.name} from job ${input.name}`); } } - await mkdir(input.dir, { recursive: true }); - const bundle = await input.harness.prepare({ model: input.model, cacheDir: input.cacheDir }); - await writeFile( - join(input.dir, "job.json"), - `${JSON.stringify( - { - job: input.name, - dataset: { - name: input.dataset.name, - version: input.dataset.version, - commit: input.dataset.commit, - }, - harness: input.harness.name, - model: input.model, - attempts: input.attempts, - tasks: input.tasks.map((task) => task.name), - bundle: bundle.provenance ?? {}, - startedAt: new Date().toISOString(), - }, - null, - 2, - )}\n`, - ); const queue = input.tasks.flatMap((task) => Array.from({ length: input.attempts }, (_, index) => ({ task, attempt: index + 1 })), @@ -66,7 +109,12 @@ export async function runJob(input: JobInput): Promise { const next = queue.shift(); if (!next || input.signal.aborted) return; const dir = join(input.dir, next.task.name, String(next.attempt)); - const existing = await readTrial(dir); + const existing = await readTrial(dir, { + task: next.task.name, + attempt: next.attempt, + harness: input.harness.name, + model: input.model, + }); if (existing) { trials.push(existing); input.onTrial?.(existing, true); @@ -114,10 +162,57 @@ export async function readJobResult(dir: string): Promise { return JSON.parse(await readFile(join(dir, "result.json"), "utf8")) as JobResult; } -async function readTrial(dir: string): Promise { +async function readSavedIdentity(dir: string, name: string): Promise { + let text: string; + try { + text = await readFile(join(dir, "job.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if ((await readdir(dir)).length === 0) return undefined; + throw new Error( + `Job ${name} has existing files but no job.json. Use a new job name to run safely.`, + ); + } + let saved; + try { + saved = JSON.parse(text); + } catch { + throw new Error(`Job ${name} has invalid job.json. Use a new job name to run safely.`); + } + if (saved?.identity?.version !== 1) { + throw new Error( + `Job ${name} has no supported resume identity. Use a new job name; legacy jobs cannot be resumed safely.`, + ); + } + return saved.identity; +} + +async function readTrial( + dir: string, + expected: Pick, +): Promise { + let text: string; try { - return JSON.parse(await readFile(join(dir, "trial.json"), "utf8")) as TrialResult; + text = await readFile(join(dir, "trial.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); } catch { - return undefined; + throw new Error( + `Trial ${expected.task}#${expected.attempt} has invalid JSON; remove the job or use a new job name.`, + ); + } + assertTrialResult(parsed); + for (const key of ["task", "attempt", "harness", "model"] as const) { + if (parsed[key] !== expected[key]) { + throw new Error( + `Trial ${expected.task}#${expected.attempt} has mismatched ${key}; use a new job name.`, + ); + } } + return parsed; } diff --git a/packages/eve-bench/src/core/report.test.ts b/packages/eve-bench/src/core/report.test.ts index 70ff0e7390..5e03733735 100644 --- a/packages/eve-bench/src/core/report.test.ts +++ b/packages/eve-bench/src/core/report.test.ts @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { diffJobs, formatDiff, formatReport } from "./report.ts"; -import { summarize, type JobResult, type TrialResult } from "./result.ts"; +import { classifyTrial, summarize, type JobResult, type TrialResult } from "./result.ts"; + +const USED = { inputTokens: 100, outputTokens: 10, cachedTokens: 0, costUsd: null }; function trial( task: string, @@ -18,6 +20,7 @@ function trial( reward, agent: { status: "completed", exitCode: 0, durationMs: 1000 }, verifier: { status: "completed", exitCode: 0, durationMs: 500 }, + usage: USED, startedAt: "2026-01-01T00:00:00.000Z", finishedAt: "2026-01-01T00:00:01.500Z", ...extra, @@ -35,29 +38,102 @@ function job(name: string, trials: TrialResult[]): JobResult { }; } -test("summarize counts resolved attempts and mean reward", () => { - const summary = summarize([trial("a", 1, 1), trial("a", 2, 0), trial("b", 1, null)]); - assert.deepEqual(summary, { tasks: 2, attempts: 3, resolved: 1, meanReward: 1 / 3 }); +const harnessInvalid = { phase: "harness", reason: "no model call completed" } as const; + +test("classify separates environment, harness, and verifier failures from scored outcomes", () => { + const base = trial("a", 1, 0); + assert.equal(classifyTrial(base, false), undefined); + assert.equal( + classifyTrial({ ...base, agent: { ...base.agent, status: "timeout" } }, false), + undefined, + ); + + const setup = classifyTrial( + { + ...base, + error: "docker pull failed", + agent: { status: "skipped", exitCode: null, durationMs: 0 }, + }, + false, + ); + assert.equal(setup?.phase, "environment"); + assert.match(setup!.reason, /before the harness started: docker pull failed/); + assert.equal( + classifyTrial({ ...base, error: "could not download /logs/agent" }, false)?.phase, + "environment", + ); + + // A 401 or broken adapter leaves zero usage; a missing log leaves none. + const zero = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: null }; + assert.equal(classifyTrial({ ...base, usage: zero }, false)?.phase, "harness"); + assert.equal(classifyTrial({ ...base, usage: undefined, reward: 1 }, false)?.phase, "harness"); + assert.equal(classifyTrial({ ...base, usage: undefined }, true), undefined); + + const verifier = classifyTrial( + { ...base, reward: null, verifier: { status: "timeout", exitCode: null, durationMs: 1 } }, + false, + ); + assert.deepEqual(verifier, { + phase: "verifier", + reason: "verifier timed out without writing a reward", + }); }); -test("junit report marks unresolved trials as failures", () => { +test("summarize scores only valid attempts and counts invalid ones by phase", () => { + const summary = summarize([ + trial("a", 1, 1), + trial("a", 2, 0), + trial("b", 1, 0, { invalid: harnessInvalid }), + trial("b", 2, null, { invalid: { phase: "verifier", reason: "verifier wrote no reward" } }), + ]); + assert.deepEqual(summary, { + tasks: 2, + attempts: 4, + scored: 2, + invalid: { environment: 0, harness: 1, verifier: 1 }, + resolved: 1, + meanReward: 0.5, + }); + assert.equal(summarize([trial("a", 1, 0, { invalid: harnessInvalid })]).meanReward, null); +}); + +test("junit reports invalid attempts as errors, not failures", () => { const output = formatReport( - job("j", [trial("a", 1, 1), trial("b", 1, 0, { error: "boom " })]), + job("j", [trial("a", 1, 1), trial("b", 1, 0), trial("c", 1, 0, { invalid: harnessInvalid })]), "junit", ); - assert.match(output, /tests="2" failures="1"/u); + assert.match(output, /tests="3" failures="1" errors="1"/u); assert.match(output, //u); - assert.match(output, //u); + assert.match(output, //u); + assert.match(output, //u); +}); + +test("console report lists every invalid attempt with its reason", () => { + const output = formatReport( + job("j", [trial("a", 1, 1), trial("a", 2, 0, { invalid: harnessInvalid })]), + "console", + ); + assert.match(output, /1\.00 x\s+#2 invalid\(harness\)/u); + assert.match(output, /scored 1 {2}resolved 1 {2}mean reward 1\.000/u); + assert.match(output, /invalid attempts excluded from scoring: harness 1/u); + assert.match(output, /a #2 \[harness\] no model call completed/u); }); -test("diff reports per-task mean reward deltas", () => { +test("diff compares scored attempts and reports invalid counts", () => { const base = job("base", [trial("a", 1, 0), trial("a", 2, 1), trial("b", 1, 1)]); - const candidate = job("cand", [trial("a", 1, 1), trial("a", 2, 1), trial("c", 1, 0)]); + const candidate = job("cand", [ + trial("a", 1, 1), + trial("a", 2, 0, { invalid: harnessInvalid }), + trial("c", 1, 0), + ]); const deltas = diffJobs(base, candidate); assert.deepEqual(deltas, [ - { task: "a", base: 0.5, candidate: 1, delta: 0.5 }, - { task: "b", base: 1, candidate: null, delta: null }, - { task: "c", base: null, candidate: 0, delta: null }, + { task: "a", base: 0.5, candidate: 1, delta: 0.5, baseInvalid: 0, candidateInvalid: 1 }, + { task: "b", base: 1, candidate: null, delta: null, baseInvalid: 0, candidateInvalid: 0 }, + { task: "c", base: null, candidate: 0, delta: null, baseInvalid: 0, candidateInvalid: 0 }, ]); - assert.match(formatDiff(deltas), /1 task\(s\) changed, net \+0.500/u); + const text = formatDiff(deltas); + assert.match(text, /1 task\(s\) changed, net \+0.500/u); + assert.match(text, /invalid 0 -> 1/u); + assert.match(text, /1 invalid attempt\(s\) excluded/u); }); diff --git a/packages/eve-bench/src/core/report.ts b/packages/eve-bench/src/core/report.ts index 5a931878da..79e3ccb10a 100644 --- a/packages/eve-bench/src/core/report.ts +++ b/packages/eve-bench/src/core/report.ts @@ -20,36 +20,59 @@ function console_(result: JobResult): string { ]; const byTask = groupByTask(result.trials); for (const [task, trials] of byTask) { - const rewards = trials.map((trial) => (trial.reward === null ? "-" : trial.reward.toFixed(2))); - const errors = trials.filter((trial) => trial.error || trial.agent.status === "timeout").length; - lines.push( - `${task.padEnd(44)} ${rewards.join(" ").padEnd(16)} ${errors > 0 ? `${errors} error(s)` : ""}`.trimEnd(), + const rewards = trials.map((trial) => + trial.invalid ? "x" : trial.reward === null ? "-" : trial.reward.toFixed(2), ); + const invalid = trials.filter((trial) => trial.invalid); + const timeouts = trials.filter((trial) => !trial.invalid && trial.agent.status === "timeout"); + const notes = [ + ...invalid.map((trial) => `#${trial.attempt} invalid(${trial.invalid!.phase})`), + ...(timeouts.length > 0 ? [`${timeouts.length} agent timeout(s)`] : []), + ]; + lines.push(`${task.padEnd(44)} ${rewards.join(" ").padEnd(16)} ${notes.join(", ")}`.trimEnd()); } const { summary } = result; + const invalid = Object.entries(summary.invalid) + .filter(([, count]) => count > 0) + .map(([phase, count]) => `${phase} ${count}`); + const mean = summary.meanReward === null ? "-" : summary.meanReward.toFixed(3); lines.push( "", - `tasks ${summary.tasks} attempts ${summary.attempts} resolved ${summary.resolved} mean reward ${summary.meanReward.toFixed(3)}`, + `tasks ${summary.tasks} attempts ${summary.attempts} scored ${summary.scored} resolved ${summary.resolved} mean reward ${mean}`, ); + if (invalid.length > 0) { + lines.push(`invalid attempts excluded from scoring: ${invalid.join(", ")}`); + for (const trial of result.trials.filter((trial) => trial.invalid)) { + lines.push( + ` ${trial.task} #${trial.attempt} [${trial.invalid!.phase}] ${trial.invalid!.reason}`, + ); + } + } return `${lines.join("\n")}\n`; } function junit(result: JobResult): string { - const failures = result.trials.filter((trial) => (trial.reward ?? 0) < 1).length; + const invalid = result.trials.filter((trial) => trial.invalid).length; + const failures = result.trials.filter( + (trial) => !trial.invalid && (trial.reward ?? 0) < 1, + ).length; const cases = result.trials.map((trial) => { const time = ((trial.agent.durationMs + trial.verifier.durationMs) / 1000).toFixed(3); const name = escape(`${trial.task} #${trial.attempt}`); + if (trial.invalid) { + const message = escape(`invalid (${trial.invalid.phase}): ${trial.invalid.reason}`); + return ` \n \n `; + } if ((trial.reward ?? 0) >= 1) return ` `; const message = escape( - trial.error ?? - `reward ${trial.reward ?? "none"}; agent ${trial.agent.status}; verifier ${trial.verifier.status}`, + `reward ${trial.reward ?? "none"}; agent ${trial.agent.status}; verifier ${trial.verifier.status}`, ); return ` \n \n `; }); return [ ``, ``, - ` `, + ` `, ...cases, ` `, ``, @@ -59,19 +82,31 @@ function junit(result: JobResult): string { export interface TaskDelta { readonly task: string; + /** Mean reward over scored attempts; null when the job has none for this task. */ readonly base: number | null; readonly candidate: number | null; readonly delta: number | null; + readonly baseInvalid: number; + readonly candidateInvalid: number; } export function diffJobs(base: JobResult, candidate: JobResult): TaskDelta[] { - const baseMeans = meanByTask(base.trials); - const candidateMeans = meanByTask(candidate.trials); - const tasks = [...new Set([...baseMeans.keys(), ...candidateMeans.keys()])].sort(); + const baseStats = statsByTask(base.trials); + const candidateStats = statsByTask(candidate.trials); + const tasks = [...new Set([...baseStats.keys(), ...candidateStats.keys()])].sort(); return tasks.map((task) => { - const b = baseMeans.get(task) ?? null; - const c = candidateMeans.get(task) ?? null; - return { task, base: b, candidate: c, delta: b === null || c === null ? null : c - b }; + const b = baseStats.get(task); + const c = candidateStats.get(task); + const baseMean = b?.mean ?? null; + const candidateMean = c?.mean ?? null; + return { + task, + base: baseMean, + candidate: candidateMean, + delta: baseMean === null || candidateMean === null ? null : candidateMean - baseMean, + baseInvalid: b?.invalid ?? 0, + candidateInvalid: c?.invalid ?? 0, + }; }); } @@ -87,11 +122,19 @@ export function formatDiff(deltas: readonly TaskDelta[]): string { : row.delta > 0 ? `+${row.delta.toFixed(3)}` : row.delta.toFixed(3); - return `${row.task.padEnd(44)} ${fmt(row.base)} -> ${fmt(row.candidate)} ${delta}`.trimEnd(); + const invalid = + row.baseInvalid + row.candidateInvalid > 0 + ? `invalid ${row.baseInvalid} -> ${row.candidateInvalid}` + : ""; + return `${row.task.padEnd(44)} ${fmt(row.base)} -> ${fmt(row.candidate)} ${delta.padEnd(7)} ${invalid}`.trimEnd(); }); const changed = deltas.filter((row) => row.delta !== null && row.delta !== 0); const net = changed.reduce((sum, row) => sum + (row.delta ?? 0), 0); + const invalid = deltas.reduce((sum, row) => sum + row.baseInvalid + row.candidateInvalid, 0); lines.push("", `${changed.length} task(s) changed, net ${net >= 0 ? "+" : ""}${net.toFixed(3)}`); + if (invalid > 0) { + lines.push(`${invalid} invalid attempt(s) excluded; compare only after resolving them`); + } return `${lines.join("\n")}\n`; } @@ -105,12 +148,21 @@ function groupByTask(trials: readonly TrialResult[]): Map return groups; } -function meanByTask(trials: readonly TrialResult[]): Map { - const means = new Map(); +function statsByTask( + trials: readonly TrialResult[], +): Map { + const stats = new Map(); for (const [task, group] of groupByTask(trials)) { - means.set(task, group.reduce((sum, trial) => sum + (trial.reward ?? 0), 0) / group.length); + const scored = group.filter((trial) => !trial.invalid); + stats.set(task, { + mean: + scored.length === 0 + ? null + : scored.reduce((sum, trial) => sum + (trial.reward ?? 0), 0) / scored.length, + invalid: group.length - scored.length, + }); } - return means; + return stats; } function escape(text: string): string { diff --git a/packages/eve-bench/src/core/result.ts b/packages/eve-bench/src/core/result.ts index d42e58a798..62933ac171 100644 --- a/packages/eve-bench/src/core/result.ts +++ b/packages/eve-bench/src/core/result.ts @@ -6,11 +6,31 @@ export interface StepResult { readonly durationMs: number; } +/** + * Root-session model usage as reported by the harness. `inputTokens` includes + * cache reads and `outputTokens` includes reasoning, so harnesses that report + * those separately are normalized to the same totals. + */ export interface Usage { readonly inputTokens: number; readonly outputTokens: number; readonly cachedTokens: number; - readonly costUsd: number; + /** Null when the harness reported no cost (for example, no pricing data for a custom provider). */ + readonly costUsd: number | null; +} + +/** + * Where a trial broke before it could measure the harness: + * - `environment`: runner-owned work (image, container, uploads, log download) failed. + * - `harness`: the harness ran but completed no model call, so its adapter, + * credentials, or launch are broken rather than its task-solving. + * - `verifier`: the verifier produced no reward. + */ +export type InvalidPhase = "environment" | "harness" | "verifier"; + +export interface InvalidTrial { + readonly phase: InvalidPhase; + readonly reason: string; } export interface TrialResult { @@ -23,6 +43,8 @@ export interface TrialResult { readonly verifier: StepResult; readonly usage?: Usage; readonly error?: string; + /** Present when the trial is excluded from scoring. */ + readonly invalid?: InvalidTrial; readonly startedAt: string; readonly finishedAt: string; } @@ -39,14 +61,101 @@ export interface JobResult { export interface JobSummary { readonly tasks: number; readonly attempts: number; + /** Attempts that measured the harness; invalid attempts are excluded. */ + readonly scored: number; + readonly invalid: Readonly>; readonly resolved: number; - readonly meanReward: number; + /** Mean reward over scored attempts; null when none were scored. */ + readonly meanReward: number | null; +} + +const PHASES: readonly InvalidPhase[] = ["environment", "harness", "verifier"]; + +export function classifyTrial( + trial: Pick, + modelFree: boolean, +): InvalidTrial | undefined { + if (trial.error !== undefined) { + return { + phase: "environment", + reason: + trial.agent.status === "skipped" + ? `trial setup failed before the harness started: ${trial.error}` + : `runner failed after the harness started: ${trial.error}`, + }; + } + if (!modelFree && !reachedModel(trial.usage)) { + return { + phase: "harness", + reason: trial.usage + ? "harness reported zero token usage; no model call completed" + : "harness reported no model usage; no model call completed", + }; + } + if (trial.reward === null) { + return { + phase: "verifier", + reason: + trial.verifier.status === "timeout" + ? "verifier timed out without writing a reward" + : "verifier wrote no reward", + }; + } + return undefined; +} + +function reachedModel(usage: Usage | undefined): boolean { + return usage !== undefined && (usage.inputTokens > 0 || usage.outputTokens > 0); +} + +export function assertTrialResult(value: unknown): asserts value is TrialResult { + if (!value || typeof value !== "object") throw new Error("trial record must be an object"); + const trial = value as Partial; + const step = (value: unknown) => { + const item = value as Partial | undefined; + return ( + item !== undefined && + ["completed", "timeout", "failed", "skipped"].includes(item.status ?? "") && + (item.exitCode === null || Number.isInteger(item.exitCode)) && + typeof item.durationMs === "number" && + Number.isFinite(item.durationMs) + ); + }; + const invalid = trial.invalid as Partial | undefined; + if ( + typeof trial.task !== "string" || + !Number.isInteger(trial.attempt) || + typeof trial.harness !== "string" || + typeof trial.model !== "string" || + !( + trial.reward === null || + (typeof trial.reward === "number" && Number.isFinite(trial.reward)) + ) || + !step(trial.agent) || + !step(trial.verifier) || + (invalid !== undefined && + (!PHASES.includes(invalid.phase as InvalidPhase) || typeof invalid.reason !== "string")) || + typeof trial.startedAt !== "string" || + typeof trial.finishedAt !== "string" + ) { + throw new Error("trial record has an invalid shape"); + } } export function summarize(trials: readonly TrialResult[]): JobSummary { const tasks = new Set(trials.map((trial) => trial.task)).size; - const rewards = trials.map((trial) => trial.reward ?? 0); - const resolved = rewards.filter((reward) => reward >= 1).length; - const meanReward = rewards.length === 0 ? 0 : rewards.reduce((a, b) => a + b, 0) / rewards.length; - return { tasks, attempts: trials.length, resolved, meanReward }; + const invalid = { environment: 0, harness: 0, verifier: 0 }; + const scored: number[] = []; + for (const trial of trials) { + if (trial.invalid) invalid[trial.invalid.phase]++; + else scored.push(trial.reward ?? 0); + } + return { + tasks, + attempts: trials.length, + scored: scored.length, + invalid, + resolved: scored.filter((reward) => reward >= 1).length, + meanReward: scored.length === 0 ? null : scored.reduce((a, b) => a + b, 0) / scored.length, + }; } diff --git a/packages/eve-bench/src/core/task.ts b/packages/eve-bench/src/core/task.ts index 8788b8e74b..d2a2b61859 100644 --- a/packages/eve-bench/src/core/task.ts +++ b/packages/eve-bench/src/core/task.ts @@ -1,6 +1,7 @@ import { access, readFile } from "node:fs/promises"; import { basename, join } from "node:path"; +import { assertSafeId } from "./id.ts"; import { parseToml, type TomlTable } from "./toml.ts"; export interface Task { @@ -33,7 +34,7 @@ export async function loadTask(dir: string): Promise { const dockerfileDir = join(dir, "environment"); if (dockerImage === undefined) await access(join(dockerfileDir, "Dockerfile")); return { - name: shortName(str(task, "name") ?? basename(dir)), + name: assertSafeId(shortName(str(task, "name") ?? basename(dir)), "task"), dir, instruction, agentTimeoutMs: seconds(agent, "timeout_sec"), diff --git a/packages/eve-bench/src/core/trial.ts b/packages/eve-bench/src/core/trial.ts index c91f71eeb1..87489b3f96 100644 --- a/packages/eve-bench/src/core/trial.ts +++ b/packages/eve-bench/src/core/trial.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Writable } from "node:stream"; import { finished } from "node:stream/promises"; @@ -13,7 +13,13 @@ import { type ProcessResult, } from "./docker.ts"; import type { Harness, HarnessBundle } from "./harness.ts"; -import type { StepResult, TrialResult, Usage } from "./result.ts"; +import { + classifyTrial, + type InvalidTrial, + type StepResult, + type TrialResult, + type Usage, +} from "./result.ts"; import type { Task } from "./task.ts"; export interface TrialInput { @@ -107,13 +113,15 @@ export async function runTrial(input: TrialInput): Promise { ), ); + const missing: string[] = []; for (const logs of ["agent", "verifier"]) { if (!(await container.download(`${LOGS_DIR}/${logs}`, input.dir))) { - containerLog.write(`[eve-bench] could not download ${LOGS_DIR}/${logs}\n`); + missing.push(`${LOGS_DIR}/${logs}`); } } reward = await readReward(join(input.dir, "verifier", "reward.txt")); - usage = await readUsage(join(input.dir, "agent", "result.json")); + usage = await input.harness.readUsage?.(join(input.dir, "agent")); + if (missing.length > 0) error = `could not download ${missing.join(", ")}`; } catch (caught) { error = caught instanceof Error ? caught.message : String(caught); } finally { @@ -121,6 +129,14 @@ export async function runTrial(input: TrialInput): Promise { containerLog.end(); await finished(containerLog); } + const invalid = classifyTrial( + { agent, verifier, reward, usage, error }, + input.harness.modelFree === true, + ); + const recorded: { usage?: Usage; error?: string; invalid?: InvalidTrial } = {}; + if (usage) recorded.usage = usage; + if (error) recorded.error = error; + if (invalid) recorded.invalid = invalid; const result: TrialResult = { task: task.name, attempt: input.attempt, @@ -129,12 +145,18 @@ export async function runTrial(input: TrialInput): Promise { reward, agent, verifier, - ...(usage ? { usage } : {}), - ...(error ? { error } : {}), + ...recorded, startedAt, finishedAt: new Date().toISOString(), }; - await writeFile(join(input.dir, "trial.json"), `${JSON.stringify(result, null, 2)}\n`); + const target = join(input.dir, "trial.json"); + const temporary = join(input.dir, `trial.${process.pid}.tmp`); + try { + await writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`, { flag: "wx" }); + await rename(temporary, target); + } finally { + await rm(temporary, { force: true }); + } return result; } @@ -195,19 +217,3 @@ async function readReward(path: string): Promise { return null; } } - -async function readUsage(path: string): Promise { - try { - const parsed = JSON.parse(await readFile(path, "utf8")) as { usage?: Partial }; - const usage = parsed.usage; - if (!usage) return undefined; - return { - inputTokens: usage.inputTokens ?? 0, - outputTokens: usage.outputTokens ?? 0, - cachedTokens: usage.cachedTokens ?? 0, - costUsd: usage.costUsd ?? 0, - }; - } catch { - return undefined; - } -} diff --git a/packages/eve-bench/src/core/usage.test.ts b/packages/eve-bench/src/core/usage.test.ts new file mode 100644 index 0000000000..668aaa43ca --- /dev/null +++ b/packages/eve-bench/src/core/usage.test.ts @@ -0,0 +1,123 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { CLI_USAGE, readCliUsage } from "../harnesses/cli/usage.ts"; +import { eveStepUsage, readEveUsage } from "../harnesses/eve/usage.ts"; +import { readJsonlUsage } from "./usage.ts"; + +// Shapes below are copied from recorded native logs (pi 0.85.1, opencode, eve 0.35). +const piAssistant = (input: number, cacheRead: number, output: number, total = 0) => ({ + type: "message_end", + message: { + role: "assistant", + usage: { + input, + output, + cacheRead, + cacheWrite: 0, + totalTokens: input + output, + cost: { total }, + }, + stopReason: "toolUse", + }, +}); + +test("pi usage sums assistant message_end records and adds cache reads to input", () => { + assert.equal( + CLI_USAGE.pi({ type: "message_end", message: { role: "user", usage: null } }), + undefined, + ); + assert.equal(CLI_USAGE.pi({ type: "agent_end", messages: [] }), undefined); + assert.deepEqual(CLI_USAGE.pi(piAssistant(100, 900, 50, 0.01)), { + input: 1000, + output: 50, + cached: 900, + cost: 0.01, + }); +}); + +test("opencode usage counts reasoning as output and cache reads as input", () => { + const stepFinish = { + type: "step_finish", + part: { + type: "step-finish", + reason: "tool-calls", + tokens: { + total: 8542, + input: 8154, + output: 33, + reasoning: 355, + cache: { write: 0, read: 12 }, + }, + cost: 0, + }, + }; + assert.deepEqual(CLI_USAGE.opencode(stepFinish), { + input: 8166, + output: 388, + cached: 12, + cost: 0, + }); + assert.equal(CLI_USAGE.opencode({ type: "text", part: {} }), undefined); +}); + +test("codex usage reads turn.completed totals", () => { + assert.deepEqual( + CLI_USAGE.codex({ + type: "turn.completed", + usage: { input_tokens: 24763, cached_input_tokens: 24448, output_tokens: 122 }, + }), + { input: 24763, output: 122, cached: 24448 }, + ); + assert.equal(CLI_USAGE.codex({ type: "turn.failed", error: { message: "401" } }), undefined); +}); + +test("eve usage reads step.completed usage", () => { + assert.deepEqual( + eveStepUsage({ + type: "step.completed", + data: { usage: { inputTokens: 10, outputTokens: 2, cacheReadTokens: 4, costUsd: 0.5 } }, + }), + { input: 10, output: 2, cached: 4, cost: 0.5 }, + ); + assert.equal(eveStepUsage({ type: "step.started", data: { modelId: "x" } }), undefined); +}); + +test("log readers skip diagnostics, report missing logs, and mark zero cost unknown", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "eve-bench-usage-")); + t.after(() => rm(dir, { recursive: true, force: true })); + + assert.equal(await readCliUsage("pi", dir), undefined); + + // A 401 still emits an assistant message_end, so usage exists but is zero. + await writeFile( + join(dir, "pi.jsonl"), + ["native warning: not json", JSON.stringify(piAssistant(0, 0, 0)), "{truncated", ""].join("\n"), + ); + assert.deepEqual(await readCliUsage("pi", dir), { + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + costUsd: null, + }); + + await writeFile( + join(dir, "events.ndjson"), + `${JSON.stringify({ type: "session.started" })}\n${JSON.stringify({ + type: "step.completed", + data: { usage: { inputTokens: 7, outputTokens: 3, costUsd: 0.25 } }, + })}\n${JSON.stringify({ type: "step.completed", data: { usage: { inputTokens: 5, outputTokens: 1 } } })}\n`, + ); + assert.deepEqual(await readEveUsage(dir), { + inputTokens: 12, + outputTokens: 4, + cachedTokens: 0, + costUsd: 0.25, + }); + + await writeFile(join(dir, "empty.jsonl"), `${JSON.stringify({ type: "session.started" })}\n`); + assert.equal(await readJsonlUsage(join(dir, "empty.jsonl"), eveStepUsage), undefined); +}); diff --git a/packages/eve-bench/src/core/usage.ts b/packages/eve-bench/src/core/usage.ts new file mode 100644 index 0000000000..90dbb58a5d --- /dev/null +++ b/packages/eve-bench/src/core/usage.ts @@ -0,0 +1,63 @@ +import { createReadStream } from "node:fs"; +import { createInterface } from "node:readline"; + +import type { Usage } from "./result.ts"; + +export interface UsageRecord { + readonly input: number; + readonly output: number; + readonly cached: number; + readonly cost?: number; +} + +/** + * Sums usage records from a JSON Lines log, skipping non-JSON diagnostics. + * Returns undefined when the log is missing or holds no usage records. A zero + * cost total is reported as unknown because harnesses without pricing data + * report zero. + */ +export async function readJsonlUsage( + path: string, + extract: (event: unknown) => UsageRecord | undefined, +): Promise { + let records = 0; + let inputTokens = 0; + let outputTokens = 0; + let cachedTokens = 0; + let cost = 0; + try { + const lines = createInterface({ input: createReadStream(path), crlfDelay: Infinity }); + for await (const line of lines) { + if (!line.startsWith("{")) continue; + let event: unknown; + try { + event = JSON.parse(line); + } catch { + continue; + } + const record = extract(event); + if (!record) continue; + records++; + inputTokens += record.input; + outputTokens += record.output; + cachedTokens += record.cached; + cost += record.cost ?? 0; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + if (records === 0) return undefined; + return { inputTokens, outputTokens, cachedTokens, costUsd: cost > 0 ? cost : null }; +} + +export function count(...values: unknown[]): number { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return 0; +} + +export function record(value: unknown): Record { + return value !== null && typeof value === "object" ? (value as Record) : {}; +} diff --git a/packages/eve-bench/src/harnesses/cli/index.ts b/packages/eve-bench/src/harnesses/cli/index.ts new file mode 100644 index 0000000000..03a74b7d9d --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/index.ts @@ -0,0 +1,50 @@ +import type { Harness } from "../../core/harness.ts"; +import { prepareCliBundle } from "./prepare.ts"; +import { readCliUsage } from "./usage.ts"; +import { + GATEWAY_BASE_URL, + shellQuote, + validateModel, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; + +export function createCliHarness( + name: "hermes" | "pi" | "opencode" | "codex", + options: { version: string; baseUrl?: string; reasoning?: string }, +): Harness { + const supported = validateOptions(name, options); + const pinned = { ...options }; + return { + name: `${name}@${pinned.version}`, + credentials: ["AI_GATEWAY_API_KEY"], + prepare: async (ctx) => { + validateModel(ctx.model); + const bundle = await prepareCliBundle(supported, pinned, ctx); + return { + ...bundle, + provenance: { + ...bundle.provenance, + model: ctx.model, + baseUrl: pinned.baseUrl ?? GATEWAY_BASE_URL, + reasoning: pinned.reasoning ?? null, + protocol: supported === "codex" ? "responses" : "chat-completions", + }, + }; + }, + command: (ctx) => + `exec ${shellQuote(`${ctx.installDir}/arch/node`)} ${shellQuote(`${ctx.installDir}/runner.ts`)}`, + readUsage: (agentLogsDir) => readCliUsage(supported, agentLogsDir), + env: (ctx) => { + validateModel(ctx.model); + return { + EVE_BENCH_CLI_NAME: name, + EVE_BENCH_CLI_OPTIONS: JSON.stringify(pinned), + EVE_BENCH_CLI_CONTEXT: JSON.stringify(ctx), + }; + }, + }; +} + +export type { CliName, CliOptions }; diff --git a/packages/eve-bench/src/harnesses/cli/output.ts b/packages/eve-bench/src/harnesses/cli/output.ts new file mode 100644 index 0000000000..7c5e8eb7f7 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/output.ts @@ -0,0 +1,53 @@ +import { StringDecoder } from "node:string_decoder"; + +/** Retain a suffix so even a credential split across pipe chunks is redacted. */ +export class SecretRedactor { + private readonly decoder = new StringDecoder("utf8"); + private pending = ""; + private readonly secrets: string[]; + private readonly keep: number; + + constructor(secret: string) { + this.secrets = [ + ...new Set([secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]), + ] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + this.keep = Math.max(1, ...this.secrets.map((value) => value.length)); + } + + push(chunk: Buffer): string { + this.pending += this.decoder.write(chunk); + return this.drain(false); + } + + finish(): string { + this.pending += this.decoder.end(); + return this.drain(true); + } + + private drain(final: boolean): string { + let output = ""; + while (this.pending.length > 0) { + const safe = final ? this.pending.length : Math.max(0, this.pending.length - this.keep + 1); + if (!safe) break; + let start = this.pending.length; + let matched = ""; + for (const secret of this.secrets) { + const index = this.pending.indexOf(secret); + if (index >= 0 && index < start) { + start = index; + matched = secret; + } + } + if (matched && start < safe) { + output += this.pending.slice(0, start) + "[REDACTED]"; + this.pending = this.pending.slice(start + matched.length); + } else { + output += this.pending.slice(0, safe); + this.pending = this.pending.slice(safe); + } + } + return output; + } +} diff --git a/packages/eve-bench/src/harnesses/cli/plan.test.ts b/packages/eve-bench/src/harnesses/cli/plan.test.ts new file mode 100644 index 0000000000..7be2b4233c --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createCliHarness } from "./index.ts"; +import { + executionPlan, + packageName, + validateModel, + validateOptions, + type SupportedCliName, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const ctx = { + model: "deepseek/deepseek-v4", + installDir: "/installed-agent", + instructionPath: "/instruction.md", + taskWorkdir: "/workspace", + logsDir: "/logs/agent", +}; + +for (const name of ["pi", "opencode", "codex"] as const) { + test(`${name} requires a pin and rejects credential-bearing endpoints`, () => { + for (const version of [ + "", + "latest", + "next", + "^1.2.3", + "1", + "1.2", + "*", + "1.2.3; echo unsafe", + "v1.2.3", + "01.2.3", + ]) { + assert.throws(() => createCliHarness(name, { version }), /explicit exact version/); + } + assert.equal(createCliHarness(name, { version: "1.2.3-rc.1" }).name, `${name}@1.2.3-rc.1`); + for (const baseUrl of [ + "https://user:key@example.com/v1", + "https://example.com/v1?key=secret", + "https://example.com/#secret", + "http://example.com", + "not a URL", + ]) { + assert.throws( + () => createCliHarness(name, { version: "1.2.3", baseUrl }), + /without credentials/, + ); + } + }); + + test(`${name} retains full provider-qualified model IDs`, () => { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6", "openai/acme/served-model"]) { + const plan = executionPlan( + name, + { version: "1.2.3", reasoning: "high" }, + { ...ctx, model }, + "/runtime", + "--a 'quote'\n$(touch /never)", + ); + assert.ok( + plan.args.includes(model) || plan.args.includes(`--model=eve-bench-gateway/${model}`), + ); + if (name === "pi") { + assert.equal(plan.stdin, "--a 'quote'\n$(touch /never)"); + assert.ok(!plan.args.includes("--")); + } else { + assert.equal(plan.args.at(-1), "--a 'quote'\n$(touch /never)"); + assert.equal(plan.args.at(-2), "--"); + } + assert.match(plan.config, /https:\/\/ai-gateway.vercel.sh\/v1/); + assert.ok( + !JSON.stringify( + createCliHarness(name, { version: "1.2.3" }).env({ ...ctx, model }), + ).includes("API_KEY"), + ); + } + }); +} + +test("pi custom models use runtime environment references and explicit thinking capability", () => { + const plan = executionPlan( + "pi", + { version: "0.74.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "chat-completions"); + assert.deepEqual(JSON.parse(plan.config).providers["eve-bench-gateway"], { + baseUrl: "https://ai-gateway.vercel.sh/v1", + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [{ id: ctx.model, reasoning: true }], + }); + assert.deepEqual(plan.args.slice(0, 4), ["--print", "--mode", "json", "--session-dir"]); + assert.ok(plan.args.includes("--thinking")); +}); + +test("OpenCode uses a custom compatible provider, not OpenAI's Responses loader", () => { + const plan = executionPlan( + "opencode", + { version: "1.2.3", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + const provider = JSON.parse(plan.config).provider["eve-bench-gateway"]; + assert.equal(provider.npm, "@ai-sdk/openai-compatible"); + assert.deepEqual(provider.env, ["OPENAI_API_KEY"]); + assert.equal(provider.options.baseURL, "https://ai-gateway.vercel.sh/v1"); + assert.deepEqual(provider.models[ctx.model].variants, { high: { reasoningEffort: "high" } }); + assert.ok(plan.args.includes("--dangerously-skip-permissions")); + assert.ok(plan.args.includes("--variant")); +}); + +test("Codex has env-key auth, a Responses wire API, and no auth.json", () => { + const plan = executionPlan( + "codex", + { version: "0.118.0", reasoning: "high" }, + ctx, + "/runtime", + "do it", + ); + assert.equal(plan.protocol, "responses"); + assert.match(plan.config, /env_key = "AI_GATEWAY_API_KEY"/); + assert.match(plan.config, /requires_openai_auth = false/); + assert.match(plan.config, /wire_api = "responses"/); + assert.doesNotMatch(plan.config, /auth.json/); + assert.ok(plan.args.includes("--dangerously-bypass-approvals-and-sandbox")); + assert.ok(plan.args.includes("model_reasoning_effort=high")); +}); + +test("no reasoning option means no imposed reasoning setting", () => { + for (const name of ["pi", "opencode", "codex"] as const) { + const plan = executionPlan(name, { version: "1.2.3" }, ctx, "/runtime", "instruction"); + assert.doesNotMatch(plan.config, /reasoningEffort|model_reasoning_effort|"reasoning"/); + assert.ok(!plan.args.includes("--variant")); + } + assert.throws(() => validateOptions("pi", { version: "1.2.3", reasoning: "max" }), /reasoning/); + assert.throws( + () => validateOptions("codex", { version: "1.2.3", reasoning: "off" }), + /reasoning/, + ); +}); + +test("package names follow Harbor's pi rename boundary", () => { + assert.equal(packageName("pi", "0.73.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0-rc.1"), "@mariozechner/pi-coding-agent"); + assert.equal(packageName("pi", "0.74.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("pi", "1.0.0"), "@earendil-works/pi-coding-agent"); + assert.equal(packageName("codex", "0.118.0"), "@openai/codex"); + assert.equal(packageName("opencode", "1.2.3"), "opencode-ai"); +}); + +test("Hermes fails explicitly before installation, without a substitute agent", () => { + assert.throws( + () => createCliHarness("hermes", { version: "v2026.7.20" }), + /Hermes is not supported.*pinned, audited Python\/uv bundle/, + ); +}); + +test("model validation rejects unqualified IDs and command/control injection", () => { + for (const model of [ + "gpt-5.6", + "openai/", + "openai/gpt\nmodel", + "openai/$(cmd)", + "openai/gpt model", + ]) { + assert.throws(() => validateModel(model), /full provider-qualified/); + } +}); + +test("constructor snapshots options instead of accepting later mutation", () => { + const options = { version: "1.2.3" }; + const harness = createCliHarness("pi", options); + options.version = "latest"; + assert.match(harness.env(ctx).EVE_BENCH_CLI_OPTIONS!, /1.2.3/); +}); + +test("redaction covers every byte boundary including escaped and Unicode secrets", () => { + for (const secret of ["fake-key-abcdef", 'fake-"key', "fake-秘密"]) { + for (const form of [secret, JSON.stringify(secret).slice(1, -1), encodeURIComponent(secret)]) { + const input = Buffer.from(`prefix ${form} suffix ${form}!`); + for (let split = 0; split <= input.length; split++) { + const redactor = new SecretRedactor(secret); + const output = + redactor.push(input.subarray(0, split)) + + redactor.push(input.subarray(split)) + + redactor.finish(); + assert.equal(output, "prefix [REDACTED] suffix [REDACTED]!"); + } + } + } +}); + +// Keep the supported surface explicit in the pure tests, independently of parent wiring. +const supported: SupportedCliName[] = ["pi", "opencode", "codex"]; +test("all supported adapters implement the shared Harness contract", () => { + for (const name of supported) { + const harness = createCliHarness(name, { version: "1.2.3" }); + assert.equal(typeof harness.prepare, "function"); + assert.match(harness.command({ ...ctx, installDir: "/has space/and'quote" }), /^exec '/); + } +}); diff --git a/packages/eve-bench/src/harnesses/cli/plan.ts b/packages/eve-bench/src/harnesses/cli/plan.ts new file mode 100644 index 0000000000..d87ce508b0 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/plan.ts @@ -0,0 +1,205 @@ +import type { HarnessRunContext } from "../../core/harness.ts"; + +export type CliName = "hermes" | "pi" | "opencode" | "codex"; +export type SupportedCliName = Exclude; +export interface CliOptions { + readonly version: string; + readonly baseUrl?: string; + readonly reasoning?: string; +} + +export const GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"; +const EXACT_VERSION = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/; + +export function validateOptions(name: CliName, options: CliOptions): SupportedCliName { + if (name === "hermes") { + throw new Error( + "Hermes is not supported by this adapter: Harbor installs a mutable main/scripts/install.sh and only establishes OPENAI_BASE_URL routing for openai/*, not Gateway DeepSeek. A pinned, audited Python/uv bundle and verified custom-provider routing are required; no fallback agent is run.", + ); + } + if (!["pi", "opencode", "codex"].includes(name)) throw new Error("Unsupported CLI harness"); + if (!EXACT_VERSION.test(options.version ?? "")) { + throw new Error( + "CLI harness requires an explicit exact version (for example 1.2.3); tags, ranges, and latest are not allowed.", + ); + } + let url: URL; + try { + url = new URL(options.baseUrl ?? GATEWAY_BASE_URL); + } catch { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) { + throw new Error( + "CLI baseUrl must be an HTTPS API base URL without credentials, query, or fragment.", + ); + } + if (options.reasoning !== undefined) { + const allowed = + name === "pi" + ? ["off", "minimal", "low", "medium", "high", "xhigh"] + : name === "codex" + ? ["none", "minimal", "low", "medium", "high", "xhigh"] + : undefined; + if ( + allowed + ? !allowed.includes(options.reasoning) + : !/^[a-zA-Z0-9_-]{1,64}$/.test(options.reasoning) + ) { + throw new Error(`Unsupported ${name} reasoning value`); + } + } + return name; +} + +export function validateModel(model: string): void { + if ( + model.length > 256 || + !/^[a-zA-Z0-9][a-zA-Z0-9._-]*\/[a-zA-Z0-9][a-zA-Z0-9/._:-]*$/.test(model) + ) { + throw new Error( + "CLI model must be a full provider-qualified model ID, without whitespace or control characters.", + ); + } +} + +export function packageName(name: SupportedCliName, version: string): string { + if (name === "codex") return "@openai/codex"; + if (name === "opencode") return "opencode-ai"; + const [major, minor] = version.split(".").map(Number); + // Harbor pi.py changes package namespace at 0.74.0 (prereleases precede it). + return major === 0 && (minor! < 74 || (minor === 74 && version.startsWith("0.74.0-"))) + ? "@mariozechner/pi-coding-agent" + : "@earendil-works/pi-coding-agent"; +} + +export interface ExecutionPlan { + readonly args: string[]; + readonly stdin?: string; + readonly env: Record; + readonly configPath: string; + readonly config: string; + readonly protocol: "chat-completions" | "responses"; +} + +/** Native commands/config follow Harbor installed/{pi,opencode,codex}.py at + * 2fe1615503fed39ad82b7ce09b22497996b30f1f. Gateway model IDs deliberately stay intact. + */ +export function executionPlan( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessRunContext, + runtimeDir: string, + instruction: string, +): ExecutionPlan { + validateOptions(name, options); + validateModel(ctx.model); + const baseUrl = options.baseUrl ?? GATEWAY_BASE_URL; + const json = (value: unknown) => `${JSON.stringify(value, null, 2)}\n`; + if (name === "pi") { + const model: { id: string; reasoning?: boolean } = { id: ctx.model }; + if (options.reasoning) model.reasoning = options.reasoning !== "off"; + return { + protocol: "chat-completions", + args: [ + "--print", + "--mode", + "json", + "--session-dir", + `${runtimeDir}/pi/sessions`, + "--provider", + "eve-bench-gateway", + "--model", + ctx.model, + ...(options.reasoning ? ["--thinking", options.reasoning] : []), + ], + // pi 0.80.3's parseArgs treats -- as an unknown flag. main.readPipedStdin + // passes stdin to prepareInitialMessage without flag or @file interpretation. + stdin: instruction, + env: { PI_CODING_AGENT_DIR: `${runtimeDir}/pi` }, + configPath: `${runtimeDir}/pi/models.json`, + config: json({ + providers: { + "eve-bench-gateway": { + baseUrl, + apiKey: "$AI_GATEWAY_API_KEY", + api: "openai-completions", + models: [model], + }, + }, + }), + }; + } + if (name === "opencode") { + return { + protocol: "chat-completions", + // Only the first component selects the CLI provider; the API ID stays intact. + args: [ + `--model=eve-bench-gateway/${ctx.model}`, + "run", + "--format=json", + ...(options.reasoning ? ["--variant", options.reasoning] : []), + "--thinking", + "--dangerously-skip-permissions", + "--", + instruction, + ], + env: { + OPENCODE_FAKE_VCS: "git", + OPENCODE_CONFIG: `${runtimeDir}/opencode/opencode.json`, + }, + configPath: `${runtimeDir}/opencode/opencode.json`, + // Harbor's mimo.py (OpenCode derivative) documents the compatible SDK override. + // OpenCode provider.ts resolves single-entry provider.env into the SDK API key. + config: json({ + provider: { + "eve-bench-gateway": { + npm: "@ai-sdk/openai-compatible", + env: ["OPENAI_API_KEY"], + options: { baseURL: baseUrl }, + models: { + [ctx.model]: options.reasoning + ? { + reasoning: true, + variants: { [options.reasoning]: { reasoningEffort: options.reasoning } }, + } + : {}, + }, + }, + }, + }), + }; + } + return { + protocol: "responses", + args: [ + "exec", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--model", + ctx.model, + "--json", + "--enable", + "unified_exec", + ...(options.reasoning ? ["-c", `model_reasoning_effort=${options.reasoning}`] : []), + "--", + instruction, + ], + env: { CODEX_HOME: `${runtimeDir}/codex`, OPENAI_BASE_URL: baseUrl }, + configPath: `${runtimeDir}/codex/config.toml`, + // Unlike Harbor's auth.json, Codex's native env_key needs no persisted credential. + // Source: openai/codex, codex-rs/model-provider-info/src/lib.rs, ModelProviderInfo. + config: + `model_provider = "eve-bench-gateway"\nopenai_base_url = ${JSON.stringify(baseUrl)}\n` + + `[model_providers.eve-bench-gateway]\nname = "eve-bench-gateway"\n` + + `base_url = ${JSON.stringify(baseUrl)}\nwire_api = "responses"\n` + + `env_key = "AI_GATEWAY_API_KEY"\nrequires_openai_auth = false\n`, + }; +} + +export function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} diff --git a/packages/eve-bench/src/harnesses/cli/prepare.ts b/packages/eve-bench/src/harnesses/cli/prepare.ts new file mode 100644 index 0000000000..9fd28ff377 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/prepare.ts @@ -0,0 +1,247 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; +import { packageName, type CliOptions, type SupportedCliName } from "./plan.ts"; + +const exec = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const NODE_VERSION = "24.19.0"; +const ARCHES = ["x64", "arm64"] as const; +const SOURCES = ["plan.ts", "runner.ts", "output.ts", "prepare.ts", "index.ts"]; + +/** Installation is host-only, serial, and credential-free. Both Linux/glibc + * architectures are prepared because the Harness contract selects arch only after upload. + * Lifecycle scripts are disabled; native CLI launchers use their optional platform packages. + */ +export async function prepareCliBundle( + name: SupportedCliName, + options: CliOptions, + ctx: HarnessPrepareContext, +): Promise { + const cache = resolve(ctx.cacheDir, "cli"); + await mkdir(cache, { recursive: true }); + const scratch = await mkdtemp(join(cache, ".prepare-")); + const cleanEnv = { PATH: process.env.PATH, HOME: join(scratch, "home"), CI: "true" }; + await mkdir(cleanEnv.HOME, { recursive: true }); + try { + let pnpmVersion: string; + try { + pnpmVersion = ( + await exec("pnpm", ["--version"], { env: cleanEnv, cwd: scratch }) + ).stdout.trim(); + } catch { + throw new Error( + "CLI bundles require pnpm 11+ on the host; pnpm could not run. No container installation was attempted.", + ); + } + if (!/^(?:1[1-9]|[2-9]\d)\./.test(pnpmVersion)) { + throw new Error( + "CLI bundles require pnpm 11+ for explicit Linux CPU/libc installs. No container installation was attempted.", + ); + } + const sources = await Promise.all(SOURCES.map((file) => readFile(join(here, file)))); + const sourceSha256 = sha256(Buffer.concat(sources)); + const pkg = packageName(name, options.version); + const key = sha256( + JSON.stringify({ pkg, version: options.version, pnpmVersion, NODE_VERSION, sourceSha256 }), + ); + const dir = join(cache, key); + if (await exists(join(dir, "provenance.json"))) return await bundleAt(dir); + + const started = Date.now(); + const building = join(scratch, "bundle"); + const common = join(building, "common"); + await mkdir(common, { recursive: true }); + for (const source of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(here, source), join(common, source)); + const lockSha256: Record = {}; + const nodeSha256: Record = {}; + for (const arch of ARCHES) { + const archDir = join(building, arch); + const app = join(archDir, "app"); + await mkdir(app, { recursive: true }); + await writeFile( + join(app, "package.json"), + JSON.stringify({ + name: "eve-bench-cli-bundle", + private: true, + type: "module", + dependencies: { [pkg]: options.version }, + }), + ); + await writeFile(join(app, ".npmrc"), "registry=https://registry.npmjs.org/\n"); + try { + await exec( + "pnpm", + [ + "install", + "--ignore-workspace", + "--ignore-scripts", + "--ignore-pnpmfile", + "--no-frozen-lockfile", + "--prod", + "--os=linux", + `--cpu=${arch}`, + "--libc=glibc", + "--package-import-method=copy", + "--network-concurrency=2", + "--child-concurrency=1", + "--store-dir", + join(cache, "store"), + "--reporter=silent", + ], + { cwd: app, env: cleanEnv, maxBuffer: 1024 * 1024 }, + ); + } catch { + // Do not embed package-manager stdout/errors: registry diagnostics may contain credentials. + throw new Error( + `Host pnpm install failed for ${pkg}@${options.version} (linux/${arch}, scripts disabled). No CLI setup was moved into the task timeout.`, + ); + } + const manifest = JSON.parse( + await readFile(join(app, "node_modules", pkg, "package.json"), "utf8"), + ) as { + version: string; + bin?: string | Record; + }; + if (manifest.version !== options.version) + throw new Error("Installed CLI version does not match the explicit pin"); + const bin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.[name]; + if (!bin || isAbsolute(bin)) + throw new Error("Pinned CLI package has no supported relative executable entry"); + const packageRoot = join(app, "node_modules", pkg); + let entry = resolve(packageRoot, bin); + let native = false; + if (name === "opencode") { + const platform = arch === "x64" ? "opencode-linux-x64-baseline" : "opencode-linux-arm64"; + entry = join( + app, + "node_modules/.pnpm", + `${platform}@${options.version}`, + "node_modules", + platform, + "bin/opencode", + ); + native = true; + } + if (!(await exists(entry))) { + throw new Error( + "Pinned CLI package executable is missing for the requested Linux platform", + ); + } + if (!native && relative(packageRoot, entry).startsWith("..")) { + throw new Error("Pinned CLI package executable escapes its package"); + } + await writeFile( + join(archDir, "entry.json"), + JSON.stringify({ entry: relative(archDir, entry), native }), + ); + lockSha256[arch] = sha256(await readFile(join(app, "pnpm-lock.yaml"))); + nodeSha256[arch] = await installNode(archDir, arch, scratch); + } + const provenance = { + key, + name, + version: options.version, + package: pkg, + sourceSha256, + installLocation: "host", + containerInstallMs: 0, + hostPrepareMs: Date.now() - started, + pnpmVersion, + nodeVersion: NODE_VERSION, + platforms: ["linux-x64-glibc", "linux-arm64-glibc"], + lifecycleScripts: false, + lockSha256, + nodeSha256, + adapterSources: { + harbor: + "https://github.com/harbor-framework/harbor/tree/2fe1615503fed39ad82b7ce09b22497996b30f1f/src/harbor/agents/installed", + codexEnvAuth: + "https://github.com/openai/codex/blob/1aaa453ce2868e14c10089c226ebec265657cb27/codex-rs/model-provider-info/src/lib.rs", + opencodeProviders: + "https://github.com/anomalyco/opencode/blob/ac1758c0e6b8e7368be133e87e8e27a5f60dec04/packages/opencode/src/provider/provider.ts", + }, + limitations: [ + "Native CLI/platform compatibility is not inferred from mock tests.", + "Host pnpm 11+ and Linux glibc containers are required; no musl/system-package bootstrap.", + "CLI-owned startup work is included in invocation wall time, not independently measured.", + "Custom model context/output limits use CLI defaults; reasoning settings are not normalized between agents.", + "Codex requires a Responses-compatible endpoint/model; no chat-completions fallback.", + "Native state stays ephemeral; only redacted stdout/stderr is exported. Usage is not normalized.", + ], + }; + await writeFile(join(building, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + await cp(join(building, "provenance.json"), join(common, "provenance.json")); + try { + await rename(building, dir); + } catch (error) { + if (!(await exists(join(dir, "provenance.json")))) throw error; + } + return await bundleAt(dir); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function bundleAt(dir: string): Promise { + // Keep platform trees out of the common upload (which otherwise doubles every transfer). + return { + dir: join(dir, "common"), + arch: { x64: join(dir, "x64"), arm64: join(dir, "arm64") }, + provenance: JSON.parse(await readFile(join(dir, "provenance.json"), "utf8")) as Record< + string, + unknown + >, + }; +} + +async function installNode(archDir: string, arch: string, scratch: string): Promise { + const name = `node-v${NODE_VERSION}-linux-${arch}`; + const base = `https://nodejs.org/dist/v${NODE_VERSION}`; + const sums = (await download(`${base}/SHASUMS256.txt`, 1024 * 1024)).toString("utf8"); + const expected = sums + .split("\n") + .find((line) => line.trim().endsWith(` ${name}.tar.xz`)) + ?.split(/\s+/)[0]; + if (!expected || !/^[a-f0-9]{64}$/.test(expected)) + throw new Error("Pinned Node checksum is missing"); + const archive = await download(`${base}/${name}.tar.xz`, 128 * 1024 * 1024); + if (sha256(archive) !== expected) throw new Error("Pinned Node archive checksum mismatch"); + const path = join(scratch, `${name}.tar.xz`); + await writeFile(path, archive); + await exec("tar", ["-xJf", path, "--strip-components=2", "-C", archDir, `${name}/bin/node`]); + await chmod(join(archDir, "node"), 0o755); + await rm(path); + return expected; +} + +async function download(url: string, limit: number): Promise { + const response = await fetch(url, { signal: AbortSignal.timeout(120_000), redirect: "error" }); + if (!response.ok || !response.body) + throw new Error(`Pinned Node download failed (HTTP ${response.status})`); + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > limit) throw new Error("Pinned Node download exceeds size limit"); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/eve-bench/src/harnesses/cli/runner.ts b/packages/eve-bench/src/harnesses/cli/runner.ts new file mode 100644 index 0000000000..d22645f000 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/runner.ts @@ -0,0 +1,237 @@ +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { finished } from "node:stream/promises"; + +import type { HarnessRunContext } from "../../core/harness.ts"; +import { + GATEWAY_BASE_URL, + executionPlan, + validateOptions, + type CliName, + type CliOptions, +} from "./plan.ts"; +import { SecretRedactor } from "./output.ts"; + +const OUTPUT_LIMIT = 64 * 1024 * 1024; +const INSTRUCTION_LIMIT = 1024 * 1024; + +async function main(): Promise { + const ctx = JSON.parse(process.env.EVE_BENCH_CLI_CONTEXT!) as HarnessRunContext; + const options = JSON.parse(process.env.EVE_BENCH_CLI_OPTIONS!) as CliOptions; + const name = validateOptions(process.env.EVE_BENCH_CLI_NAME as CliName, options); + const start = Date.now(); + let agentStart: number | undefined; + let exitCode = 1; + let failure: string | undefined; + let signal: string | null = null; + await mkdir(ctx.logsDir, { recursive: true }); + const runtimeDir = await mkdtemp(join(ctx.installDir, ".cli-runtime-")); + const secret = process.env.AI_GATEWAY_API_KEY; + try { + if (!secret?.trim()) + throw new Error("AI_GATEWAY_API_KEY must be forwarded to the CLI runtime environment."); + if (secret.length > 8192 || /[\r\n\0]/.test(secret)) + throw new Error("AI_GATEWAY_API_KEY has an invalid format."); + if ((await stat(ctx.instructionPath)).size > INSTRUCTION_LIMIT) + throw new Error("CLI instruction exceeds 1 MiB."); + const instruction = await readFile(ctx.instructionPath, "utf8"); + const plan = executionPlan(name, options, ctx, runtimeDir, instruction); + await mkdir(dirname(plan.configPath), { recursive: true }); + await writeFile(plan.configPath, plan.config, { mode: 0o600 }); + const { entry, native = false } = JSON.parse( + await readFile(join(ctx.installDir, "arch/entry.json"), "utf8"), + ) as { entry: string; native?: boolean }; + const executable = join(ctx.installDir, "arch", entry); + // The CLI and its tool subprocesses keep the task container's own environment + // (HOME, PATH, image ENV), as eve's tools do, so state they leave in the task + // survives into verification. Only harness-owned config paths are added. + const { + EVE_BENCH_CLI_CONTEXT: _context, + EVE_BENCH_CLI_OPTIONS: _options, + EVE_BENCH_CLI_NAME: _name, + AI_GATEWAY_API_KEY: _gatewayKey, + OPENAI_API_KEY: _openaiKey, + ...inherited + } = process.env; + const baseEnv = { ...(inherited as Record), ...plan.env }; + const version = await run( + executable, + ["--version"], + baseEnv, + ctx, + "cli-setup.log", + secret, + false, + undefined, + native, + ); + const escapedVersion = options.version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if ( + version.code !== 0 || + !new RegExp(`(?:^|\\s)${escapedVersion}(?:\\s|$)`).test(version.text) + ) { + throw new Error( + "Pinned CLI failed its --version check; native package/runtime compatibility must be checked before benchmarking.", + ); + } + agentStart = Date.now(); + const result = await run( + executable, + plan.args, + { ...baseEnv, AI_GATEWAY_API_KEY: secret, OPENAI_API_KEY: secret }, + ctx, + `${name}.jsonl`, + secret, + name !== "codex", + plan.stdin, + native, + ); + exitCode = result.code; + signal = result.signal; + if (result.errorEvent && exitCode === 0) { + exitCode = 1; + failure = `${name} emitted an error event despite exiting zero.`; + } + } catch (error) { + // Only our actionable messages are emitted; filesystem/spawn diagnostics can contain arbitrary data. + const message = error instanceof Error ? error.message : ""; + failure = + /^(AI_GATEWAY_API_KEY |CLI instruction |Pinned CLI |CLI model |Unsupported .* reasoning)/.test( + message, + ) + ? message + : "CLI setup or execution failed; inspect the redacted CLI logs and the pinned bundle."; + } finally { + const end = Date.now(); + await rm(runtimeDir, { recursive: true, force: true }); + const runtime: Record = { + name, + version: options.version, + model: ctx.model, + baseUrl: options.baseUrl ?? GATEWAY_BASE_URL, + reasoning: options.reasoning ?? null, + protocol: name === "codex" ? "responses" : "chat-completions", + installLocation: "host", + containerInstallMs: 0, + setupMs: (agentStart ?? end) - start, + agentMs: agentStart === undefined ? 0 : end - agentStart, + exitCode, + signal, + }; + if (failure) runtime.failure = failure; + await writeFile(join(ctx.logsDir, "cli-runtime.json"), `${JSON.stringify(runtime, null, 2)}\n`); + } + if (failure) process.stderr.write(`[eve-bench] ${failure}\n`); + return exitCode; +} + +async function run( + entry: string, + args: string[], + env: Record, + ctx: HarnessRunContext, + logName: string, + secret: string, + detectErrors = false, + input?: string, + native = false, +): Promise<{ code: number; signal: string | null; text: string; errorEvent: boolean }> { + const log = createWriteStream(join(ctx.logsDir, logName), { mode: 0o600 }); + let bytes = 0; + let text = ""; + let line = ""; + let skipLine = false; + let errorEvent = false; + let truncated = false; + const emit = (chunk: string, stdout: boolean) => { + if (stdout && detectErrors) { + for (const piece of chunk.split(/(?<=\n)/)) { + if (!skipLine) line += piece; + if (line.length > INSTRUCTION_LIMIT) { + line = ""; + skipLine = true; + } + if (piece.endsWith("\n")) { + if (!skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Native diagnostics need not be JSON. */ + } + } + line = ""; + skipLine = false; + } + } + } + if (text.length < 65536) text += chunk.slice(0, 65536 - text.length); + const buffer = Buffer.from(chunk); + const kept = buffer.subarray(0, Math.max(0, OUTPUT_LIMIT - bytes)); + bytes += kept.length; + if (kept.length > 0) { + log.write(kept); + process.stdout.write(kept); + } + if (!truncated && kept.length < buffer.length) { + truncated = true; + log.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + process.stdout.write("\n[eve-bench] CLI output truncated at 64 MiB\n"); + } + }; + const child = spawn(native ? entry : process.execPath, native ? args : [entry, ...args], { + cwd: ctx.taskWorkdir, + env, + stdio: ["pipe", "pipe", "pipe"], + }); + // A CLI may exit before consuming stdin; its exit status remains authoritative. + child.stdin.on("error", () => {}); + child.stdin.end(input); + const forwardTerm = () => child.kill("SIGTERM"); + const forwardInt = () => child.kill("SIGINT"); + process.on("SIGTERM", forwardTerm); + process.on("SIGINT", forwardInt); + const stdout = new SecretRedactor(secret); + const stderr = new SecretRedactor(secret); + child.stdout.on("data", (chunk: Buffer) => emit(stdout.push(chunk), true)); + child.stderr.on("data", (chunk: Buffer) => emit(stderr.push(chunk), false)); + let spawnFailed = false; + child.on("error", () => { + spawnFailed = true; + }); + const result = await new Promise<{ code: number; signal: string | null }>((resolve) => { + child.on("close", (code, signal) => resolve({ code: spawnFailed ? 1 : (code ?? 1), signal })); + }); + process.off("SIGTERM", forwardTerm); + process.off("SIGINT", forwardInt); + emit(stdout.finish(), true); + emit(stderr.finish(), false); + if (detectErrors && line && !skipLine) { + try { + if (isErrorEvent(JSON.parse(line))) errorEvent = true; + } catch { + /* Partial diagnostic. */ + } + } + log.end(); + await finished(log); + return { ...result, text, errorEvent }; +} + +function isErrorEvent(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const event = value as { type?: string; errorMessage?: unknown; message?: unknown }; + if (event.type === "error" || typeof event.errorMessage === "string") return true; + return event.message !== undefined && isErrorEvent(event.message); +} + +main().then( + (code) => { + process.exitCode = code; + }, + () => { + process.stderr.write("[eve-bench] Invalid CLI runtime context or unwritable logs directory.\n"); + process.exitCode = 1; + }, +); diff --git a/packages/eve-bench/src/harnesses/cli/usage.ts b/packages/eve-bench/src/harnesses/cli/usage.ts new file mode 100644 index 0000000000..3437d57822 --- /dev/null +++ b/packages/eve-bench/src/harnesses/cli/usage.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import type { Usage } from "../../core/result.ts"; +import { count, readJsonlUsage, record, type UsageRecord } from "../../core/usage.ts"; +import type { SupportedCliName } from "./plan.ts"; + +/** Reads usage from the native JSON event stream the runner saves as `.jsonl`. */ +export function readCliUsage( + name: SupportedCliName, + agentLogsDir: string, +): Promise { + return readJsonlUsage(join(agentLogsDir, `${name}.jsonl`), CLI_USAGE[name]); +} + +export const CLI_USAGE: Record UsageRecord | undefined> = { + // pi --mode json: one assistant `message_end` per model call; `input` excludes cache reads. + pi(event) { + const value = record(event); + const message = record(value.message); + if (value.type !== "message_end" || message.role !== "assistant") return undefined; + const usage = record(message.usage); + const cached = count(usage.cacheRead); + return { + input: count(usage.input) + cached, + output: count(usage.output), + cached, + cost: count(record(usage.cost).total), + }; + }, + // opencode run --format=json: one `step_finish` per model call; reasoning is reported apart from output. + opencode(event) { + const value = record(event); + if (value.type !== "step_finish") return undefined; + const part = record(value.part); + const tokens = record(part.tokens); + const cached = count(record(tokens.cache).read); + return { + input: count(tokens.input) + cached, + output: count(tokens.output) + count(tokens.reasoning), + cached, + cost: count(part.cost), + }; + }, + // codex exec --json: one `turn.completed` per turn; input_tokens already include cached input. + codex(event) { + const value = record(event); + if (value.type !== "turn.completed") return undefined; + const usage = record(value.usage); + return { + input: count(usage.input_tokens), + output: count(usage.output_tokens), + cached: count(usage.cached_input_tokens), + }; + }, +}; diff --git a/packages/eve-bench/src/harnesses/e0/container-backend.ts b/packages/eve-bench/src/harnesses/e0/container-backend.ts new file mode 100644 index 0000000000..3f15a9c6ec --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/container-backend.ts @@ -0,0 +1,394 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rm } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import type { Readable } from "node:stream"; +import type { SandboxBackend, SandboxProcess, SandboxSession } from "eve/sandbox"; + +const MAX_BYTES = 16 * 1024 * 1024; +const ENV_KEYS = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "TMP", + "TEMP", + "TZ", + "TERM", +]; + +function checkSize(bytes: number, operation: string): void { + if (bytes > MAX_BYTES) throw new Error(`${operation} exceeds 16 MiB`); +} + +/** + * Attach to the task container; the runner owns isolation, networking and deletion. + * Explicit roots permit local tests, not an isolated host sandbox. No symlinks are + * created. File paths and cwd map /workspace to root, but shell source is never + * rewritten: commands (including code__apply_patch's root) must use the actual + * task directory or relative paths, not literal /workspace paths in shell text. + * Built-in glob/grep embed paths in shell commands: pass the actual root as path + * instead of relying on their /workspace default. + */ +export function containerBackend( + root = process.env.EVE_BENCH_TASK_WORKDIR!, + { commandTimeoutMs = 120_000 }: { commandTimeoutMs?: number } = {}, +): SandboxBackend { + if (!root?.trim() || !isAbsolute(root)) + throw new Error( + "EVE_BENCH_TASK_WORKDIR must be an absolute task directory (or pass an explicit test root)", + ); + if (arguments[0] === undefined && process.env.EVE_BENCH_CONTAINER !== "1") + throw new Error("The e0 backend requires EVE_BENCH_CONTAINER=1 inside a task container"); + if ( + !Number.isFinite(commandTimeoutMs) || + commandTimeoutMs <= 0 || + commandTimeoutMs > 2_147_483_647 + ) + throw new Error("commandTimeoutMs must be positive and at most 2147483647"); + root = resolve(root); + const name = "e0-task-container"; + const children = new Set(); + const skillsHome = () => process.env.HOME && resolve(process.env.HOME, ".agents/skills"); + const within = (parent: string, path: string) => path === parent || path.startsWith(`${parent}/`); + const resolvePath = (path: string): string => { + let target: string; + if (path === "$HOME" || path.startsWith("$HOME/")) { + if (!process.env.HOME) throw new Error("HOME is unavailable in the task container"); + target = resolve(process.env.HOME, path.slice(6)); + } else if (path === "/workspace" || path.startsWith("/workspace/")) { + target = resolve(root, path === "/workspace" ? "." : path.slice(11)); + } else { + target = resolve(root, path); + } + const skills = skillsHome(); + if (!within(root, target) && !(skills && within(skills, target))) { + throw new Error("Sandbox paths must stay inside the task workspace or seeded skills"); + } + return target; + }; + const rejectSymlinkPath = async (target: string) => { + const base = within(root, target) ? root : skillsHome()!; + let current = base; + for (const component of relative(base, target).split(sep).filter(Boolean)) { + current = resolve(current, component); + try { + if ((await lstat(current)).isSymbolicLink()) { + throw new Error("Sandbox file operations do not follow symbolic links"); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + } + }; + const stop = async () => { + await Promise.all([...children].map((child) => child.kill())); + }; + const session: SandboxSession & { readonly description: string } = { + id: `${name}:${root}`, + description: `Task container. Work in ${JSON.stringify(root)}; use this actual root for code__apply_patch. File operations and cwd map /workspace here, but shell command text does not. Use relative paths or the actual task directory in bash. Always pass the actual task directory as glob/grep's path; their /workspace default is not remapped. Skill files are under $HOME/.agents/skills.`, + resolvePath, + async spawn({ command, workingDirectory, env = {}, abortSignal }) { + abortSignal?.throwIfAborted(); + const shellEnv: NodeJS.ProcessEnv = {}; + for (const key of ENV_KEYS) { + if (process.env[key] !== undefined) shellEnv[key] = process.env[key]; + } + // Neither inherit credentials nor let login profiles reload production env. + const child = spawn("/bin/bash", ["--noprofile", "--norc", "-c", command], { + cwd: resolvePath(workingDirectory ?? "."), + env: { ...shellEnv, ...env }, + detached: true, + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let killed = false; + let timedOut = false; + let failure: Error | undefined; + let bytes = 0; + const killGroup = () => { + if (settled || killed || child.pid === undefined) return; + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + killed = true; + }; + const abort = () => killGroup(); + abortSignal?.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(() => { + timedOut = true; + killGroup(); + }, commandTimeoutMs); + const stream = (source: Readable) => { + let closed = false; + return new ReadableStream({ + start(controller) { + source.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength; + if (bytes > MAX_BYTES && !failure) { + failure = new Error("Command output exceeds 16 MiB"); + killGroup(); + } + if (closed) return; + if (failure) { + closed = true; + controller.error(failure); + } else controller.enqueue(chunk); + }); + source.once("error", (error) => { + failure ??= error; + killGroup(); + if (!closed) { + closed = true; + controller.error(error); + } + }); + source.once("close", () => { + if (!closed) { + closed = true; + controller.close(); + } + }); + }, + cancel() { + closed = true; + killGroup(); + }, + }); + }; + const stdout = stream(child.stdout); + const stderr = stream(child.stderr); + // A shell can exit while a background descendant has redirected its pipes. + // Kill the group on exit as well as timeout, before dropping its PID. + child.once("exit", killGroup); + const completed = new Promise<{ exitCode: number }>((resolve, reject) => { + child.once("error", (error) => { + failure ??= error; + }); + child.once("close", (code) => { + settled = true; + clearTimeout(timer); + abortSignal?.removeEventListener("abort", abort); + children.delete(handle); + if (abortSignal?.aborted) reject(abortSignal.reason); + else if (failure) reject(failure); + else resolve({ exitCode: timedOut ? 124 : (code ?? 137) }); + }); + }); + // Stream consumers may wait only after draining output, or never wait at all. + void completed.catch(() => {}); + const handle: SandboxProcess = { + pid: child.pid, + stdout, + stderr, + wait: () => completed, + async kill() { + killGroup(); + await completed.catch(() => {}); + }, + }; + children.add(handle); + if (abortSignal?.aborted) killGroup(); + return handle; + }, + async run(input) { + const child = await session.spawn(input); + const collect = async (stream: ReadableStream) => { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); + }; + try { + const [stdout, stderr, result] = await Promise.all([ + collect(child.stdout), + collect(child.stderr), + child.wait(), + ]); + return { + ...result, + stdout, + stderr: + result.exitCode === 124 + ? `${stderr}\nCommand ended with exit 124 (timeout limit ${commandTimeoutMs}ms); its process group was stopped.\n` + : stderr, + }; + } finally { + await child.kill(); + } + }, + async readBinaryFile({ path, abortSignal }) { + abortSignal?.throwIfAborted(); + let file; + try { + // Nonblocking open + regular-file check avoid hangs on FIFOs/devices. + const target = resolvePath(path); + await rejectSymlinkPath(target); + file = await open(target, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + try { + const stat = await file.stat(); + if (!stat.isFile()) throw new Error("Only regular files can be read"); + checkSize(stat.size, "File read"); + const chunks: Uint8Array[] = []; + let bytes = 0; + for (;;) { + abortSignal?.throwIfAborted(); + const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_BYTES + 1 - bytes)); + const { bytesRead } = await file.read(buffer); + abortSignal?.throwIfAborted(); + if (bytesRead === 0) return Buffer.concat(chunks); + bytes += bytesRead; + checkSize(bytes, "File read"); + chunks.push(buffer.subarray(0, bytesRead)); + } + } finally { + await file.close(); + } + }, + async readFile(options) { + const bytes = await session.readBinaryFile(options); + return bytes === null + ? null + : new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + }, + async readTextFile({ encoding = "utf-8", startLine, endLine, ...options }) { + if ( + (startLine !== undefined && (!Number.isInteger(startLine) || startLine < 1)) || + (endLine !== undefined && (!Number.isInteger(endLine) || endLine < (startLine ?? 1))) + ) + throw new Error("Invalid 1-based line range"); + const bytes = await session.readBinaryFile(options); + if (bytes === null) return null; + const text = + encoding === "utf-8" || encoding === "utf8" + ? new TextDecoder("utf-8", { fatal: true }).decode(bytes) + : Buffer.from(bytes).toString(bufferEncoding(encoding)); + return startLine === undefined && endLine === undefined + ? text + : text + .split("\n") + .slice((startLine ?? 1) - 1, endLine) + .join("\n"); + }, + async writeBinaryFile({ path, content, abortSignal }) { + abortSignal?.throwIfAborted(); + checkSize(content.byteLength, "File write"); + const target = resolvePath(path); + await rejectSymlinkPath(target); + await mkdir(dirname(target), { recursive: true }); + await rejectSymlinkPath(target); + abortSignal?.throwIfAborted(); + const file = await open( + target, + constants.O_WRONLY | constants.O_CREAT | constants.O_NONBLOCK, + ); + try { + if (!(await file.stat()).isFile()) throw new Error("Only regular files can be written"); + abortSignal?.throwIfAborted(); + await file.truncate(0); + await file.writeFile(content, { signal: abortSignal }); + } finally { + await file.close(); + } + }, + async writeTextFile({ content, encoding = "utf8", ...options }) { + const codec = bufferEncoding(encoding); + checkSize(Buffer.byteLength(content, codec), "File write"); + await session.writeBinaryFile({ ...options, content: Buffer.from(content, codec) }); + }, + async writeFile({ content, ...options }) { + options.abortSignal?.throwIfAborted(); + const reader = content.getReader(); + const abort = () => { + void reader.cancel().catch(() => {}); + }; + options.abortSignal?.addEventListener("abort", abort, { once: true }); + const chunks: Uint8Array[] = []; + let bytes = 0; + try { + for (;;) { + options.abortSignal?.throwIfAborted(); + const next = await reader.read(); + options.abortSignal?.throwIfAborted(); + if (next.done) break; + bytes += next.value.byteLength; + checkSize(bytes, "File write"); + if (next.value.byteLength) chunks.push(Buffer.from(next.value)); + } + await session.writeBinaryFile({ ...options, content: Buffer.concat(chunks) }); + } finally { + options.abortSignal?.removeEventListener("abort", abort); + void reader.cancel().catch(() => {}); + reader.releaseLock(); + } + }, + async removePath({ path, recursive = false, force = false, abortSignal }) { + abortSignal?.throwIfAborted(); + const target = resolvePath(path); + await rejectSymlinkPath(target); + await rm(target, { recursive, force }); + }, + async setNetworkPolicy(policy) { + if (policy !== "allow-all") + throw new Error( + "The runner owns network policy; this backend cannot change the firewall or broker credentials", + ); + }, + }; + return { + name, + async prewarm({ seedFiles, bootstrap }) { + if (bootstrap) + throw new Error("Production sandbox bootstrap must not run in the task container"); + for (const file of seedFiles) { + const path = resolvePath(file.path); + const skills = skillsHome(); + if (!(path.startsWith(`${root}/`) || (skills && path.startsWith(`${skills}/`)))) + throw new Error("Unexpected seed path: use the task workspace or $HOME/.agents/skills"); + checkSize( + typeof file.content === "string" + ? Buffer.byteLength(file.content) + : file.content.byteLength, + "Seed file", + ); + } + await mkdir(root, { recursive: true }); + for (const file of seedFiles) { + await session.writeBinaryFile({ + path: file.path, + content: typeof file.content === "string" ? Buffer.from(file.content) : file.content, + }); + } + return { reused: false }; + }, + async create({ sessionKey }) { + await mkdir(root, { recursive: true }); + const liveSession = { ...session, id: sessionKey }; + return { + session: liveSession, + useSessionFn: async () => liveSession, + captureState: async () => ({ backendName: name, metadata: {}, sessionKey }), + stop, + shutdown: stop, + }; + }, + }; +} + +function bufferEncoding(encoding: string): BufferEncoding { + if (!Buffer.isEncoding(encoding)) throw new Error(`Unsupported encoding: ${encoding}`); + return encoding; +} diff --git a/packages/eve-bench/src/harnesses/e0/index.ts b/packages/eve-bench/src/harnesses/e0/index.ts new file mode 100644 index 0000000000..412b3ec4ca --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/index.ts @@ -0,0 +1,252 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import type { Harness, HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; +import { ensureNode, NODE_VERSION } from "../eve/index.ts"; +import { readEveUsage } from "../eve/usage.ts"; +import { copySource, hashTree } from "./snapshot.ts"; + +const exec = promisify(execFile); +const here = dirname(fileURLToPath(import.meta.url)); +const REASONING = ["provider-default", "none", "minimal", "low", "medium", "high", "xhigh"]; + +export interface E0Options { + readonly agent: string; + readonly reasoning?: string; +} + +export function createE0Harness(options: E0Options): Harness { + if (!options.agent?.trim()) + throw new Error( + "--agent must point to the installed e0 application (containing agent/ and package.json)", + ); + if (options.reasoning && !REASONING.includes(options.reasoning)) + throw new Error(`Unsupported e0 reasoning: ${options.reasoning}`); + const agent = resolve(options.agent); + return { + name: "e0", + credentials: ["AI_GATEWAY_API_KEY"], + prepare: (ctx) => prepareE0({ ...options, agent }, ctx), + command: (ctx) => `exec '${ctx.installDir}/arch/node' '${ctx.installDir}/app/runner.mjs'`, + readUsage: readEveUsage, + env: (ctx) => { + const env: Record = { + EVE_PROJECT_DIR: `${ctx.installDir}/app`, + EVE_INSTRUCTION_PATH: ctx.instructionPath, + EVE_AGENT_LOGS_DIR: ctx.logsDir, + EVE_BENCH_CONTAINER: "1", + EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, + E0_MODEL: ctx.model, + }; + if (options.reasoning) env.E0_REASONING = options.reasoning; + return env; + }, + }; +} + +export async function adaptE0(app: string): Promise { + const agent = join(app, "agent"); + // These integrations are outside the self-contained terminal workload. Authored + // coding behavior remains intact, including the extension's worker model. + for (const slot of ["sandbox", "channels", "schedules", "connections", "instrumentation"]) { + await rm(join(agent, slot), { recursive: true, force: true }); + await rm(join(agent, `${slot}.ts`), { force: true }); + } + await mkdir(join(agent, "lib"), { recursive: true }); + await cp(join(here, "container-backend.ts"), join(agent, "lib/eve-bench-backend.ts")); + await writeFile( + join(agent, "sandbox.ts"), + `import { defineSandbox } from "eve/sandbox";\nimport { containerBackend } from "./lib/eve-bench-backend.ts";\nexport default defineSandbox({ backend: () => containerBackend() });\n`, + ); + await mkdir(join(agent, "channels"), { recursive: true }); + await writeFile( + join(agent, "channels/eve.ts"), + `import { eveChannel } from "eve/channels/eve";\nimport { localDev } from "eve/channels/auth";\nexport default eveChannel({ auth: [localDev()] });\n`, + ); + await writeFile( + join(agent, "extensions/code.ts"), + `import code from "eve-code";\nexport default code({});\n`, + ); +} + +async function sourceIdentity(agent: string, extension: string) { + const pkg = JSON.parse(await readFile(join(agent, "package.json"), "utf8")) as { + dependencies: Record; + }; + const dependencies: Record = {}; + for (const name of Object.keys(pkg.dependencies).sort()) { + const installed = JSON.parse( + await readFile(join(agent, "node_modules", name, "package.json"), "utf8"), + ) as { version: string }; + dependencies[name] = installed.version; + } + const gitRoot = ( + await exec("git", ["rev-parse", "--show-toplevel"], { cwd: agent }) + ).stdout.trim(); + const gitSha = (await exec("git", ["rev-parse", "HEAD"], { cwd: agent })).stdout.trim(); + const scope = [ + relative(gitRoot, agent), + relative(gitRoot, extension), + "pnpm-lock.yaml", + "pnpm-workspace.yaml", + ]; + const conflicts = ( + await exec("git", ["diff", "--name-only", "--diff-filter=U", "--", ...scope], { cwd: gitRoot }) + ).stdout.trim(); + if (conflicts) throw new Error(`e0 source has unresolved merge conflicts:\n${conflicts}`); + const status = ( + await exec("git", ["status", "--porcelain", "--", ...scope], { cwd: gitRoot }) + ).stdout + .split("\n") + .filter((line) => !line.includes("/.eve-bench-")) + .join("\n"); + const metadata = createHash("sha256"); + for (const file of [ + join(agent, "package.json"), + join(agent, "tsconfig.json"), + join(extension, "package.json"), + join(extension, "tsconfig.json"), + join(gitRoot, "pnpm-lock.yaml"), + join(gitRoot, "pnpm-workspace.yaml"), + ]) { + metadata + .update(relative(gitRoot, file)) + .update("\0") + .update(await readFile(file)); + } + return { + gitSha, + dirty: status.trim().length > 0, + agentSha256: await hashTree(join(agent, "agent")), + extensionSha256: await hashTree(join(extension, "extension")), + metadataSha256: metadata.digest("hex"), + eveDistSha256: await hashTree(join(agent, "node_modules/eve/dist")), + dependencies, + }; +} + +async function prepareE0(options: E0Options, ctx: HarnessPrepareContext): Promise { + const agent = await realpath(options.agent); + let extension: string; + try { + extension = await realpath(join(agent, "node_modules/eve-code")); + await stat(join(extension, "extension/extension.ts")); + await stat(join(agent, "node_modules/.bin/eve")); + await stat(join(agent, "agent/extensions/code.ts")); + } catch { + throw new Error( + "e0 requires installed workspace dependencies and source-backed eve-code. Run pnpm install in internal-agents first.", + ); + } + const source = await sourceIdentity(agent, extension); + const adapter = createHash("sha256"); + for (const file of [ + "index.ts", + "snapshot.ts", + "container-backend.ts", + "../eve/index.ts", + "../eve/runner.mjs", + ]) + adapter.update(await readFile(join(here, file))); + const identity = { + source, + model: ctx.model, + reasoning: options.reasoning ?? "authored", + workerModel: "authored (not overridden)", + nodeVersion: NODE_VERSION, + adapterSha256: adapter.digest("hex"), + adaptation: + "Task-container sandbox; local eve channel; production extension configuration removed. No production channels, schedules, connections, instrumentation, sandbox bootstrap, or Connect configuration. Instructions, coding capabilities, tools, skills, and authored worker preserved.", + }; + const key = createHash("sha256").update(JSON.stringify(identity)).digest("hex"); + const cache = resolve(ctx.cacheDir); + const destination = join(cache, "e0", key); + const nodeDir = join(cache, `node-v${NODE_VERSION}`); + await Promise.all((["x64", "arm64"] as const).map((arch) => ensureNode(nodeDir, arch))); + const bundle = { + dir: destination, + arch: { x64: join(nodeDir, "x64"), arm64: join(nodeDir, "arm64") }, + provenance: { key, ...identity }, + }; + if (await exists(join(destination, "provenance.json"))) return bundle; + + // An adjacent disposable directory resolves the already-installed dependency + // graph without modifying source files, installing new versions, or adding links. + const scratch = await mkdtemp(join(agent, ".eve-bench-")); + const buildEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + CI: "true", + E0_MODEL: ctx.model, + }; + if (options.reasoning) buildEnv.E0_REASONING = options.reasoning; + try { + await copySource(join(agent, "agent"), join(scratch, "agent")); + for (const file of ["package.json", "tsconfig.json"]) + await cp(join(agent, file), join(scratch, file)); + const stagedExtension = join(scratch, "node_modules/eve-code"); + await mkdir(stagedExtension, { recursive: true }); + await copySource(join(extension, "extension"), join(stagedExtension, "extension")); + for (const file of ["package.json", "tsconfig.json"]) + await cp(join(extension, file), join(stagedExtension, file)); + const eve = join(agent, "node_modules/.bin/eve"); + await exec(eve, ["extension", "build"], { + cwd: stagedExtension, + env: buildEnv, + maxBuffer: 8 * 1024 * 1024, + }); + await adaptE0(scratch); + await exec(eve, ["build"], { cwd: scratch, env: buildEnv, maxBuffer: 16 * 1024 * 1024 }); + const after = await sourceIdentity(agent, extension); + if (JSON.stringify(after) !== JSON.stringify(source)) { + throw new Error( + "e0 source or dependencies changed during preparation; retry after edits settle", + ); + } + await mkdir(join(cache, "e0"), { recursive: true }); + const publishing = await mkdtemp(join(cache, "e0/.publish-")); + try { + await mkdir(join(publishing, "app")); + await cp(join(scratch, ".output"), join(publishing, "app/.output"), { + recursive: true, + dereference: true, + }); + await cp(join(here, "../eve/runner.mjs"), join(publishing, "app/runner.mjs")); + await writeFile( + join(publishing, "provenance.json"), + `${JSON.stringify(bundle.provenance, null, 2)}\n`, + ); + try { + await rename(publishing, destination); + } catch (error) { + if (!(await exists(join(destination, "provenance.json")))) throw error; + } + } finally { + await rm(publishing, { recursive: true, force: true }); + } + return bundle; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.test.ts b/packages/eve-bench/src/harnesses/e0/snapshot.test.ts new file mode 100644 index 0000000000..01174b85b4 --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/snapshot.test.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { copySource, hashTree, includeSource } from "./snapshot.ts"; + +test("source filter excludes generated and secret-bearing files", () => { + for (const path of [ + "node_modules", + ".git", + ".eve", + ".output", + ".next", + "dist", + ".env", + ".env.local", + ".eve-bench-work", + ]) { + assert.equal(includeSource(path), false, path); + } + assert.equal(includeSource("agent.ts"), true); +}); + +test("hashTree is content-sensitive and copySource preserves its identity", async (t) => { + const root = await mkdtemp(join(tmpdir(), "eve-bench-source-")); + t.after(() => rm(root, { recursive: true, force: true })); + const source = join(root, "source"); + const destination = join(root, "destination"); + await mkdir(join(source, "nested"), { recursive: true }); + await writeFile(join(source, "agent.ts"), "export default 1;\n"); + await writeFile(join(source, "nested/file.txt"), "one\n"); + const before = await hashTree(source); + await copySource(source, destination); + assert.equal(await hashTree(destination), before); + await writeFile(join(source, "nested/file.txt"), "two\n"); + assert.notEqual(await hashTree(source), before); +}); + +test("hashTree rejects symlinks", async (t) => { + const root = await mkdtemp(join(tmpdir(), "eve-bench-source-link-")); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, "target"), "data"); + await symlink(join(root, "target"), join(root, "link")); + await assert.rejects(hashTree(root), /does not support symlinks/); +}); diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.ts b/packages/eve-bench/src/harnesses/e0/snapshot.ts new file mode 100644 index 0000000000..dfddddec4e --- /dev/null +++ b/packages/eve-bench/src/harnesses/e0/snapshot.ts @@ -0,0 +1,53 @@ +import { createHash } from "node:crypto"; +import { cp, lstat, readdir, readFile } from "node:fs/promises"; +import { basename, join, relative } from "node:path"; + +const EXCLUDED = new Set(["node_modules", ".git", ".eve", ".output", ".next", "dist"]); + +export function includeSource(path: string): boolean { + const name = basename(path); + return !EXCLUDED.has(name) && !name.startsWith(".env") && !name.startsWith(".eve-bench-"); +} + +/** Source snapshots never follow links into credentials, build output, or another checkout. */ +export async function hashTree(root: string): Promise { + const hash = createHash("sha256"); + let count = 0; + let bytes = 0; + async function walk(dir: string, depth: number): Promise { + if (depth > 32) throw new Error("Source tree exceeds 32 directory levels"); + for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name), + )) { + if (!includeSource(entry.name)) continue; + const path = join(dir, entry.name); + if (++count > 20_000) throw new Error("Source tree exceeds 20000 entries"); + if (entry.isSymbolicLink()) + throw new Error(`Source snapshot does not support symlinks: ${path}`); + if (entry.isDirectory()) await walk(path, depth + 1); + else if (entry.isFile()) { + const info = await lstat(path); + bytes += info.size; + if (bytes > 128 * 1024 * 1024) throw new Error("Source tree exceeds 128 MiB"); + hash + .update(relative(root, path)) + .update("\0") + .update(String(info.mode & 0o777)) + .update("\0"); + hash.update(await readFile(path)).update("\0"); + } else throw new Error(`Source snapshot requires regular files: ${path}`); + } + } + await walk(root, 0); + return hash.digest("hex"); +} + +export async function copySource(source: string, destination: string): Promise { + const before = await hashTree(source); + await cp(source, destination, { recursive: true, filter: includeSource }); + if ((await hashTree(destination)) !== before || (await hashTree(source)) !== before) { + throw new Error( + "Source changed while preparing the benchmark snapshot; retry after edits settle", + ); + } +} diff --git a/packages/eve-bench/src/harnesses/eve/index.ts b/packages/eve-bench/src/harnesses/eve/index.ts index 55dcc4dabf..d01a6976d6 100644 --- a/packages/eve-bench/src/harnesses/eve/index.ts +++ b/packages/eve-bench/src/harnesses/eve/index.ts @@ -12,12 +12,13 @@ import type { HarnessPrepareContext, HarnessRunContext, } from "../../core/harness.ts"; +import { readEveUsage } from "./usage.ts"; const execFileAsync = promisify(execFile); const here = dirname(fileURLToPath(import.meta.url)); const packageRoot = resolve(here, "../../.."); const repoRoot = resolve(packageRoot, "../.."); -const NODE_VERSION = "24.19.0"; +export const NODE_VERSION = "24.19.0"; const ARCHES = ["x64", "arm64"] as const; export type EveSource = @@ -32,8 +33,10 @@ export type EveSource = export function createEveHarness(source: EveSource): Harness { return { name: source.kind === "local" ? "eve@local" : `eve@${source.version}`, + credentials: ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"], prepare: (ctx) => prepare(source, ctx), command: (ctx) => `sh ${ctx.installDir}/run.sh`, + readUsage: readEveUsage, env: (ctx: HarnessRunContext) => ({ EVE_PROJECT_DIR: `${ctx.installDir}/app`, EVE_INSTRUCTION_PATH: ctx.instructionPath, @@ -163,7 +166,7 @@ async function packLocalEve(artifacts: string): Promise { return resolve(artifacts, packed.filename); } -async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { +export async function ensureNode(nodeDir: string, arch: (typeof ARCHES)[number]): Promise { const archDir = join(nodeDir, arch); const target = join(archDir, "node"); if (await exists(target)) return; diff --git a/packages/eve-bench/src/harnesses/eve/runner.mjs b/packages/eve-bench/src/harnesses/eve/runner.mjs index d2e593afca..6a16082445 100644 --- a/packages/eve-bench/src/harnesses/eve/runner.mjs +++ b/packages/eve-bench/src/harnesses/eve/runner.mjs @@ -1,9 +1,10 @@ // Runs inside the task container: boots the built eve server, sends the task // instruction as one session, follows the event stream to its boundary, and -// writes /logs/agent/{events.ndjson,result.json,server.log}. Talks to the -// server over plain HTTP so the bundle ships no node_modules. +// writes /logs/agent/{events.ndjson,result.json,server.log}. Events are +// appended as they arrive so the host can read usage after an agent timeout. +// Talks to the server over plain HTTP so the bundle ships no node_modules. import { spawn } from "node:child_process"; -import { createWriteStream } from "node:fs"; +import { appendFileSync, createWriteStream, writeFileSync } from "node:fs"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -28,9 +29,9 @@ try { const instruction = await readFile(instructionPath, "utf8"); const sessionId = await createSession(instruction); + writeFileSync(eventsPath, ""); const events = await followStream(sessionId); - await writeFile(eventsPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); const status = statusFromEvents(events); const summary = { status, @@ -39,7 +40,6 @@ try { inputRequests: events .filter((event) => event.type === "input.requested") .flatMap((event) => event.data?.requests ?? []), - usage: summarizeUsage(events), eventCount: events.length, failure: failureFromEvents(events) ?? failureFromStatus(status), }; @@ -54,7 +54,6 @@ try { { status: "failed", inputRequests: [], - usage: {}, eventCount: 0, failure: { type: "runner.error", message: error?.message ?? String(error) }, }, @@ -77,15 +76,23 @@ function requireEnv(name) { function startServer() { const log = createWriteStream(serverLogPath, { flags: "a" }); const entry = join(projectDir, ".output", "server", "index.mjs"); + const serverEnv = { + EVE_DEV: "1", + HOST: host, + NITRO_HOST: host, + NITRO_PORT: String(port), + PORT: String(port), + }; + // Agent tools restore these so task commands see the container's own values. + const taskEnv = Object.fromEntries( + Object.keys(serverEnv).map((key) => [key, process.env[key] ?? null]), + ); const child = spawn(process.execPath, [entry], { cwd: projectDir, env: { ...process.env, - EVE_DEV: "1", - HOST: host, - NITRO_HOST: host, - NITRO_PORT: String(port), - PORT: String(port), + ...serverEnv, + EVE_BENCH_TASK_ENV_RESTORE: JSON.stringify(taskEnv), }, stdio: ["ignore", "pipe", "pipe"], }); @@ -141,6 +148,7 @@ async function followStream(sessionId) { throw new Error(`stream failed: ${response.status} ${await response.text()}`); for await (const event of ndjson(response.body)) { events.push(event); + appendFileSync(eventsPath, `${JSON.stringify(event)}\n`); if (isBoundary(event)) return events; } await new Promise((resolve) => setTimeout(resolve, 250)); @@ -207,30 +215,6 @@ function failureFromStatus(status) { return { type: "session.unsettled", message: `eve run ended with status: ${status}` }; } -function summarizeUsage(events) { - const usage = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: 0 }; - for (const event of events) { - if (event.type !== "step.completed") continue; - const source = event.data?.usage ?? event.data ?? {}; - usage.inputTokens += numberValue(source.inputTokens, source.input_tokens); - usage.outputTokens += numberValue(source.outputTokens, source.output_tokens); - usage.cachedTokens += numberValue( - source.cacheReadTokens, - source.cachedTokens, - source.cached_tokens, - ); - usage.costUsd += numberValue(source.costUsd, source.cost_usd); - } - return usage; -} - -function numberValue(...values) { - for (const value of values) { - if (typeof value === "number" && Number.isFinite(value)) return value; - } - return 0; -} - async function stopServer(child) { if (child.exitCode !== null) return; child.kill("SIGTERM"); diff --git a/packages/eve-bench/src/harnesses/eve/usage.ts b/packages/eve-bench/src/harnesses/eve/usage.ts new file mode 100644 index 0000000000..51d3bfef86 --- /dev/null +++ b/packages/eve-bench/src/harnesses/eve/usage.ts @@ -0,0 +1,22 @@ +import { join } from "node:path"; + +import type { Usage } from "../../core/result.ts"; +import { count, readJsonlUsage, record, type UsageRecord } from "../../core/usage.ts"; + +/** Sums `step.completed` usage from the runner's incrementally written event log. */ +export function readEveUsage(agentLogsDir: string): Promise { + return readJsonlUsage(join(agentLogsDir, "events.ndjson"), eveStepUsage); +} + +export function eveStepUsage(event: unknown): UsageRecord | undefined { + const value = record(event); + if (value.type !== "step.completed") return undefined; + const data = record(value.data); + const usage = record(data.usage ?? data); + return { + input: count(usage.inputTokens, usage.input_tokens), + output: count(usage.outputTokens, usage.output_tokens), + cached: count(usage.cacheReadTokens, usage.cachedTokens, usage.cached_tokens), + cost: count(usage.costUsd, usage.cost_usd), + }; +} diff --git a/packages/eve-bench/src/harnesses/oracle.ts b/packages/eve-bench/src/harnesses/oracle.ts index 1389f91a22..c8e764f2c7 100644 --- a/packages/eve-bench/src/harnesses/oracle.ts +++ b/packages/eve-bench/src/harnesses/oracle.ts @@ -11,6 +11,7 @@ import type { Harness } from "../core/harness.ts"; export function createOracleHarness(): Harness { return { name: "oracle", + modelFree: true, async prepare(ctx) { const dir = join(ctx.cacheDir, "bundles", "oracle"); await mkdir(dir, { recursive: true }); diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts index cf35a1a996..1b5576a1d5 100644 --- a/packages/eve-bench/src/options.ts +++ b/packages/eve-bench/src/options.ts @@ -6,6 +6,8 @@ import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts import type { Harness } from "./core/harness.ts"; import { createEveHarness } from "./harnesses/eve/index.ts"; import { createOracleHarness } from "./harnesses/oracle.ts"; +import { createE0Harness } from "./harnesses/e0/index.ts"; +import { createCliHarness } from "./harnesses/cli/index.ts"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -76,7 +78,23 @@ export function defaultJobName(label: string): string { return `${stamp}-${label.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}`; } -export function selectHarness(name: string, eve: string): Harness { +export function selectHarness( + name: string, + eve: string, + options: { agent?: string; version?: string; baseUrl?: string; reasoning?: string } = {}, +): Harness { + if (name !== "e0" && options.agent) throw new Error("--agent is only supported by --harness e0"); + if (name === "e0") + return createE0Harness({ agent: options.agent ?? "", reasoning: options.reasoning }); + if (name === "pi" || name === "opencode" || name === "codex" || name === "hermes") { + return createCliHarness(name, { + version: options.version ?? "", + baseUrl: options.baseUrl, + reasoning: options.reasoning, + }); + } + if (options.version || options.baseUrl || options.reasoning) + throw new Error("--version, --base-url and --reasoning require a supporting harness"); if (name === "oracle") return createOracleHarness(); if (name === "eve") { return createEveHarness( @@ -86,17 +104,19 @@ export function selectHarness(name: string, eve: string): Harness { throw new Error(`unknown harness: ${name}`); } -export function forwardedEnv(): Record { +/** Host credentials the harness may receive; fails before any trial when none are exported. */ +export function forwardedEnv(harness: Harness): Record { + const names = harness.credentials ?? []; const env: Record = {}; - for (const key of [ - "AI_GATEWAY_API_KEY", - "VERCEL_OIDC_TOKEN", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - ]) { + for (const key of names) { const value = process.env[key]; if (value) env[key] = value; } + if (names.length > 0 && Object.keys(env).length === 0) { + throw new Error( + `No model credentials exported for ${harness.name}. Set one of ${names.join(", ")} (AI_GATEWAY_API_KEY for gateway models); use prepare for a model-free build.`, + ); + } return env; } diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts index 09f410af5f..89697ee570 100644 --- a/packages/eve-bench/src/tools/run_bench.ts +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -13,7 +13,11 @@ export const inputSchema = z cohort: z.string().min(1).optional(), tasks: z.array(z.string().min(1)).min(1).optional(), model: z.string().min(1).optional(), - harness: z.enum(["eve", "oracle"]).default("eve"), + harness: z.enum(["eve", "e0", "oracle", "pi", "opencode", "codex", "hermes"]).default("eve"), + agent: z.string().min(1).optional(), + version: z.string().min(1).optional(), + baseUrl: z.string().url().optional(), + reasoning: z.string().min(1).optional(), eve: z.string().min(1).default("local"), attempts: z.number().int().positive().default(1), concurrency: z.number().int().positive().default(4), @@ -23,13 +27,20 @@ export const inputSchema = z if (input.harness !== "oracle" && !input.model) { ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); } + if (input.harness === "e0" && !input.agent) { + ctx.addIssue({ code: "custom", path: ["agent"], message: "agent is required for e0" }); + } + if (["pi", "opencode", "codex", "hermes"].includes(input.harness) && !input.version) { + ctx.addIssue({ code: "custom", path: ["version"], message: "version is required" }); + } }); export default defineTool({ description: "Run an eve benchmark job and return its result.", inputSchema, async execute(input, ctx) { - const harness = selectHarness(input.harness, input.eve); + const harness = selectHarness(input.harness, input.eve, input); + const forwardEnv = forwardedEnv(harness); const model = input.model ?? "none"; const selection = await selectTasks({ cohort: input.cohort, task: input.tasks }); const datasetDir = await syncDataset(join(paths.generatedRoot, "datasets"), selection.lock); @@ -48,7 +59,7 @@ export default defineTool({ model, attempts: input.attempts, concurrency: input.concurrency, - forwardEnv: forwardedEnv(), + forwardEnv, signal: ctx.abortSignal, }); }, diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts index 41515610e8..1e5933f587 100644 --- a/packages/eve-bench/src/tools/tools.test.ts +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -22,6 +22,19 @@ describe("run_bench input", () => { true, ); assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); + assert.equal( + runBenchInput.safeParse({ harness: "e0", agent: "/apps/e0", model: "openai/gpt-5.6-terra" }) + .success, + true, + ); + assert.equal( + runBenchInput.safeParse({ + harness: "pi", + version: "0.80.3", + model: "deepseek/deepseek-v4-pro", + }).success, + true, + ); }); it("rejects invalid inputs", () => { @@ -29,6 +42,14 @@ describe("run_bench input", () => { assert.equal(runBenchInput.safeParse({ harness: "other", model: "model" }).success, false); assert.equal(runBenchInput.safeParse({ harness: "oracle", attempts: 0 }).success, false); assert.equal(runBenchInput.safeParse({ harness: "oracle", tasks: "fix-git" }).success, false); + assert.equal( + runBenchInput.safeParse({ harness: "e0", model: "openai/gpt-5.6-terra" }).success, + false, + ); + assert.equal( + runBenchInput.safeParse({ harness: "codex", model: "openai/gpt-5.6-terra" }).success, + false, + ); }); }); diff --git a/packages/eve-bench/test/scenarios/cli-adapters.test.ts b/packages/eve-bench/test/scenarios/cli-adapters.test.ts new file mode 100644 index 0000000000..3d10c8af26 --- /dev/null +++ b/packages/eve-bench/test/scenarios/cli-adapters.test.ts @@ -0,0 +1,320 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { createCliHarness } from "../../src/harnesses/cli/index.ts"; +import { shellQuote, type SupportedCliName } from "../../src/harnesses/cli/plan.ts"; +import type { Harness, HarnessBundle, HarnessRunContext } from "../../src/core/harness.ts"; + +const exec = promisify(execFile); +const sourceDir = fileURLToPath(new URL("../../src/harnesses/cli/", import.meta.url)); +const FAKE_KEY = 'model-free-fake-key-"abcdef'; +const TASK_HOME = "/task-home"; + +async function temp(t: TestContext): Promise { + const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-cli-"))); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} + +const mockCli = `import { readFile } from 'node:fs/promises'; +const args = process.argv.slice(2); +if (args[0] === '--version') { + if (process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY) process.exit(92); + console.log('mock-cli 1.2.3'); +} else { + const name = JSON.parse(await readFile(new URL('./identity.json', import.meta.url), 'utf8')).name; + const path = name === 'pi' ? process.env.PI_CODING_AGENT_DIR + '/models.json' + : name === 'codex' ? process.env.CODEX_HOME + '/config.toml' + : process.env.OPENCODE_CONFIG; + const config = await readFile(path, 'utf8'); + let input = ''; + for await (const chunk of process.stdin) input += chunk.toString(); + const prompt = name === 'pi' ? input : args.at(-1); + if (name === 'pi' && args.includes('--')) throw new Error('pi does not support a -- separator'); + console.log(JSON.stringify({ type: 'mock', args, prompt, cwd: process.cwd(), config, + env: { PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR, CODEX_HOME: process.env.CODEX_HOME, + OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, OPENCODE_FAKE_VCS: process.env.OPENCODE_FAKE_VCS, + HOME: process.env.HOME, PATH: process.env.PATH, TASK_IMAGE_ENV: process.env.TASK_IMAGE_ENV, + XDG_DATA_HOME: process.env.XDG_DATA_HOME, runnerContext: process.env.EVE_BENCH_CLI_CONTEXT }, + credentialMapped: process.env.AI_GATEWAY_API_KEY === ${JSON.stringify(FAKE_KEY)} && process.env.OPENAI_API_KEY === process.env.AI_GATEWAY_API_KEY })); + const key = process.env.AI_GATEWAY_API_KEY; + process.stdout.write(key.slice(0, 12)); + await new Promise(r => setTimeout(r, 10)); + process.stdout.write(key.slice(12) + '\\n'); + process.stderr.write(JSON.stringify(key) + '\\n'); + if (prompt === '__opencode_error__') process.stdout.write(JSON.stringify({ type: 'error', error: 'mock failure' })); + if (prompt === '__exit_7__') process.exitCode = 7; + if (prompt === '__signal__') process.kill(process.pid, 'SIGTERM'); +} +`; + +async function mockInstalled(t: TestContext, name: SupportedCliName) { + const root = await temp(t); + const installDir = join(root, "install with 'quote"); + await mkdir(join(installDir, "arch"), { recursive: true }); + for (const file of ["runner.ts", "plan.ts", "output.ts"]) + await cp(join(sourceDir, file), join(installDir, file)); + await writeFile( + join(installDir, "arch/node"), + `#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`, + { mode: 0o755 }, + ); + await writeFile(join(installDir, "arch/mock.mjs"), mockCli); + await writeFile(join(installDir, "arch/identity.json"), JSON.stringify({ name })); + await writeFile(join(installDir, "arch/entry.json"), JSON.stringify({ entry: "mock.mjs" })); + const ctx = { + model: "deepseek/deepseek-v4", + installDir, + instructionPath: join(root, "instruction with 'quote.md"), + taskWorkdir: join(root, "task with space"), + logsDir: join(root, "logs"), + }; + await mkdir(ctx.taskWorkdir); + return { root, ctx, harness: createCliHarness(name, { version: "1.2.3", reasoning: "high" }) }; +} + +async function invoke( + harness: Harness, + ctx: HarnessRunContext, + instruction: string, + key: string | undefined = FAKE_KEY, +) { + await writeFile(ctx.instructionPath, instruction); + try { + const result = await exec("sh", ["-c", harness.command(ctx)], { + env: { + PATH: process.env.PATH, + HOME: TASK_HOME, + TASK_IMAGE_ENV: "from-image", + AI_GATEWAY_API_KEY: key, + ...harness.env(ctx), + }, + maxBuffer: 4 * 1024 * 1024, + }); + return { code: 0, ...result }; + } catch (error) { + const failed = error as Error & { code: number; stdout: string; stderr: string }; + return { code: failed.code, stdout: failed.stdout, stderr: failed.stderr }; + } +} + +for (const name of ["pi", "opencode", "codex"] as const) { + for (const model of ["deepseek/deepseek-v4", "openai/gpt-5.6"]) { + test(`${name} mock binary receives native flags, full ${model}, safe config and runtime credentials`, async (t) => { + const { ctx, harness } = await mockInstalled(t, name); + ctx.model = model; + const prompt = "--untrusted 'quote'\n$(touch SHOULD_NOT_EXIST)"; + const result = await invoke(harness, ctx, prompt); + assert.equal(result.code, 0, result.stderr); + const log = await readFile(join(ctx.logsDir, `${name}.jsonl`), "utf8"); + const record = JSON.parse(log.split("\n")[0]!); + assert.equal(record.cwd, ctx.taskWorkdir); + assert.equal(record.prompt, prompt); + assert.equal(record.credentialMapped, true); + // Task-side state must stay where the verifier will look for it. + assert.equal(record.env.HOME, TASK_HOME); + assert.equal(record.env.PATH, process.env.PATH); + assert.equal(record.env.TASK_IMAGE_ENV, "from-image"); + assert.equal(record.env.XDG_DATA_HOME, undefined); + assert.equal(record.env.runnerContext, undefined); + assert.ok( + record.args.includes(model) || record.args.includes(`--model=eve-bench-gateway/${model}`), + ); + assert.match(record.config, /ai-gateway.vercel.sh/); + if (name === "pi") { + assert.equal(JSON.parse(record.config).providers["eve-bench-gateway"].models[0].id, model); + assert.match(record.config, /\$AI_GATEWAY_API_KEY/); + } else if (name === "opencode") { + assert.ok(JSON.parse(record.config).provider["eve-bench-gateway"].models[model]); + assert.equal(record.env.OPENCODE_FAKE_VCS, "git"); + } else { + assert.match(record.config, /wire_api = "responses"/); + assert.match(record.config, /env_key = "AI_GATEWAY_API_KEY"/); + } + for (const file of await readdir(ctx.logsDir)) { + const content = await readFile(join(ctx.logsDir, file), "utf8"); + assert.ok(!content.includes(FAKE_KEY)); + assert.ok(!content.includes(JSON.stringify(FAKE_KEY).slice(1, -1))); + } + assert.ok(!result.stdout.includes(FAKE_KEY)); + assert.ok(!result.stderr.includes(FAKE_KEY)); + assert.match(log, /\[REDACTED\]/); + assert.deepEqual(await readdir(ctx.taskWorkdir), []); + assert.ok(!(await readdir(ctx.installDir)).some((file) => file.startsWith(".cli-runtime-"))); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.installLocation, "host"); + assert.equal(metrics.containerInstallMs, 0); + assert.ok(metrics.setupMs >= 0); + assert.ok(metrics.agentMs > 0); + assert.equal(metrics.exitCode, 0); + }); + } +} + +test("native nonzero exits, signals, and OpenCode zero-exit error events fail the harness", async (t) => { + for (const [name, instruction, expected] of [ + ["pi", "__exit_7__", 7], + ["codex", "__signal__", 1], + ["opencode", "__opencode_error__", 1], + ] as const) { + const { ctx, harness } = await mockInstalled(t, name); + const result = await invoke(harness, ctx, instruction); + assert.equal(result.code, expected); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.exitCode, expected); + if (instruction === "__signal__") assert.equal(metrics.signal, "SIGTERM"); + if (instruction === "__opencode_error__") assert.match(metrics.failure, /error event/); + } +}); + +test("missing key, oversized prompt, and version mismatch fail before the agent starts", async (t) => { + const { ctx, harness } = await mockInstalled(t, "pi"); + const missing = await invoke(harness, ctx, "do not run", ""); + assert.equal(missing.code, 1); + assert.match(missing.stderr, /AI_GATEWAY_API_KEY must be forwarded/); + assert.deepEqual(await readdir(ctx.logsDir), ["cli-runtime.json"]); + const oversized = await invoke(harness, ctx, "x".repeat(1024 * 1024 + 1)); + assert.equal(oversized.code, 1); + assert.match(oversized.stderr, /exceeds 1 MiB/); + const wrong = await invoke(createCliHarness("pi", { version: "1.2.30" }), ctx, "do not run"); + assert.equal(wrong.code, 1); + assert.match(wrong.stderr, /failed its --version check/); + const metrics = JSON.parse(await readFile(join(ctx.logsDir, "cli-runtime.json"), "utf8")); + assert.equal(metrics.agentMs, 0); +}); + +test("host prepare uses pinned pnpm installs, verifies archives, caches across models, and relocates", async (t) => { + const root = await temp(t); + const bin = join(root, "bin"); + await mkdir(bin); + const calls = join(root, "pnpm-calls.jsonl"); + const shebang = `#!${process.execPath}\n`; + await writeFile( + join(bin, "pnpm"), + shebang + + ` +import { readFile, writeFile, mkdir, appendFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +if (process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY || process.env.UNRELATED_TEST_API_KEY) process.exit(91); +if (args[0] === '--version') { console.log('11.21.0'); } +else { + await appendFile(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); + const manifest = JSON.parse(await readFile('package.json', 'utf8')); + const [pkg, version] = Object.entries(manifest.dependencies)[0]; + const dir = join(process.cwd(), 'node_modules', pkg); + const name = pkg.includes('pi-coding-agent') ? 'pi' : pkg === 'opencode-ai' ? 'opencode' : 'codex'; + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'package.json'), JSON.stringify({ version, bin: { [name]: 'cli.mjs' } })); + await writeFile(join(dir, 'cli.mjs'), ${JSON.stringify(mockCli)}); + await writeFile(join(dir, 'identity.json'), JSON.stringify({ name })); + await writeFile('pnpm-lock.yaml', 'lockfileVersion: 9.0\\n'); +} +`, + { mode: 0o755 }, + ); + await writeFile( + join(bin, "tar"), + shebang + + ` +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +const args = process.argv.slice(2); +await writeFile(join(args[args.indexOf('-C') + 1], 'node'), ${JSON.stringify(`#!/bin/sh\nexec ${shellQuote(process.execPath)} "$@"\n`)}); +`, + { mode: 0o755 }, + ); + const originalPath = process.env.PATH; + const originalKey = process.env.AI_GATEWAY_API_KEY; + process.env.PATH = `${bin}:${originalPath}`; + process.env.AI_GATEWAY_API_KEY = FAKE_KEY; + t.after(() => { + process.env.PATH = originalPath; + if (originalKey === undefined) delete process.env.AI_GATEWAY_API_KEY; + else process.env.AI_GATEWAY_API_KEY = originalKey; + }); + const archive = Buffer.from("mock node archive"); + const hash = createHash("sha256").update(archive).digest("hex"); + let fetches = 0; + t.mock.method(globalThis, "fetch", async (url: string) => { + fetches++; + assert.match(url, /^https:\/\/nodejs.org\/dist\/v24\.19\.0\//); + return new Response( + url.endsWith("SHASUMS256.txt") + ? `${hash} node-v24.19.0-linux-x64.tar.xz\n${hash} node-v24.19.0-linux-arm64.tar.xz\n` + : archive, + ); + }); + const harness = createCliHarness("pi", { version: "1.2.3" }); + const cacheDir = join(root, "cache"); + const bundle = await harness.prepare({ model: "deepseek/deepseek-v4", cacheDir }); + assert.equal(fetches, 4); + const sameBundle = await harness.prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(sameBundle.dir, bundle.dir); + assert.equal(bundle.provenance?.model, "deepseek/deepseek-v4"); + assert.equal(sameBundle.provenance?.model, "openai/gpt-5.6"); + const differentConfig = await createCliHarness("pi", { + version: "1.2.3", + reasoning: "high", + baseUrl: "https://gateway.example.test/v1", + }).prepare({ model: "openai/gpt-5.6", cacheDir }); + assert.equal(differentConfig.dir, bundle.dir); + assert.equal(differentConfig.provenance?.reasoning, "high"); + assert.equal(differentConfig.provenance?.baseUrl, "https://gateway.example.test/v1"); + assert.equal(fetches, 4); + const installs = (await readFile(calls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + assert.equal(installs.length, 2); + for (const [i, args] of installs.entries()) { + assert.ok(args.includes("--ignore-scripts")); + assert.ok(args.includes("--ignore-workspace")); + assert.ok(args.includes("--os=linux")); + assert.ok(args.includes(`--cpu=${i === 0 ? "x64" : "arm64"}`)); + assert.ok(!args.some((arg) => arg.includes("latest"))); + } + assert.equal(bundle.provenance?.installLocation, "host"); + assert.equal(bundle.provenance?.version, "1.2.3"); + assert.equal(bundle.provenance?.containerInstallMs, 0); + assert.match(JSON.stringify(bundle.provenance?.lockSha256), /[a-f0-9]{64}/); + assert.ok(!JSON.stringify(bundle.provenance).includes(FAKE_KEY)); + for (const arch of ["x64", "arm64"] as const) await assertRelocated(bundle, harness, root, arch); + + // A failed build must not publish a cache entry or retain partial build trees. + t.mock.method(globalThis, "fetch", async () => new Response("bad checksum")); + await assert.rejects( + createCliHarness("codex", { version: "1.2.3" }).prepare({ model: "openai/gpt-5.6", cacheDir }), + /checksum is missing/, + ); + assert.ok(!(await readdir(join(cacheDir, "cli"))).some((name) => name.startsWith(".prepare-"))); +}); + +async function assertRelocated( + bundle: HarnessBundle, + harness: Harness, + root: string, + arch: "x64" | "arm64", +) { + const installDir = join(root, `relocated ${arch}`); + await cp(bundle.dir, installDir, { recursive: true }); + await cp(bundle.arch![arch], join(installDir, "arch"), { recursive: true }); + const ctx = { + model: "openai/gpt-5.6", + installDir, + taskWorkdir: root, + instructionPath: join(root, `${arch}-instruction.md`), + logsDir: join(root, `${arch}-logs`), + }; + const result = await invoke(harness, ctx, "mock relocation"); + assert.equal(result.code, 0, result.stderr); + assert.match(await readFile(join(ctx.logsDir, "pi.jsonl"), "utf8"), /mock relocation/); + assert.equal(dirname(bundle.dir), dirname(bundle.arch![arch])); +} diff --git a/packages/eve-bench/test/scenarios/cli-report.test.ts b/packages/eve-bench/test/scenarios/cli-report.test.ts new file mode 100644 index 0000000000..27db41d22e --- /dev/null +++ b/packages/eve-bench/test/scenarios/cli-report.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { test, type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { summarize, type JobResult, type TrialResult } from "../../src/core/result.ts"; + +const exec = promisify(execFile); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +function trial(attempt: number, extra: Partial = {}): TrialResult { + return { + task: "a", + attempt, + harness: "pi@0.85.1", + model: "test/model", + reward: 0, + agent: { status: "completed", exitCode: 0, durationMs: 1 }, + verifier: { status: "completed", exitCode: 0, durationMs: 1 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + ...extra, + }; +} + +async function report(t: TestContext, trials: TrialResult[], ...args: string[]) { + const job = `cli-report-test-${randomUUID().slice(0, 8)}`; + const dir = join(packageRoot, ".generated", "jobs", job); + await mkdir(dir, { recursive: true }); + t.after(() => rm(dir, { recursive: true, force: true })); + const result: JobResult = { + job, + dataset: { name: "local", version: "0", commit: "" }, + harness: "pi@0.85.1", + model: "test/model", + trials, + summary: summarize(trials), + }; + await writeFile(join(dir, "result.json"), JSON.stringify(result)); + return exec(process.execPath, ["src/cli.ts", "report", job, ...args], { cwd: packageRoot }).then( + ({ stdout, stderr }) => ({ code: 0, stdout, stderr }), + (error: { code: number; stdout: string; stderr: string }) => error, + ); +} + +test("report --fail-on-invalid exits 1 and names each invalid attempt", async (t) => { + const result = await report( + t, + [ + trial(1), + trial(2, { invalid: { phase: "harness", reason: "harness reported zero token usage" } }), + ], + "--fail-on-invalid", + ); + assert.equal(result.code, 1); + assert.match(result.stdout, /a #2 \[harness\] harness reported zero token usage/); + assert.match(result.stderr, /1 of 2 attempt\(s\) are invalid/); +}); + +test("report --fail-on-invalid passes when every attempt was scored, including failures", async (t) => { + const result = await report(t, [trial(1), trial(2, { reward: 1 })], "--fail-on-invalid"); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /scored 2 {2}resolved 1/); +}); diff --git a/packages/eve-bench/test/scenarios/e0-container.test.ts b/packages/eve-bench/test/scenarios/e0-container.test.ts new file mode 100644 index 0000000000..28afd82ee1 --- /dev/null +++ b/packages/eve-bench/test/scenarios/e0-container.test.ts @@ -0,0 +1,539 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + lstat, + mkdir, + mkdtemp, + open, + readFile, + readdir, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import type { SandboxBackend, SandboxSession } from "eve/sandbox"; +import { containerBackend } from "../../src/harnesses/e0/container-backend.ts"; + +const MAX_BYTES = 16 * 1024 * 1024; +const exec = promisify(execFile); +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +const node = (script: string) => `${quote(process.execPath)} -e ${quote(script)}`; + +function env(t: TestContext, values: Record) { + const previous = Object.fromEntries(Object.keys(values).map((key) => [key, process.env[key]])); + const set = (entries: typeof values) => { + for (const [key, value] of Object.entries(entries)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }; + set(values); + t.after(() => set(previous)); +} + +async function fixture(t: TestContext, commandTimeoutMs = 3_000) { + const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-e0-"))); + const root = join(dir, "task with 'quote"); + const backend: SandboxBackend = containerBackend(root, { commandTimeoutMs }); + const input = { sessionKey: `test:${root}`, templateKey: null, runtimeContext: { appRoot: dir } }; + const handle = await backend.create(input); + t.after(async () => { + await handle.shutdown(); + await rm(dir, { recursive: true, force: true }); + }); + return { dir, root, backend, handle, session: handle.session, input }; +} + +async function collect(stream: ReadableStream) { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); +} + +async function ready(root: string) { + for (let i = 0; i < 100; i++) { + try { + return Number(await readFile(join(root, "descendant.pid"), "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await delay(10); + } + throw new Error("Descendant did not start"); +} + +async function stopped(pid: number) { + for (let i = 0; i < 100; i++) { + try { + const { stdout } = await exec("ps", ["-o", "stat=", "-p", String(pid)]); + // Linux init may reap a killed orphan later; a zombie cannot execute work. + if (stdout.trim().startsWith("Z") || !stdout.trim()) return; + } catch (error) { + if ((error as { code?: number }).code === 1) return; + throw error; + } + await delay(10); + } + assert.fail(`Process ${pid} is still running`); +} + +const descendant = "(sleep 0.5; printf leaked > escaped) & echo $! > descendant.pid; wait"; + +test("default root requires workdir and container guard; explicit roots support local tests", async (t) => { + const { root } = await fixture(t); + env(t, { EVE_BENCH_TASK_WORKDIR: undefined, EVE_BENCH_CONTAINER: "1" }); + assert.throws(() => containerBackend(), /EVE_BENCH_TASK_WORKDIR/); + process.env.EVE_BENCH_TASK_WORKDIR = root; + delete process.env.EVE_BENCH_CONTAINER; + assert.throws(() => containerBackend(), /EVE_BENCH_CONTAINER=1/); + assert.throws(() => containerBackend(undefined, {}), /EVE_BENCH_CONTAINER=1/); + assert.doesNotThrow(() => containerBackend(root)); + process.env.EVE_BENCH_CONTAINER = "1"; + assert.doesNotThrow(() => containerBackend()); + for (const invalid of ["", "relative"]) + assert.throws(() => containerBackend(invalid), /absolute task directory/); + for (const commandTimeoutMs of [0, -1, NaN, Infinity, 2_147_483_648]) + assert.throws(() => containerBackend(root, { commandTimeoutMs }), /commandTimeoutMs/); +}); + +test("workspace paths, cwd, actual roots and skill home work without symlinks or shell rewriting", async (t) => { + const { session, backend, dir, root } = await fixture(t); + const home = join(dir, "home"); + env(t, { HOME: home }); + await mkdir(home); + const prewarm = { + templateKey: "test", + runtimeContext: { appRoot: dir }, + seedFiles: [ + { path: "/workspace/nested/seed.txt", content: "seed\n" }, + { path: "$HOME/.agents/skills/test/SKILL.md", content: "skill\n" }, + { path: join(home, ".agents/skills/test/reference.md"), content: Buffer.from("reference\n") }, + ], + }; + assert.deepEqual(await backend.prewarm(prewarm), { reused: false }); + assert.equal(session.resolvePath("/workspace"), root); + assert.equal(session.resolvePath("/workspace/nested"), join(root, "nested")); + assert.equal(session.resolvePath("nested/seed.txt"), join(root, "nested/seed.txt")); + for (const escaped of [ + "/workspace-other", + "/proc/self/environ", + "/installed-agent", + "../../outside", + ]) { + assert.throws(() => session.resolvePath(escaped), /must stay inside/); + await assert.rejects(session.readTextFile({ path: escaped }), /must stay inside/); + } + assert.equal(await session.readTextFile({ path: "/workspace/nested/seed.txt" }), "seed\n"); + await symlink("/proc", join(root, "proc-link")); + await assert.rejects( + session.readTextFile({ path: "/workspace/proc-link/self/environ" }), + /do not follow symbolic links/, + ); + await rm(join(root, "proc-link")); + assert.equal( + await session.readTextFile({ path: "$HOME/.agents/skills/test/SKILL.md" }), + "skill\n", + ); + const shell = await session.run({ + command: 'printf "%s\\n" "$PWD"; cat seed.txt; cat "$HOME/.agents/skills/test/reference.md"', + workingDirectory: "/workspace/nested", + }); + assert.equal(shell.exitCode, 0, shell.stderr); + assert.equal(shell.stdout, `${root}/nested\nseed\nreference\n`); + const literal = await session.run({ command: "printf '%s' '/workspace/literal'" }); + assert.equal(literal.stdout, "/workspace/literal"); + const description = (session as SandboxSession & { description: string }).description; + assert.ok(description.includes(JSON.stringify(root))); + assert.match(description, /shell command text does not/); + assert.equal((await lstat(root)).isSymbolicLink(), false); + assert.deepEqual(await readdir(root), ["nested"]); + await assert.rejects( + backend.prewarm({ ...prewarm, bootstrap: () => assert.fail("must not bootstrap") }), + /Production sandbox bootstrap/, + ); + await assert.rejects( + backend.prewarm({ ...prewarm, seedFiles: [{ path: join(dir, "unexpected"), content: "no" }] }), + /must stay inside/, + ); +}); + +test("shell inherits only operational env, not provider/production secrets or startup hooks", async (t) => { + const { session, dir } = await fixture(t); + const startup = join(dir, "startup.sh"); + await writeFile(startup, "export INJECTED_SECRET=should-not-run\n"); + env(t, { + AI_GATEWAY_API_KEY: "fake-provider", + OPENAI_API_KEY: "fake-openai", + ANTHROPIC_API_KEY: "fake-anthropic", + D0_JUDGE_API_KEY: "fake-judge", + VERCEL_TOKEN: "fake-vercel", + AWS_SECRET_ACCESS_KEY: "fake-aws", + DATABASE_URL: "fake-database", + CUSTOM_PRODUCTION_VALUE: "fake-custom", + GITHUB_TOKEN: "fake-github", + BASH_ENV: startup, + ENV: startup, + NODE_OPTIONS: "--trace-warnings", + LANG: "C", + }); + const result = await session.run({ + command: node("console.log(JSON.stringify(process.env))"), + env: { EVE_CODE_DIAGNOSTICS_REQUEST: "explicit-tool-data" }, + }); + assert.equal(result.exitCode, 0, result.stderr); + const forwarded = JSON.parse(result.stdout) as Record; + for (const key of [ + "AI_GATEWAY_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "D0_JUDGE_API_KEY", + "VERCEL_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "DATABASE_URL", + "CUSTOM_PRODUCTION_VALUE", + "GITHUB_TOKEN", + "BASH_ENV", + "ENV", + "NODE_OPTIONS", + "INJECTED_SECRET", + ]) + assert.equal(forwarded[key], undefined, key); + assert.equal(forwarded.PATH, process.env.PATH); + assert.equal(forwarded.LANG, "C"); + assert.equal(forwarded.EVE_CODE_DIAGNOSTICS_REQUEST, "explicit-tool-data"); + const override = await session.run({ command: 'printf "%s" "$LANG"', env: { LANG: "explicit" } }); + assert.equal(override.stdout, "explicit"); +}); + +test("binary, text and streaming file operations preserve bytes, ranges, encodings and removal", async (t) => { + const { session } = await fixture(t); + for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) + assert.equal(await session[method]({ path: "missing" }), null); + await session.writeTextFile({ path: "/workspace/text", content: "one\n😀two\nthree\n" }); + assert.equal(await session.readTextFile({ path: "text", startLine: 2, endLine: 2 }), "😀two"); + assert.equal(await session.readTextFile({ path: "text", startLine: 3, endLine: 99 }), "three\n"); + await assert.rejects( + async () => session.readTextFile({ path: "text", startLine: 0 }), + /line range/, + ); + await assert.rejects( + async () => session.readTextFile({ path: "text", startLine: 3, endLine: 2 }), + /line range/, + ); + await session.writeTextFile({ path: "latin", content: "café", encoding: "latin1" }); + assert.equal(await session.readTextFile({ path: "latin", encoding: "latin1" }), "café"); + await assert.rejects(async () => session.readTextFile({ path: "latin" }), /encoded data/); + const bytes = Uint8Array.from([0, 255, 128, 1]); + await session.writeBinaryFile({ path: "binary", content: bytes }); + assert.deepEqual( + Buffer.from((await session.readBinaryFile({ path: "binary" }))!), + Buffer.from(bytes), + ); + const stream = await session.readFile({ path: "text" }); + assert.ok(stream); + await session.writeFile({ path: "copy/nested", content: stream }); + assert.equal(await session.readTextFile({ path: "copy/nested" }), "one\n😀two\nthree\n"); + await session.removePath({ path: "/workspace/copy", recursive: true }); + await session.removePath({ path: "missing", force: true }); + assert.equal(await session.readFile({ path: "copy/nested" }), null); +}); + +test("all file surfaces are bounded and oversized streamed writes leave existing files untouched", async (t) => { + const { session, root } = await fixture(t); + const file = await open(join(root, "large"), "w"); + await file.truncate(MAX_BYTES + 1); + await file.close(); + for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) + await assert.rejects( + async () => session[method]({ path: "large" }), + /File read exceeds 16 MiB/, + ); + await assert.rejects(async () => session.readBinaryFile({ path: "." }), /regular files/); + await session.writeTextFile({ path: "target", content: "unchanged" }); + await assert.rejects( + async () => session.writeBinaryFile({ path: "target", content: new Uint8Array(MAX_BYTES + 1) }), + /File write exceeds 16 MiB/, + ); + await assert.rejects( + async () => session.writeTextFile({ path: "target", content: "😀".repeat(MAX_BYTES / 4 + 1) }), + /File write exceeds 16 MiB/, + ); + let cancelled = false; + const content = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(1024 * 1024)); + }, + cancel() { + cancelled = true; + }, + }); + await assert.rejects( + async () => session.writeFile({ path: "target", content }), + /File write exceeds 16 MiB/, + ); + assert.equal(cancelled, true); + assert.equal(await session.readTextFile({ path: "target" }), "unchanged"); + await session.writeBinaryFile({ path: "boundary", content: new Uint8Array(MAX_BYTES) }); + assert.equal((await session.readBinaryFile({ path: "boundary" }))?.byteLength, MAX_BYTES); + assert.equal((await session.run({ command: "mkfifo pipe" })).exitCode, 0); + await assert.rejects(async () => session.readBinaryFile({ path: "pipe" }), /regular files/); + await assert.rejects( + async () => session.writeTextFile({ path: "pipe", content: "no" }), + /ENXIO|regular files/, + ); + await assert.rejects( + async () => session.writeTextFile({ path: "/dev/null", content: "no" }), + /must stay inside/, + ); +}); + +test("aborted file operations and a blocked stream reject without writing", async (t) => { + const { session } = await fixture(t); + const controller = new AbortController(); + const reason = new Error("cancel file"); + const content = new ReadableStream(); + const pending = session.writeFile({ path: "never", content, abortSignal: controller.signal }); + controller.abort(reason); + await assert.rejects( + async () => pending, + (error) => error === reason, + ); + await assert.rejects( + async () => session.readFile({ path: "never", abortSignal: controller.signal }), + (error) => error === reason, + ); + await assert.rejects( + async () => + session.writeBinaryFile({ + path: "never", + content: new Uint8Array(), + abortSignal: controller.signal, + }), + (error) => error === reason, + ); + await assert.rejects( + session.removePath({ path: "never", abortSignal: controller.signal }), + (error) => error === reason, + ); + assert.equal(await session.readFile({ path: "never" }), null); +}); + +test("run and spawn preserve split UTF-8, stderr and native nonzero exits", async (t) => { + const { session } = await fixture(t); + const command = node( + "process.stdout.write(Buffer.from([0xf0, 0x9f])); setTimeout(() => { process.stdout.write(Buffer.from([0x98, 0x80])); process.stderr.write('err'); process.exitCode = 7; }, 20)", + ); + assert.deepEqual(await session.run({ command }), { stdout: "😀", stderr: "err", exitCode: 7 }); + const child = await session.spawn({ command }); + const [stdout, stderr, result] = await Promise.all([ + collect(child.stdout), + collect(child.stderr), + child.wait(), + ]); + assert.deepEqual({ stdout, stderr, ...result }, { stdout: "😀", stderr: "err", exitCode: 7 }); + await child.kill(); + await child.kill(); + await assert.rejects( + async () => session.run({ command: "true", workingDirectory: "absent" }), + /ENOENT/, + ); +}); + +for (const method of ["run", "spawn"] as const) { + test(`${method} timeout kills the entire process group`, async (t) => { + const { session, root } = await fixture(t, 200); + const operation = session[method]({ command: descendant }); + const pid = await ready(root); + if (method === "run") { + const result = (await operation) as Awaited>; + assert.equal(result.exitCode, 124); + assert.match(result.stderr, /process group was stopped/); + } else { + const child = (await operation) as Awaited>; + assert.equal((await child.wait()).exitCode, 124); + } + await stopped(pid); + await delay(550); + assert.equal(await session.readFile({ path: "escaped" }), null); + }); + test(`${method} abort kills descendants and rejects with the abort reason`, async (t) => { + const { session, root } = await fixture(t); + const controller = new AbortController(); + const reason = new Error("cancel command"); + const operation = session[method]({ command: descendant, abortSignal: controller.signal }); + const completion = + method === "run" + ? operation + : Promise.resolve(operation).then((child) => + (child as Awaited>).wait(), + ); + const rejection = assert.rejects( + async () => completion, + (error) => error === reason, + ); + const pid = await ready(root); + controller.abort(reason); + await rejection; + await stopped(pid); + await assert.rejects( + async () => session[method]({ command: "touch never", abortSignal: controller.signal }), + (error) => error === reason, + ); + assert.equal(await session.readFile({ path: "never" }), null); + }); + test(`${method} bounds combined stdout/stderr even when spawn output is not consumed`, async (t) => { + const { session } = await fixture(t); + const command = node( + "const chunk = Buffer.alloc(1024 * 1024, 'x'); setInterval(() => { process.stdout.write(chunk); process.stderr.write(chunk); }, 1)", + ); + if (method === "run") + await assert.rejects(async () => session.run({ command }), /Command output exceeds 16 MiB/); + else { + const child = await session.spawn({ command }); + await assert.rejects(async () => child.wait(), /Command output exceeds 16 MiB/); + await stopped(child.pid!); + } + }); +} + +test("stop/shutdown are idempotent, retain task files and allow reattachment; shell exit kills background work", async (t) => { + const { session, backend, handle, input, root } = await fixture(t); + await session.writeTextFile({ path: "keep", content: "task artifact" }); + for (const stop of [handle.stop, handle.shutdown]) { + const child = await session.spawn({ command: descendant }); + const pid = await ready(root); + await stop(); + await stop(); + await stopped(pid); + assert.equal((await child.wait()).exitCode, 137); + await session.removePath({ path: "descendant.pid" }); + } + assert.deepEqual(await handle.captureState(), { + backendName: backend.name, + metadata: {}, + sessionKey: input.sessionKey, + }); + const reopened = await backend.create(input); + assert.equal(reopened.session.id, session.id); + assert.equal((await reopened.useSessionFn()).id, session.id); + assert.equal(await reopened.session.readTextFile({ path: "keep" }), "task artifact"); + await session.run({ + command: "(sleep 0.5; printf leaked > escaped) >/dev/null 2>&1 & echo $! > descendant.pid", + }); + await stopped(await ready(root)); + await session.setNetworkPolicy("allow-all"); + await assert.rejects(session.setNetworkPolicy("deny-all"), /runner owns network policy/); +}); + +test("installed eve built-in bash/file tools work with workspace, actual root and home paths", async (t) => { + const { session, dir, root } = await fixture(t); + env(t, { HOME: join(dir, "home") }); + const installed = new URL("../../node_modules/eve/dist/src/", import.meta.url); + const { ContextContainer, contextStorage } = await import( + new URL("context/container.js", installed).href + ); + const { executeBashOnSandbox } = await import( + new URL("execution/sandbox/bash-tool.js", installed).href + ); + const { executeReadFileOnSandbox } = await import( + new URL("execution/sandbox/read-file-tool.js", installed).href + ); + const { executeWriteFileOnSandbox } = await import( + new URL("execution/sandbox/write-file-tool.js", installed).href + ); + const { executeGlobOnSandbox } = await import( + new URL("execution/sandbox/glob-tool.js", installed).href + ); + const { executeGrepOnSandbox } = await import( + new URL("execution/sandbox/grep-tool.js", installed).href + ); + await contextStorage.run(new ContextContainer(), async () => { + for (const filePath of ["/workspace/builtin.txt", "$HOME/.agents/skills/example/SKILL.md"]) { + assert.equal( + (await executeWriteFileOnSandbox(session, { filePath, content: "first\n" })).existed, + false, + ); + assert.equal((await executeReadFileOnSandbox(session, { filePath })).content, "1: first"); + assert.equal( + (await executeWriteFileOnSandbox(session, { filePath, content: "second\n" })).existed, + true, + ); + } + const result = await executeBashOnSandbox(session, { command: "cat builtin.txt" }); + assert.equal(result.exitCode, 0); + assert.equal(result.stdout, "second\n"); + const glob = await executeGlobOnSandbox(session, { path: root, pattern: "*.txt" }); + assert.equal(glob.count, 1); + assert.equal(glob.content, join(root, "builtin.txt")); + const grep = await executeGrepOnSandbox(session, { path: root, pattern: "second" }); + assert.equal(grep.matchCount, 1); + assert.ok(grep.content.includes("builtin.txt:1:second")); + }); +}); + +// Optional real extension check: supply the eve-code package path from the e0 +// source checkout. Run on Linux (GNU realpath/stat), with no model or network. +test( + "real e0 code__apply_patch adds, updates, moves and deletes files in the actual task root", + { + skip: !process.env.EVE_BENCH_EVE_CODE_DIR + ? "Set EVE_BENCH_EVE_CODE_DIR to the eve-code package" + : false, + }, + async (t) => { + const { session, root } = await fixture(t); + const { default: tool } = await import( + pathToFileURL(join(process.env.EVE_BENCH_EVE_CODE_DIR!, "extension/tools/apply_patch.ts")) + .href + ); + const initialized = await session.run({ command: "git init -q" }); + assert.equal(initialized.exitCode, 0, initialized.stderr); + const ctx = { getSandbox: async () => session }; + const added = await tool.execute( + { root, patchText: "*** Begin Patch\n*** Add File: hello.txt\n+before\n*** End Patch" }, + ctx, + ); + assert.equal(added.files[0].operation, "add"); + assert.deepEqual(added.diagnostics, []); + const updated = await tool.execute( + { + root, + patchText: + "*** Begin Patch\n*** Update File: hello.txt\n@@\n-before\n+after\n*** End Patch", + }, + ctx, + ); + assert.equal(updated.files[0].operation, "update"); + const moved = await tool.execute( + { + root, + patchText: + "*** Begin Patch\n*** Update File: hello.txt\n*** Move to: nested/moved.txt\n*** End Patch", + }, + ctx, + ); + assert.equal(moved.files[0].operation, "move"); + assert.equal(await session.readTextFile({ path: "/workspace/nested/moved.txt" }), "after\n"); + assert.equal(await session.readTextFile({ path: "hello.txt" }), null); + const deleted = await tool.execute( + { root, patchText: "*** Begin Patch\n*** Delete File: nested/moved.txt\n*** End Patch" }, + ctx, + ); + assert.equal(deleted.files[0].operation, "delete"); + assert.equal(await session.readTextFile({ path: "nested/moved.txt" }), null); + await assert.rejects( + tool.execute( + { root, patchText: "*** Begin Patch\n*** Add File: ../escape\n+no\n*** End Patch" }, + ctx, + ), + /invalid patch path/, + ); + }, +); diff --git a/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts b/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts new file mode 100644 index 0000000000..b7abe10b5f --- /dev/null +++ b/packages/eve-bench/test/scenarios/eve-agent-tools.test.ts @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import bash from "../../agent/tools/bash.ts"; +import grep from "../../agent/tools/grep.ts"; + +interface CommandResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +type Executable = { + execute: (input: never, ctx: { abortSignal: AbortSignal }) => Promise; +}; + +function run( + tool: unknown, + input: Record, + signal = new AbortController().signal, +): Promise { + return (tool as Executable).execute(input as never, { + abortSignal: signal, + }) as Promise; +} + +/** Points the tools at a fresh task directory and restores the process env afterwards. */ +async function taskEnvironment(t: TestContext, env: Record = {}): Promise { + const workdir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-tools-"))); + const saved = { ...process.env }; + Object.assign(process.env, { EVE_BENCH_TASK_WORKDIR: workdir, ...env }); + t.after(async () => { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]; + Object.assign(process.env, saved); + await rm(workdir, { recursive: true, force: true }); + }); + return workdir; +} + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test("bash runs in the task directory and passes exit codes and streams through", async (t) => { + const workdir = await taskEnvironment(t); + assert.deepEqual(await run(bash, { command: "pwd; echo err >&2; exit 7" }), { + exitCode: 7, + stdout: `${workdir}\n`, + stderr: "err\n", + }); +}); + +test("bash commands see the task's own PORT and HOST, not the eve server's", async (t) => { + // The runner overrides these for the server and records the container's values. + await taskEnvironment(t, { + PORT: "3024", + HOST: "127.0.0.1", + NITRO_PORT: "3024", + EVE_BENCH_TASK_ENV_RESTORE: JSON.stringify({ PORT: null, HOST: "task-host", NITRO_PORT: null }), + }); + const result = await run(bash, { + command: + 'echo "PORT=${PORT-unset} HOST=$HOST NITRO_PORT=${NITRO_PORT-unset} RESTORE=${EVE_BENCH_TASK_ENV_RESTORE-unset}"', + }); + assert.equal(result.stdout, "PORT=unset HOST=task-host NITRO_PORT=unset RESTORE=unset\n"); +}); + +test("bash timeout returns 124 with partial output and kills the whole process group", async (t) => { + await taskEnvironment(t); + const started = Date.now(); + const result = await run(bash, { + command: "sleep 30 & echo child=$!; sleep 30", + timeoutSeconds: 1, + }); + assert.ok(Date.now() - started < 5_000, `took ${Date.now() - started}ms`); + assert.equal(result.exitCode, 124); + assert.match(result.stderr, /stopped after 1 seconds/); + const child = Number(/child=(\d+)/.exec(result.stdout)?.[1]); + assert.ok(child > 0, result.stdout); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal(alive(child), false, "background child survived the timeout"); +}); + +test("bash returns when the shell exits even if a background job holds the output pipe", async (t) => { + await taskEnvironment(t); + const started = Date.now(); + const result = await run(bash, { command: "sleep 5 & echo started" }); + assert.ok(Date.now() - started < 2_000, `took ${Date.now() - started}ms`); + assert.deepEqual(result, { exitCode: 0, stdout: "started\n", stderr: "" }); +}); + +test("bash abort stops the command", async (t) => { + await taskEnvironment(t); + const controller = new AbortController(); + const started = Date.now(); + const pending = run(bash, { command: "sleep 30" }, controller.signal); + setTimeout(() => controller.abort(), 100); + const result = await pending; + assert.ok(Date.now() - started < 5_000, `took ${Date.now() - started}ms`); + assert.notEqual(result.exitCode, 0); +}); + +test("grep reports a missing ripgrep instead of returning no matches", async (t) => { + await taskEnvironment(t, { PATH: "/nonexistent" }); + await assert.rejects(run(grep, { pattern: "needle" }), /ripgrep \(rg\) is not installed/); +}); + +let hasRipgrep = true; +try { + execFileSync("rg", ["--version"], { stdio: "ignore" }); +} catch { + hasRipgrep = false; +} + +test( + "grep distinguishes matches from no matches", + { skip: !hasRipgrep && "rg not installed" }, + async (t) => { + const workdir = await taskEnvironment(t); + await writeFile(join(workdir, "notes.txt"), "alpha\nneedle here\n"); + const found = await run(grep, { pattern: "needle" }); + assert.equal(found.exitCode, 0); + assert.match(found.stdout, /notes\.txt:2:needle here/); + assert.deepEqual(await run(grep, { pattern: "absent" }), { + exitCode: 1, + stderr: "", + stdout: "", + }); + }, +); diff --git a/packages/eve-bench/test/scenarios/job-resume.test.ts b/packages/eve-bench/test/scenarios/job-resume.test.ts new file mode 100644 index 0000000000..bfefcba339 --- /dev/null +++ b/packages/eve-bench/test/scenarios/job-resume.test.ts @@ -0,0 +1,364 @@ +import assert from "node:assert/strict"; +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + symlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import { runJob, type JobInput } from "../../src/core/job.ts"; +import { hashTask, identityHash } from "../../src/core/job-identity.ts"; +import { loadTask } from "../../src/core/task.ts"; +import type { TrialResult } from "../../src/core/result.ts"; + +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), "eve-bench-resume-")); + const bin = join(root, "bin"); + const calls = join(root, "docker-calls.jsonl"); + const containers = join(root, "containers.json"); + const taskDir = join(root, "tasks", "a"); + await mkdir(bin); + for (const dir of ["environment", "tests", "solution"]) + await mkdir(join(taskDir, dir), { recursive: true }); + await writeFile(join(taskDir, "instruction.md"), "Do the task.\n"); + await writeFile(join(taskDir, "task.toml"), '[task]\nname = "a"\n'); + await writeFile(join(taskDir, "environment", "Dockerfile"), "FROM scratch\n"); + await writeFile(join(taskDir, "environment", "input.bin"), Buffer.from([0, 255, 1])); + await writeFile(join(taskDir, "tests", "test.sh"), "true\n"); + await writeFile(join(taskDir, "solution", "solve.sh"), "true\n"); + await writeFile(calls, ""); + await writeFile(containers, "[]"); + // No daemon or model is used: every Docker invocation terminates in this local stub. + await writeFile( + join(bin, "docker"), + `#!${process.execPath} +import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; +const args = process.argv.slice(2); +appendFileSync(${JSON.stringify(calls)}, JSON.stringify(args) + '\\n'); +if (args[0] === 'version') console.log('fake-docker'); +else if (args[0] === 'ps') for (const row of JSON.parse(readFileSync(${JSON.stringify(containers)}, 'utf8'))) console.log(JSON.stringify(row)); +else if (args[0] === 'rm') writeFileSync(${JSON.stringify(containers)}, '[]'); +else if (args[0] === 'exec') console.log('/workspace'); +`, + { mode: 0o755 }, + ); + const previousPath = process.env.PATH; + process.env.PATH = `${bin}:${previousPath}`; + t.after(async () => { + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + await rm(root, { recursive: true, force: true }); + }); + const input: JobInput = { + name: basename(root), + dir: join(root, "job"), + cacheDir: join(root, "cache"), + dataset: { + name: "bench", + version: "2", + gitUrl: "https://example.com/tasks", + commit: "abc", + tasks: ["a"], + }, + tasks: [await loadTask(taskDir)], + harness: { + name: "fake", + prepare: async () => ({ + dir: bin, + provenance: { key: "v1", source: { sha: "abc", dirty: false } }, + }), + command: () => "true", + env: () => ({}), + }, + model: "no-model", + attempts: 1, + concurrency: 1, + forwardEnv: {}, + signal: new AbortController().signal, + }; + const trial: TrialResult = { + task: "a", + attempt: 1, + harness: "fake", + model: "no-model", + reward: 1, + agent: { status: "completed", exitCode: 0, durationMs: 1 }, + verifier: { status: "completed", exitCode: 0, durationMs: 1 }, + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + }; + async function initialize() { + await runJob({ ...input, signal: AbortSignal.abort() }); + await mkdir(join(input.dir, "a", "1"), { recursive: true }); + await writeFile(join(input.dir, "a", "1", "trial.json"), JSON.stringify(trial)); + await writeFile(calls, ""); + } + async function leftovers() { + await writeFile( + containers, + JSON.stringify([ + { + ID: "leftover", + Names: "leftover", + Labels: `eve-bench=1,eve-bench.job=${input.name}`, + CreatedAt: "yesterday", + }, + ]), + ); + } + return { root, input, taskDir, calls, containers, trial, initialize, leftovers }; +} + +async function snapshot(dir: string) { + return Promise.all( + ["job.json", "result.json", "a/1/trial.json"].map((file) => readFile(join(dir, file), "utf8")), + ); +} + +test("every resume identity mismatch rejects before metadata/results overwrite or Docker cleanup", async (t) => { + const f = await fixture(t); + await f.initialize(); + await f.leftovers(); + const before = await snapshot(f.input.dir); + const changes: Partial[] = [ + { model: "different" }, + { attempts: 2 }, + { harness: { ...f.input.harness, name: "other" } }, + { + harness: { + ...f.input.harness, + prepare: async () => ({ dir: f.root, provenance: { key: "v2" } }), + }, + }, + { tasks: [{ ...f.input.tasks[0]!, name: "renamed" }] }, + { tasks: [] }, + { tasks: [...f.input.tasks, { ...f.input.tasks[0]!, name: "added" }] }, + { tasks: [{ ...f.input.tasks[0]!, instruction: "Loaded instruction override" }] }, + { tasks: [{ ...f.input.tasks[0]!, agentTimeoutMs: 123 }] }, + ...(["name", "version", "gitUrl", "commit"] as const).map((key) => ({ + dataset: { ...f.input.dataset, [key]: "changed" }, + })), + ]; + for (const change of changes) { + await assert.rejects( + runJob({ ...f.input, ...change }), + /resume identity differs.*new job name/, + ); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + } + for (const path of [ + "instruction.md", + "task.toml", + "tests/test.sh", + "environment/Dockerfile", + "environment/input.bin", + "solution/solve.sh", + ]) { + const file = join(f.taskDir, path); + const content = await readFile(file); + await writeFile(file, Buffer.concat([content, Buffer.from("changed")])); + await assert.rejects(runJob(f.input), /differs \(tasks\).*new job name/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + await writeFile(file, content); + } +}); + +test("matching resume keeps original metadata and cached trials, and then cleans leftovers", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + await f.leftovers(); + const seen: boolean[] = []; + const result = await runJob({ + ...f.input, + concurrency: 4, + forwardEnv: { FAKE_KEY: "not-identity-or-metadata" }, + harness: { + ...f.input.harness, + prepare: async () => ({ + dir: f.root, + provenance: { source: { dirty: false, sha: "abc" }, key: "v1" }, + }), + }, + onTrial: (_trial, resumed) => seen.push(resumed), + }); + assert.deepEqual(result.trials, [f.trial]); + assert.deepEqual(seen, [true]); + const after = await snapshot(f.input.dir); + assert.equal(after[0], before[0]); + assert.equal(after[2], before[2]); + assert.match(await readFile(f.calls, "utf8"), /\["rm","--force","--volumes","leftover"\]/); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["run"/); + assert.equal(await readFile(f.containers, "utf8"), "[]"); +}); + +test("legacy, malformed, unsupported and orphaned jobs refuse without preparation or Docker calls", async (t) => { + const f = await fixture(t); + await mkdir(f.input.dir); + let prepared = false; + const input = { + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + prepared = true; + throw new Error("must not prepare"); + }, + }, + }; + for (const text of [ + JSON.stringify({ job: f.input.name, harness: "fake" }), + "{invalid", + "null", + JSON.stringify({ identity: { version: 2 } }), + ]) { + await writeFile(join(input.dir, "job.json"), text); + await assert.rejects(runJob(input), /new job name/); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), text); + } + await rm(join(input.dir, "job.json")); + await writeFile(join(input.dir, "result.json"), "orphaned"); + await assert.rejects(runJob(input), /no job.json.*new job name/); + assert.equal(await readFile(join(input.dir, "result.json"), "utf8"), "orphaned"); + assert.equal(prepared, false); + assert.equal(await readFile(f.calls, "utf8"), ""); +}); + +test("task hashes are relocatable and ignore unselected trees and mtimes, but include paths and executable bits", async (t) => { + const f = await fixture(t); + const original = await hashTask(f.input.tasks[0]!); + const relocated = join(f.root, "relocated"); + await cp(f.taskDir, relocated, { recursive: true }); + const task = await loadTask(relocated); + assert.equal(await hashTask(task), original); + await utimes(join(relocated, "instruction.md"), new Date(0), new Date(0)); + await writeFile(join(f.root, "tasks", "unselected"), "does not affect a"); + assert.equal(await hashTask(task), original); + await rename( + join(relocated, "environment", "input.bin"), + join(relocated, "environment", "renamed.bin"), + ); + assert.notEqual(await hashTask(task), original); + await rename( + join(relocated, "environment", "renamed.bin"), + join(relocated, "environment", "input.bin"), + ); + await chmod(join(relocated, "solution", "solve.sh"), 0o755); + assert.notEqual(await hashTask(task), original); + await chmod(join(relocated, "solution", "solve.sh"), 0o644); + assert.equal(await hashTask(task), original); + await rm(join(relocated, "environment", "input.bin")); + assert.notEqual(await hashTask(task), original); +}); + +test("task hashing refuses symlinks and environments outside the selected tree", async (t) => { + const f = await fixture(t); + await symlink(f.calls, join(f.taskDir, "external")); + await assert.rejects(hashTask(f.input.tasks[0]!), /regular files\/directories/); + await rm(join(f.taskDir, "external")); + const task = f.input.tasks[0]!; + await assert.rejects( + hashTask({ ...task, environment: { ...task.environment, dockerfileDir: f.root } }), + /inside the selected task tree/, + ); +}); + +test("an aborted job retains its identity, releases the lock, and resumes the same remaining work", async (t) => { + const f = await fixture(t); + const controller = new AbortController(); + const input = { ...f.input, attempts: 2 }; + const partial = await runJob({ + ...input, + signal: controller.signal, + onTrial: () => controller.abort(), + }); + assert.equal(partial.trials.length, 1); + const metadata = await readFile(join(input.dir, "job.json"), "utf8"); + const seen: boolean[] = []; + const result = await runJob({ ...input, onTrial: (_trial, resumed) => seen.push(resumed) }); + assert.deepEqual(seen, [true, false]); + assert.equal(result.trials.length, 2); + assert.deepEqual(result.trials[0], partial.trials[0]); + assert.equal(await readFile(join(input.dir, "job.json"), "utf8"), metadata); +}); + +test("same-job lock rejects concurrent runs even in another output directory, and releases on failure", async (t) => { + const f = await fixture(t); + await f.initialize(); + const before = await snapshot(f.input.dir); + let enter!: () => void; + let release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + const active = runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + enter(); + await released; + return f.input.harness.prepare({ model: f.input.model, cacheDir: f.input.cacheDir }); + }, + }, + }); + try { + await entered; + await assert.rejects(runJob(f.input), /is locked.*active run/); + const otherDir = join(f.root, "other-job-dir"); + await assert.rejects(runJob({ ...f.input, dir: otherDir }), /is locked/); + assert.deepEqual(await snapshot(f.input.dir), before); + assert.equal(await readFile(f.calls, "utf8"), ""); + assert.ok(!(await readdir(f.root)).includes("other-job-dir")); + } finally { + release(); + await active; + } + await assert.rejects( + runJob({ + ...f.input, + harness: { + ...f.input.harness, + prepare: async () => { + throw new Error("prepare failed"); + }, + }, + }), + /prepare failed/, + ); + await runJob(f.input); +}); + +test("stale locks fail closed with explicit recovery, and unknown containers are not removed", async (t) => { + const f = await fixture(t); + const lock = join(tmpdir(), `eve-bench-job-${identityHash(f.input.name)}.lock`); + await writeFile(lock, "crashed owner", { flag: "wx" }); + try { + await assert.rejects(runJob(f.input), /confirm it has stopped before removing/); + assert.equal(await readFile(lock, "utf8"), "crashed owner"); + assert.equal(await readFile(f.calls, "utf8"), ""); + } finally { + await rm(lock); + } + await f.leftovers(); + const containers = await readFile(f.containers, "utf8"); + await assert.rejects(runJob(f.input), /containers but no resume identity.*new job name/); + assert.equal(await readFile(f.containers, "utf8"), containers); + assert.deepEqual(await readdir(f.input.dir), []); + assert.doesNotMatch(await readFile(f.calls, "utf8"), /\["rm"/); +}); diff --git a/packages/eve-bench/test/scenarios/trial-validity.test.ts b/packages/eve-bench/test/scenarios/trial-validity.test.ts new file mode 100644 index 0000000000..553478b69c --- /dev/null +++ b/packages/eve-bench/test/scenarios/trial-validity.test.ts @@ -0,0 +1,208 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test, type TestContext } from "node:test"; + +import type { Harness } from "../../src/core/harness.ts"; +import { runJob } from "../../src/core/job.ts"; +import type { JobResult, TrialResult } from "../../src/core/result.ts"; +import { loadTask } from "../../src/core/task.ts"; +import { readCliUsage } from "../../src/harnesses/cli/usage.ts"; + +/** Files the harness and verifier leave in the container; null leaves the directory absent. */ +interface ContainerLogs { + readonly agent: Readonly> | null; + readonly verifier: Readonly> | null; +} + +interface Scenario { + readonly logs: ContainerLogs; + readonly buildFails?: boolean; + readonly agentTimeoutSec?: number; + readonly agentRunsMs?: number; + readonly modelFree?: boolean; +} + +// pi --mode json records one assistant message_end per model call; a rejected key records zero usage. +const piCall = (input: number, output: number) => + `${JSON.stringify({ + type: "message_end", + message: { role: "assistant", usage: { input, output, cacheRead: 0, cost: { total: 0 } } }, + })}\n`; +const reward = (value: string) => ({ "reward.txt": `${value}\n` }); + +async function runScenario(t: TestContext, scenario: Scenario) { + const root = await mkdtemp(join(tmpdir(), "eve-bench-validity-")); + const bin = join(root, "bin"); + const fs = join(root, "container"); + const taskDir = join(root, "tasks", "a"); + await mkdir(bin); + for (const dir of ["environment", "tests"]) await mkdir(join(taskDir, dir), { recursive: true }); + await writeFile(join(taskDir, "instruction.md"), "Do the task.\n"); + await writeFile( + join(taskDir, "task.toml"), + `[task]\nname = "a"\n${scenario.agentTimeoutSec ? `[agent]\ntimeout_sec = ${scenario.agentTimeoutSec}\n` : ""}`, + ); + await writeFile(join(taskDir, "environment", "Dockerfile"), "FROM scratch\n"); + await writeFile(join(taskDir, "tests", "test.sh"), "true\n"); + for (const [dir, files] of Object.entries(scenario.logs)) { + if (files === null) continue; + await mkdir(join(fs, "logs", dir), { recursive: true }); + for (const [name, content] of Object.entries(files)) + await writeFile(join(fs, "logs", dir, name), content); + } + // No daemon or model: copying out of the container serves the scripted log tree. + await writeFile( + join(bin, "docker"), + `#!${process.execPath} +import { cpSync, existsSync } from 'node:fs'; +import { basename, join } from 'node:path'; +const args = process.argv.slice(2); +const fail = (message) => { process.stderr.write(message + '\\n'); process.exit(1); }; +switch (args[0]) { + case 'version': console.log('fake-docker'); break; + case 'image': process.exit(1); + case 'build': if (${scenario.buildFails === true}) fail('fake build failed'); break; + case 'exec': { + const command = args.at(-1); + if (command === 'pwd') console.log('/workspace'); + if (command === 'run-agent') await new Promise((r) => setTimeout(r, ${scenario.agentRunsMs ?? 0})); + break; + } + case 'cp': { + const [source, target] = args.slice(1); + const match = /^[^/][^:]*:(\\/.*)$/.exec(source); + if (!match) break; + const from = join(${JSON.stringify(fs)}, match[1]); + if (!existsSync(from)) fail('No such container path: ' + match[1]); + cpSync(from, join(target, basename(from)), { recursive: true }); + break; + } +} +`, + { mode: 0o755 }, + ); + const previousPath = process.env.PATH; + process.env.PATH = `${bin}:${previousPath}`; + t.after(async () => { + process.env.PATH = previousPath; + await rm(root, { recursive: true, force: true }); + }); + const harness: Harness = { + name: "fake", + modelFree: scenario.modelFree === true, + prepare: async () => ({ dir: bin, provenance: { key: "v1" } }), + command: () => "run-agent", + env: () => ({}), + readUsage: (dir) => readCliUsage("pi", dir), + }; + const dir = join(root, "job"); + const result: JobResult = await runJob({ + name: basename(root), + dir, + cacheDir: join(root, "cache"), + dataset: { name: "local", version: "0", gitUrl: "", commit: "", tasks: ["a"] }, + tasks: [await loadTask(taskDir)], + harness, + model: "test/model", + attempts: 1, + concurrency: 1, + forwardEnv: {}, + signal: new AbortController().signal, + }); + const saved = JSON.parse( + await readFile(join(dir, "a", "1", "trial.json"), "utf8"), + ) as TrialResult; + assert.deepEqual(saved, result.trials[0], "trial.json must record the classification"); + return { trial: saved, summary: result.summary }; +} + +test("a trial with a completed model call and a reward is scored", async (t) => { + const { trial, summary } = await runScenario(t, { + logs: { agent: { "pi.jsonl": piCall(100, 20) + piCall(50, 5) }, verifier: reward("1") }, + }); + assert.equal(trial.invalid, undefined); + assert.deepEqual(trial.usage, { + inputTokens: 150, + outputTokens: 25, + cachedTokens: 0, + costUsd: null, + }); + assert.deepEqual(summary, { + tasks: 1, + attempts: 1, + scored: 1, + invalid: { environment: 0, harness: 0, verifier: 0 }, + resolved: 1, + meanReward: 1, + }); +}); + +test("a harness whose model calls all failed is invalid, not a task failure", async (t) => { + const { trial, summary } = await runScenario(t, { + logs: { agent: { "pi.jsonl": `not json\n${piCall(0, 0)}` }, verifier: reward("0") }, + }); + assert.equal(trial.invalid?.phase, "harness"); + assert.match(trial.invalid!.reason, /zero token usage/); + assert.deepEqual(summary.invalid, { environment: 0, harness: 1, verifier: 0 }); + assert.equal(summary.scored, 0); + assert.equal(summary.meanReward, null); +}); + +test("a harness that logged no usage is invalid even when the verifier passes", async (t) => { + const { trial } = await runScenario(t, { logs: { agent: {}, verifier: reward("1") } }); + assert.equal(trial.reward, 1); + assert.equal(trial.invalid?.phase, "harness"); + assert.match(trial.invalid!.reason, /no model usage/); +}); + +test("a model-free harness needs no usage to be scored", async (t) => { + const { trial } = await runScenario(t, { + modelFree: true, + logs: { agent: {}, verifier: reward("1") }, + }); + assert.equal(trial.invalid, undefined); +}); + +test("an agent timeout after model calls is scored against the harness", async (t) => { + const { trial, summary } = await runScenario(t, { + agentTimeoutSec: 0.3, + agentRunsMs: 5_000, + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: reward("0") }, + }); + assert.equal(trial.agent.status, "timeout"); + assert.equal(trial.invalid, undefined); + assert.equal(trial.usage?.inputTokens, 10); + assert.equal(summary.scored, 1); + assert.equal(summary.meanReward, 0); +}); + +test("a missing reward is a verifier failure", async (t) => { + const { trial } = await runScenario(t, { + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: {} }, + }); + assert.equal(trial.reward, null); + assert.deepEqual(trial.invalid, { phase: "verifier", reason: "verifier wrote no reward" }); +}); + +test("undownloadable agent logs are an environment failure", async (t) => { + const { trial } = await runScenario(t, { logs: { agent: null, verifier: reward("1") } }); + assert.equal(trial.agent.status, "completed"); + assert.equal(trial.invalid?.phase, "environment"); + assert.match( + trial.invalid!.reason, + /after the harness started: could not download \/logs\/agent/, + ); +}); + +test("an image build failure is an environment failure before the harness starts", async (t) => { + const { trial, summary } = await runScenario(t, { + buildFails: true, + logs: { agent: { "pi.jsonl": piCall(10, 1) }, verifier: reward("1") }, + }); + assert.equal(trial.agent.status, "skipped"); + assert.equal(trial.invalid?.phase, "environment"); + assert.match(trial.invalid!.reason, /before the harness started: .*fake build failed/); + assert.deepEqual(summary.invalid, { environment: 1, harness: 0, verifier: 0 }); +}); diff --git a/packages/eve-bench/test/stress/stress.test.ts b/packages/eve-bench/test/stress/stress.test.ts index 85977542bd..e18b3b48af 100644 --- a/packages/eve-bench/test/stress/stress.test.ts +++ b/packages/eve-bench/test/stress/stress.test.ts @@ -20,11 +20,13 @@ interface TrialResult { agent: StepResult; verifier: StepResult; error?: string; + invalid?: { phase: "environment" | "harness" | "verifier"; reason: string }; } interface JobResult { dataset: { name: string; version: string; commit: string }; trials: TrialResult[]; + summary: { scored: number; invalid: Record; resolved: number }; } interface CliResult { @@ -154,7 +156,11 @@ test("happy: records completed steps and reward 1", { timeout }, async () => { assert.equal(trial.reward, 1); assert.equal(trial.agent.status, "completed"); assert.equal(trial.verifier.status, "completed"); - assert.deepEqual((await readResult(job)).dataset, { name: "local", version: "0", commit: "" }); + assert.equal(trial.invalid, undefined); + const saved = await readResult(job); + assert.deepEqual(saved.dataset, { name: "local", version: "0", commit: "" }); + assert.equal(saved.summary.scored, 1); + assert.equal(saved.summary.resolved, 1); }); test("agent-timeout: verifier still runs and the container is removed", { timeout }, async () => { @@ -167,6 +173,8 @@ test("agent-timeout: verifier still runs and the container is removed", { timeou assert.equal(trial.agent.status, "timeout"); assert.equal(trial.verifier.status, "completed"); assert.equal(trial.reward, 0); + // A timeout is the harness's outcome, so it stays scored. + assert.equal(trial.invalid, undefined); assert.deepEqual(await dockerNames(job), []); }); @@ -180,6 +188,8 @@ test("verifier-timeout: records timeout and null reward", { timeout }, async () assert.equal(trial.agent.status, "completed"); assert.equal(trial.verifier.status, "timeout"); assert.equal(trial.reward, null); + assert.equal(trial.invalid?.phase, "verifier"); + assert.equal((await readResult(job)).summary.scored, 0); }); test( @@ -195,12 +205,14 @@ test( assert.ok(trial.error); assert.equal(trial.agent.status, "skipped"); assert.equal(trial.verifier.status, "skipped"); + assert.equal(trial.invalid?.phase, "environment"); + assert.equal((await readResult(job)).summary.invalid.environment, 1); assert.deepEqual(await dockerNames(job), []); }, ); test( - "abort: exits 130, writes a partial result, cleans up, and resumes with a new task set", + "abort: exits 130, cleans up, and rejects a different resume identity", { timeout }, async () => { const job = "stress-abort"; @@ -228,11 +240,9 @@ test( await waitFor(async () => (await dockerNames(job)).length === 0, 15_000, "abort cleanup"); const resumed = await runCli(job, [taskDir("abort-resume")], [], false); - assertPassed(resumed); - const result = await readResult(job); - assert.equal(result.trials.length, 1); - assert.equal(result.trials[0]?.task, "stress-abort-resume"); - assert.equal(result.trials[0]?.reward, 1); + assert.equal(resumed.code, 1); + assert.match(resumed.stderr, /resume identity differs/); + assert.deepEqual(await dockerNames(job), []); } finally { if (child.exitCode === null) child.kill("SIGKILL"); await cleanupContainers(job); From 983accf7333e38b098919b7443b30a4704d766ca Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:28:21 -0400 Subject: [PATCH 4/9] fix(eve-bench): remove the e0 harness The e0 harness was committed by mistake with the CLI harnesses. It targets the sandbox backend API that eve 0.64 replaced, so it cannot typecheck against the workspace eve. It is not part of eve-bench. Signed-off-by: Rui Conti --- packages/eve-bench/README.md | 23 +- packages/eve-bench/src/cli.ts | 5 +- .../src/harnesses/e0/container-backend.ts | 394 ------------- packages/eve-bench/src/harnesses/e0/index.ts | 252 -------- .../src/harnesses/e0/snapshot.test.ts | 46 -- .../eve-bench/src/harnesses/e0/snapshot.ts | 53 -- packages/eve-bench/src/options.ts | 6 +- packages/eve-bench/src/tools/run_bench.ts | 6 +- packages/eve-bench/src/tools/tools.test.ts | 6 +- .../test/scenarios/e0-container.test.ts | 539 ------------------ 10 files changed, 8 insertions(+), 1322 deletions(-) delete mode 100644 packages/eve-bench/src/harnesses/e0/container-backend.ts delete mode 100644 packages/eve-bench/src/harnesses/e0/index.ts delete mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.test.ts delete mode 100644 packages/eve-bench/src/harnesses/e0/snapshot.ts delete mode 100644 packages/eve-bench/test/scenarios/e0-container.test.ts diff --git a/packages/eve-bench/README.md b/packages/eve-bench/README.md index 6d1bec4eb5..048c72442c 100644 --- a/packages/eve-bench/README.md +++ b/packages/eve-bench/README.md @@ -28,15 +28,7 @@ AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attem pnpm bench run --harness oracle --cohort smoke # reference solutions; validates the runner, no model pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0 # benchmark a published eve release pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task - -# Build and cache a local e0 snapshot without calling a model. -pnpm bench prepare --harness e0 --agent /path/to/internal-agents/agents/e0 \ - --model openai/gpt-5.6-terra - -# Run one e0 smoke task. The source app must have installed workspace dependencies. -AI_GATEWAY_API_KEY=... pnpm bench run --harness e0 \ - --agent /path/to/internal-agents/agents/e0 \ - --model deepseek/deepseek-v4-pro --cohort smoke --task fix-git --job e0-deepseek +pnpm bench prepare --harness eve --model zai/glm-5.2 # build and cache a bundle without calling a model pnpm bench report --format junit --out junit.xml pnpm bench diff @@ -82,7 +74,7 @@ task. Resolve invalid attempts before comparing jobs; `pnpm bench report --fail-on-invalid` exits 1 when any remain, and CI applies it to every smoke job. Usage is read on the host from each harness's own event log (`events.ndjson` -for eve and e0, `.jsonl` for pi, OpenCode, and Codex), so it survives agent +for eve, `.jsonl` for pi, OpenCode, and Codex), so it survives agent timeouts. Input tokens include cache reads and output tokens include reasoning. `costUsd` is null when the harness reports no cost; the CLI competitors report none for custom Gateway providers, so compare tokens rather than cost across @@ -139,15 +131,6 @@ trials. `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass eve's nested sandbox on purpose. -- `e0`: snapshots an installed local e0 app and its source-backed `eve-code` - extension. The bundle records source, dependency, and installed eve hashes. - It preserves e0's instructions, coding capabilities, tools, skills, and - authored worker model. The adapter replaces production sandbox and transport - integrations with the task-container backend and a local eve channel. It - removes production Connect configuration, channels, connections, schedules, - instrumentation, and sandbox bootstrap. `--model` changes the root model, - not e0's authored worker model. - - `pi`, `opencode`, and `codex`: prepare explicitly versioned CLI bundles on the host and route full provider-qualified model IDs through an HTTPS API base. Pass an exact `--version`. The default base is Vercel AI Gateway. Codex uses @@ -192,7 +175,7 @@ diff the result against a downloaded base-job artifact. - `agent/`: harness behavior under evaluation (model binding, instructions, tools) - `src/core/`: runner library with no dependency on eve or third parties -- `src/harnesses/`: `eve`, `e0`, CLI competitors, and `oracle` +- `src/harnesses/`: `eve`, CLI competitors, and `oracle` - `src/tools/`: eve tool definitions over the runner core - `src/options.ts`: run options shared by the CLI and tools - `src/cli.ts`: `eve-bench` CLI diff --git a/packages/eve-bench/src/cli.ts b/packages/eve-bench/src/cli.ts index cfd1d5a178..b15c73de12 100644 --- a/packages/eve-bench/src/cli.ts +++ b/packages/eve-bench/src/cli.ts @@ -23,8 +23,8 @@ const USAGE = `eve-bench: zero-dependency Terminal-Bench runner for eve eve-bench tasks sync [--dataset ] fetch pinned datasets into .generated/datasets eve-bench tasks list [--cohort ] print task names eve-bench run --model [--cohort ] [--task ...] [--task-dir ...] [--attempts N] - [--concurrency N] [--harness eve|e0|oracle|pi|opencode|codex] [--job ] - [--eve local|] [--agent ] [--version ] + [--concurrency N] [--harness eve|oracle|pi|opencode|codex] [--job ] + [--eve local|] [--version ] [--reasoning ] [--base-url ] [--format console|json|junit] eve-bench prepare --harness --model [harness options] build/cache without model calls eve-bench report [--format console|json|junit] [--fail-on-invalid] @@ -99,7 +99,6 @@ async function run(args: string[], prepareOnly = false): Promise { concurrency: { type: "string", default: "4" }, eve: { type: "string", default: "local" }, harness: { type: "string", default: "eve" }, - agent: { type: "string" }, version: { type: "string" }, "base-url": { type: "string" }, reasoning: { type: "string" }, diff --git a/packages/eve-bench/src/harnesses/e0/container-backend.ts b/packages/eve-bench/src/harnesses/e0/container-backend.ts deleted file mode 100644 index 3f15a9c6ec..0000000000 --- a/packages/eve-bench/src/harnesses/e0/container-backend.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { spawn } from "node:child_process"; -import { constants } from "node:fs"; -import { lstat, mkdir, open, rm } from "node:fs/promises"; -import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; -import type { Readable } from "node:stream"; -import type { SandboxBackend, SandboxProcess, SandboxSession } from "eve/sandbox"; - -const MAX_BYTES = 16 * 1024 * 1024; -const ENV_KEYS = [ - "PATH", - "HOME", - "LANG", - "LC_ALL", - "LC_CTYPE", - "TMPDIR", - "TMP", - "TEMP", - "TZ", - "TERM", -]; - -function checkSize(bytes: number, operation: string): void { - if (bytes > MAX_BYTES) throw new Error(`${operation} exceeds 16 MiB`); -} - -/** - * Attach to the task container; the runner owns isolation, networking and deletion. - * Explicit roots permit local tests, not an isolated host sandbox. No symlinks are - * created. File paths and cwd map /workspace to root, but shell source is never - * rewritten: commands (including code__apply_patch's root) must use the actual - * task directory or relative paths, not literal /workspace paths in shell text. - * Built-in glob/grep embed paths in shell commands: pass the actual root as path - * instead of relying on their /workspace default. - */ -export function containerBackend( - root = process.env.EVE_BENCH_TASK_WORKDIR!, - { commandTimeoutMs = 120_000 }: { commandTimeoutMs?: number } = {}, -): SandboxBackend { - if (!root?.trim() || !isAbsolute(root)) - throw new Error( - "EVE_BENCH_TASK_WORKDIR must be an absolute task directory (or pass an explicit test root)", - ); - if (arguments[0] === undefined && process.env.EVE_BENCH_CONTAINER !== "1") - throw new Error("The e0 backend requires EVE_BENCH_CONTAINER=1 inside a task container"); - if ( - !Number.isFinite(commandTimeoutMs) || - commandTimeoutMs <= 0 || - commandTimeoutMs > 2_147_483_647 - ) - throw new Error("commandTimeoutMs must be positive and at most 2147483647"); - root = resolve(root); - const name = "e0-task-container"; - const children = new Set(); - const skillsHome = () => process.env.HOME && resolve(process.env.HOME, ".agents/skills"); - const within = (parent: string, path: string) => path === parent || path.startsWith(`${parent}/`); - const resolvePath = (path: string): string => { - let target: string; - if (path === "$HOME" || path.startsWith("$HOME/")) { - if (!process.env.HOME) throw new Error("HOME is unavailable in the task container"); - target = resolve(process.env.HOME, path.slice(6)); - } else if (path === "/workspace" || path.startsWith("/workspace/")) { - target = resolve(root, path === "/workspace" ? "." : path.slice(11)); - } else { - target = resolve(root, path); - } - const skills = skillsHome(); - if (!within(root, target) && !(skills && within(skills, target))) { - throw new Error("Sandbox paths must stay inside the task workspace or seeded skills"); - } - return target; - }; - const rejectSymlinkPath = async (target: string) => { - const base = within(root, target) ? root : skillsHome()!; - let current = base; - for (const component of relative(base, target).split(sep).filter(Boolean)) { - current = resolve(current, component); - try { - if ((await lstat(current)).isSymbolicLink()) { - throw new Error("Sandbox file operations do not follow symbolic links"); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return; - throw error; - } - } - }; - const stop = async () => { - await Promise.all([...children].map((child) => child.kill())); - }; - const session: SandboxSession & { readonly description: string } = { - id: `${name}:${root}`, - description: `Task container. Work in ${JSON.stringify(root)}; use this actual root for code__apply_patch. File operations and cwd map /workspace here, but shell command text does not. Use relative paths or the actual task directory in bash. Always pass the actual task directory as glob/grep's path; their /workspace default is not remapped. Skill files are under $HOME/.agents/skills.`, - resolvePath, - async spawn({ command, workingDirectory, env = {}, abortSignal }) { - abortSignal?.throwIfAborted(); - const shellEnv: NodeJS.ProcessEnv = {}; - for (const key of ENV_KEYS) { - if (process.env[key] !== undefined) shellEnv[key] = process.env[key]; - } - // Neither inherit credentials nor let login profiles reload production env. - const child = spawn("/bin/bash", ["--noprofile", "--norc", "-c", command], { - cwd: resolvePath(workingDirectory ?? "."), - env: { ...shellEnv, ...env }, - detached: true, - stdio: ["ignore", "pipe", "pipe"], - }); - let settled = false; - let killed = false; - let timedOut = false; - let failure: Error | undefined; - let bytes = 0; - const killGroup = () => { - if (settled || killed || child.pid === undefined) return; - try { - process.kill(-child.pid, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - killed = true; - }; - const abort = () => killGroup(); - abortSignal?.addEventListener("abort", abort, { once: true }); - const timer = setTimeout(() => { - timedOut = true; - killGroup(); - }, commandTimeoutMs); - const stream = (source: Readable) => { - let closed = false; - return new ReadableStream({ - start(controller) { - source.on("data", (chunk: Buffer) => { - bytes += chunk.byteLength; - if (bytes > MAX_BYTES && !failure) { - failure = new Error("Command output exceeds 16 MiB"); - killGroup(); - } - if (closed) return; - if (failure) { - closed = true; - controller.error(failure); - } else controller.enqueue(chunk); - }); - source.once("error", (error) => { - failure ??= error; - killGroup(); - if (!closed) { - closed = true; - controller.error(error); - } - }); - source.once("close", () => { - if (!closed) { - closed = true; - controller.close(); - } - }); - }, - cancel() { - closed = true; - killGroup(); - }, - }); - }; - const stdout = stream(child.stdout); - const stderr = stream(child.stderr); - // A shell can exit while a background descendant has redirected its pipes. - // Kill the group on exit as well as timeout, before dropping its PID. - child.once("exit", killGroup); - const completed = new Promise<{ exitCode: number }>((resolve, reject) => { - child.once("error", (error) => { - failure ??= error; - }); - child.once("close", (code) => { - settled = true; - clearTimeout(timer); - abortSignal?.removeEventListener("abort", abort); - children.delete(handle); - if (abortSignal?.aborted) reject(abortSignal.reason); - else if (failure) reject(failure); - else resolve({ exitCode: timedOut ? 124 : (code ?? 137) }); - }); - }); - // Stream consumers may wait only after draining output, or never wait at all. - void completed.catch(() => {}); - const handle: SandboxProcess = { - pid: child.pid, - stdout, - stderr, - wait: () => completed, - async kill() { - killGroup(); - await completed.catch(() => {}); - }, - }; - children.add(handle); - if (abortSignal?.aborted) killGroup(); - return handle; - }, - async run(input) { - const child = await session.spawn(input); - const collect = async (stream: ReadableStream) => { - const chunks: Uint8Array[] = []; - for await (const chunk of stream) chunks.push(chunk); - return Buffer.concat(chunks).toString("utf8"); - }; - try { - const [stdout, stderr, result] = await Promise.all([ - collect(child.stdout), - collect(child.stderr), - child.wait(), - ]); - return { - ...result, - stdout, - stderr: - result.exitCode === 124 - ? `${stderr}\nCommand ended with exit 124 (timeout limit ${commandTimeoutMs}ms); its process group was stopped.\n` - : stderr, - }; - } finally { - await child.kill(); - } - }, - async readBinaryFile({ path, abortSignal }) { - abortSignal?.throwIfAborted(); - let file; - try { - // Nonblocking open + regular-file check avoid hangs on FIFOs/devices. - const target = resolvePath(path); - await rejectSymlinkPath(target); - file = await open(target, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw error; - } - try { - const stat = await file.stat(); - if (!stat.isFile()) throw new Error("Only regular files can be read"); - checkSize(stat.size, "File read"); - const chunks: Uint8Array[] = []; - let bytes = 0; - for (;;) { - abortSignal?.throwIfAborted(); - const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_BYTES + 1 - bytes)); - const { bytesRead } = await file.read(buffer); - abortSignal?.throwIfAborted(); - if (bytesRead === 0) return Buffer.concat(chunks); - bytes += bytesRead; - checkSize(bytes, "File read"); - chunks.push(buffer.subarray(0, bytesRead)); - } - } finally { - await file.close(); - } - }, - async readFile(options) { - const bytes = await session.readBinaryFile(options); - return bytes === null - ? null - : new ReadableStream({ - start(controller) { - controller.enqueue(bytes); - controller.close(); - }, - }); - }, - async readTextFile({ encoding = "utf-8", startLine, endLine, ...options }) { - if ( - (startLine !== undefined && (!Number.isInteger(startLine) || startLine < 1)) || - (endLine !== undefined && (!Number.isInteger(endLine) || endLine < (startLine ?? 1))) - ) - throw new Error("Invalid 1-based line range"); - const bytes = await session.readBinaryFile(options); - if (bytes === null) return null; - const text = - encoding === "utf-8" || encoding === "utf8" - ? new TextDecoder("utf-8", { fatal: true }).decode(bytes) - : Buffer.from(bytes).toString(bufferEncoding(encoding)); - return startLine === undefined && endLine === undefined - ? text - : text - .split("\n") - .slice((startLine ?? 1) - 1, endLine) - .join("\n"); - }, - async writeBinaryFile({ path, content, abortSignal }) { - abortSignal?.throwIfAborted(); - checkSize(content.byteLength, "File write"); - const target = resolvePath(path); - await rejectSymlinkPath(target); - await mkdir(dirname(target), { recursive: true }); - await rejectSymlinkPath(target); - abortSignal?.throwIfAborted(); - const file = await open( - target, - constants.O_WRONLY | constants.O_CREAT | constants.O_NONBLOCK, - ); - try { - if (!(await file.stat()).isFile()) throw new Error("Only regular files can be written"); - abortSignal?.throwIfAborted(); - await file.truncate(0); - await file.writeFile(content, { signal: abortSignal }); - } finally { - await file.close(); - } - }, - async writeTextFile({ content, encoding = "utf8", ...options }) { - const codec = bufferEncoding(encoding); - checkSize(Buffer.byteLength(content, codec), "File write"); - await session.writeBinaryFile({ ...options, content: Buffer.from(content, codec) }); - }, - async writeFile({ content, ...options }) { - options.abortSignal?.throwIfAborted(); - const reader = content.getReader(); - const abort = () => { - void reader.cancel().catch(() => {}); - }; - options.abortSignal?.addEventListener("abort", abort, { once: true }); - const chunks: Uint8Array[] = []; - let bytes = 0; - try { - for (;;) { - options.abortSignal?.throwIfAborted(); - const next = await reader.read(); - options.abortSignal?.throwIfAborted(); - if (next.done) break; - bytes += next.value.byteLength; - checkSize(bytes, "File write"); - if (next.value.byteLength) chunks.push(Buffer.from(next.value)); - } - await session.writeBinaryFile({ ...options, content: Buffer.concat(chunks) }); - } finally { - options.abortSignal?.removeEventListener("abort", abort); - void reader.cancel().catch(() => {}); - reader.releaseLock(); - } - }, - async removePath({ path, recursive = false, force = false, abortSignal }) { - abortSignal?.throwIfAborted(); - const target = resolvePath(path); - await rejectSymlinkPath(target); - await rm(target, { recursive, force }); - }, - async setNetworkPolicy(policy) { - if (policy !== "allow-all") - throw new Error( - "The runner owns network policy; this backend cannot change the firewall or broker credentials", - ); - }, - }; - return { - name, - async prewarm({ seedFiles, bootstrap }) { - if (bootstrap) - throw new Error("Production sandbox bootstrap must not run in the task container"); - for (const file of seedFiles) { - const path = resolvePath(file.path); - const skills = skillsHome(); - if (!(path.startsWith(`${root}/`) || (skills && path.startsWith(`${skills}/`)))) - throw new Error("Unexpected seed path: use the task workspace or $HOME/.agents/skills"); - checkSize( - typeof file.content === "string" - ? Buffer.byteLength(file.content) - : file.content.byteLength, - "Seed file", - ); - } - await mkdir(root, { recursive: true }); - for (const file of seedFiles) { - await session.writeBinaryFile({ - path: file.path, - content: typeof file.content === "string" ? Buffer.from(file.content) : file.content, - }); - } - return { reused: false }; - }, - async create({ sessionKey }) { - await mkdir(root, { recursive: true }); - const liveSession = { ...session, id: sessionKey }; - return { - session: liveSession, - useSessionFn: async () => liveSession, - captureState: async () => ({ backendName: name, metadata: {}, sessionKey }), - stop, - shutdown: stop, - }; - }, - }; -} - -function bufferEncoding(encoding: string): BufferEncoding { - if (!Buffer.isEncoding(encoding)) throw new Error(`Unsupported encoding: ${encoding}`); - return encoding; -} diff --git a/packages/eve-bench/src/harnesses/e0/index.ts b/packages/eve-bench/src/harnesses/e0/index.ts deleted file mode 100644 index 412b3ec4ca..0000000000 --- a/packages/eve-bench/src/harnesses/e0/index.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { - cp, - mkdir, - mkdtemp, - readFile, - realpath, - rename, - rm, - stat, - writeFile, -} from "node:fs/promises"; -import { dirname, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; -import type { Harness, HarnessBundle, HarnessPrepareContext } from "../../core/harness.ts"; -import { ensureNode, NODE_VERSION } from "../eve/index.ts"; -import { readEveUsage } from "../eve/usage.ts"; -import { copySource, hashTree } from "./snapshot.ts"; - -const exec = promisify(execFile); -const here = dirname(fileURLToPath(import.meta.url)); -const REASONING = ["provider-default", "none", "minimal", "low", "medium", "high", "xhigh"]; - -export interface E0Options { - readonly agent: string; - readonly reasoning?: string; -} - -export function createE0Harness(options: E0Options): Harness { - if (!options.agent?.trim()) - throw new Error( - "--agent must point to the installed e0 application (containing agent/ and package.json)", - ); - if (options.reasoning && !REASONING.includes(options.reasoning)) - throw new Error(`Unsupported e0 reasoning: ${options.reasoning}`); - const agent = resolve(options.agent); - return { - name: "e0", - credentials: ["AI_GATEWAY_API_KEY"], - prepare: (ctx) => prepareE0({ ...options, agent }, ctx), - command: (ctx) => `exec '${ctx.installDir}/arch/node' '${ctx.installDir}/app/runner.mjs'`, - readUsage: readEveUsage, - env: (ctx) => { - const env: Record = { - EVE_PROJECT_DIR: `${ctx.installDir}/app`, - EVE_INSTRUCTION_PATH: ctx.instructionPath, - EVE_AGENT_LOGS_DIR: ctx.logsDir, - EVE_BENCH_CONTAINER: "1", - EVE_BENCH_TASK_WORKDIR: ctx.taskWorkdir, - E0_MODEL: ctx.model, - }; - if (options.reasoning) env.E0_REASONING = options.reasoning; - return env; - }, - }; -} - -export async function adaptE0(app: string): Promise { - const agent = join(app, "agent"); - // These integrations are outside the self-contained terminal workload. Authored - // coding behavior remains intact, including the extension's worker model. - for (const slot of ["sandbox", "channels", "schedules", "connections", "instrumentation"]) { - await rm(join(agent, slot), { recursive: true, force: true }); - await rm(join(agent, `${slot}.ts`), { force: true }); - } - await mkdir(join(agent, "lib"), { recursive: true }); - await cp(join(here, "container-backend.ts"), join(agent, "lib/eve-bench-backend.ts")); - await writeFile( - join(agent, "sandbox.ts"), - `import { defineSandbox } from "eve/sandbox";\nimport { containerBackend } from "./lib/eve-bench-backend.ts";\nexport default defineSandbox({ backend: () => containerBackend() });\n`, - ); - await mkdir(join(agent, "channels"), { recursive: true }); - await writeFile( - join(agent, "channels/eve.ts"), - `import { eveChannel } from "eve/channels/eve";\nimport { localDev } from "eve/channels/auth";\nexport default eveChannel({ auth: [localDev()] });\n`, - ); - await writeFile( - join(agent, "extensions/code.ts"), - `import code from "eve-code";\nexport default code({});\n`, - ); -} - -async function sourceIdentity(agent: string, extension: string) { - const pkg = JSON.parse(await readFile(join(agent, "package.json"), "utf8")) as { - dependencies: Record; - }; - const dependencies: Record = {}; - for (const name of Object.keys(pkg.dependencies).sort()) { - const installed = JSON.parse( - await readFile(join(agent, "node_modules", name, "package.json"), "utf8"), - ) as { version: string }; - dependencies[name] = installed.version; - } - const gitRoot = ( - await exec("git", ["rev-parse", "--show-toplevel"], { cwd: agent }) - ).stdout.trim(); - const gitSha = (await exec("git", ["rev-parse", "HEAD"], { cwd: agent })).stdout.trim(); - const scope = [ - relative(gitRoot, agent), - relative(gitRoot, extension), - "pnpm-lock.yaml", - "pnpm-workspace.yaml", - ]; - const conflicts = ( - await exec("git", ["diff", "--name-only", "--diff-filter=U", "--", ...scope], { cwd: gitRoot }) - ).stdout.trim(); - if (conflicts) throw new Error(`e0 source has unresolved merge conflicts:\n${conflicts}`); - const status = ( - await exec("git", ["status", "--porcelain", "--", ...scope], { cwd: gitRoot }) - ).stdout - .split("\n") - .filter((line) => !line.includes("/.eve-bench-")) - .join("\n"); - const metadata = createHash("sha256"); - for (const file of [ - join(agent, "package.json"), - join(agent, "tsconfig.json"), - join(extension, "package.json"), - join(extension, "tsconfig.json"), - join(gitRoot, "pnpm-lock.yaml"), - join(gitRoot, "pnpm-workspace.yaml"), - ]) { - metadata - .update(relative(gitRoot, file)) - .update("\0") - .update(await readFile(file)); - } - return { - gitSha, - dirty: status.trim().length > 0, - agentSha256: await hashTree(join(agent, "agent")), - extensionSha256: await hashTree(join(extension, "extension")), - metadataSha256: metadata.digest("hex"), - eveDistSha256: await hashTree(join(agent, "node_modules/eve/dist")), - dependencies, - }; -} - -async function prepareE0(options: E0Options, ctx: HarnessPrepareContext): Promise { - const agent = await realpath(options.agent); - let extension: string; - try { - extension = await realpath(join(agent, "node_modules/eve-code")); - await stat(join(extension, "extension/extension.ts")); - await stat(join(agent, "node_modules/.bin/eve")); - await stat(join(agent, "agent/extensions/code.ts")); - } catch { - throw new Error( - "e0 requires installed workspace dependencies and source-backed eve-code. Run pnpm install in internal-agents first.", - ); - } - const source = await sourceIdentity(agent, extension); - const adapter = createHash("sha256"); - for (const file of [ - "index.ts", - "snapshot.ts", - "container-backend.ts", - "../eve/index.ts", - "../eve/runner.mjs", - ]) - adapter.update(await readFile(join(here, file))); - const identity = { - source, - model: ctx.model, - reasoning: options.reasoning ?? "authored", - workerModel: "authored (not overridden)", - nodeVersion: NODE_VERSION, - adapterSha256: adapter.digest("hex"), - adaptation: - "Task-container sandbox; local eve channel; production extension configuration removed. No production channels, schedules, connections, instrumentation, sandbox bootstrap, or Connect configuration. Instructions, coding capabilities, tools, skills, and authored worker preserved.", - }; - const key = createHash("sha256").update(JSON.stringify(identity)).digest("hex"); - const cache = resolve(ctx.cacheDir); - const destination = join(cache, "e0", key); - const nodeDir = join(cache, `node-v${NODE_VERSION}`); - await Promise.all((["x64", "arm64"] as const).map((arch) => ensureNode(nodeDir, arch))); - const bundle = { - dir: destination, - arch: { x64: join(nodeDir, "x64"), arm64: join(nodeDir, "arm64") }, - provenance: { key, ...identity }, - }; - if (await exists(join(destination, "provenance.json"))) return bundle; - - // An adjacent disposable directory resolves the already-installed dependency - // graph without modifying source files, installing new versions, or adding links. - const scratch = await mkdtemp(join(agent, ".eve-bench-")); - const buildEnv: NodeJS.ProcessEnv = { - PATH: process.env.PATH, - HOME: process.env.HOME, - TMPDIR: process.env.TMPDIR, - CI: "true", - E0_MODEL: ctx.model, - }; - if (options.reasoning) buildEnv.E0_REASONING = options.reasoning; - try { - await copySource(join(agent, "agent"), join(scratch, "agent")); - for (const file of ["package.json", "tsconfig.json"]) - await cp(join(agent, file), join(scratch, file)); - const stagedExtension = join(scratch, "node_modules/eve-code"); - await mkdir(stagedExtension, { recursive: true }); - await copySource(join(extension, "extension"), join(stagedExtension, "extension")); - for (const file of ["package.json", "tsconfig.json"]) - await cp(join(extension, file), join(stagedExtension, file)); - const eve = join(agent, "node_modules/.bin/eve"); - await exec(eve, ["extension", "build"], { - cwd: stagedExtension, - env: buildEnv, - maxBuffer: 8 * 1024 * 1024, - }); - await adaptE0(scratch); - await exec(eve, ["build"], { cwd: scratch, env: buildEnv, maxBuffer: 16 * 1024 * 1024 }); - const after = await sourceIdentity(agent, extension); - if (JSON.stringify(after) !== JSON.stringify(source)) { - throw new Error( - "e0 source or dependencies changed during preparation; retry after edits settle", - ); - } - await mkdir(join(cache, "e0"), { recursive: true }); - const publishing = await mkdtemp(join(cache, "e0/.publish-")); - try { - await mkdir(join(publishing, "app")); - await cp(join(scratch, ".output"), join(publishing, "app/.output"), { - recursive: true, - dereference: true, - }); - await cp(join(here, "../eve/runner.mjs"), join(publishing, "app/runner.mjs")); - await writeFile( - join(publishing, "provenance.json"), - `${JSON.stringify(bundle.provenance, null, 2)}\n`, - ); - try { - await rename(publishing, destination); - } catch (error) { - if (!(await exists(join(destination, "provenance.json")))) throw error; - } - } finally { - await rm(publishing, { recursive: true, force: true }); - } - return bundle; - } finally { - await rm(scratch, { recursive: true, force: true }); - } -} - -async function exists(path: string): Promise { - return stat(path).then( - () => true, - () => false, - ); -} diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.test.ts b/packages/eve-bench/src/harnesses/e0/snapshot.test.ts deleted file mode 100644 index 01174b85b4..0000000000 --- a/packages/eve-bench/src/harnesses/e0/snapshot.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import assert from "node:assert/strict"; -import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import { copySource, hashTree, includeSource } from "./snapshot.ts"; - -test("source filter excludes generated and secret-bearing files", () => { - for (const path of [ - "node_modules", - ".git", - ".eve", - ".output", - ".next", - "dist", - ".env", - ".env.local", - ".eve-bench-work", - ]) { - assert.equal(includeSource(path), false, path); - } - assert.equal(includeSource("agent.ts"), true); -}); - -test("hashTree is content-sensitive and copySource preserves its identity", async (t) => { - const root = await mkdtemp(join(tmpdir(), "eve-bench-source-")); - t.after(() => rm(root, { recursive: true, force: true })); - const source = join(root, "source"); - const destination = join(root, "destination"); - await mkdir(join(source, "nested"), { recursive: true }); - await writeFile(join(source, "agent.ts"), "export default 1;\n"); - await writeFile(join(source, "nested/file.txt"), "one\n"); - const before = await hashTree(source); - await copySource(source, destination); - assert.equal(await hashTree(destination), before); - await writeFile(join(source, "nested/file.txt"), "two\n"); - assert.notEqual(await hashTree(source), before); -}); - -test("hashTree rejects symlinks", async (t) => { - const root = await mkdtemp(join(tmpdir(), "eve-bench-source-link-")); - t.after(() => rm(root, { recursive: true, force: true })); - await writeFile(join(root, "target"), "data"); - await symlink(join(root, "target"), join(root, "link")); - await assert.rejects(hashTree(root), /does not support symlinks/); -}); diff --git a/packages/eve-bench/src/harnesses/e0/snapshot.ts b/packages/eve-bench/src/harnesses/e0/snapshot.ts deleted file mode 100644 index dfddddec4e..0000000000 --- a/packages/eve-bench/src/harnesses/e0/snapshot.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { createHash } from "node:crypto"; -import { cp, lstat, readdir, readFile } from "node:fs/promises"; -import { basename, join, relative } from "node:path"; - -const EXCLUDED = new Set(["node_modules", ".git", ".eve", ".output", ".next", "dist"]); - -export function includeSource(path: string): boolean { - const name = basename(path); - return !EXCLUDED.has(name) && !name.startsWith(".env") && !name.startsWith(".eve-bench-"); -} - -/** Source snapshots never follow links into credentials, build output, or another checkout. */ -export async function hashTree(root: string): Promise { - const hash = createHash("sha256"); - let count = 0; - let bytes = 0; - async function walk(dir: string, depth: number): Promise { - if (depth > 32) throw new Error("Source tree exceeds 32 directory levels"); - for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - )) { - if (!includeSource(entry.name)) continue; - const path = join(dir, entry.name); - if (++count > 20_000) throw new Error("Source tree exceeds 20000 entries"); - if (entry.isSymbolicLink()) - throw new Error(`Source snapshot does not support symlinks: ${path}`); - if (entry.isDirectory()) await walk(path, depth + 1); - else if (entry.isFile()) { - const info = await lstat(path); - bytes += info.size; - if (bytes > 128 * 1024 * 1024) throw new Error("Source tree exceeds 128 MiB"); - hash - .update(relative(root, path)) - .update("\0") - .update(String(info.mode & 0o777)) - .update("\0"); - hash.update(await readFile(path)).update("\0"); - } else throw new Error(`Source snapshot requires regular files: ${path}`); - } - } - await walk(root, 0); - return hash.digest("hex"); -} - -export async function copySource(source: string, destination: string): Promise { - const before = await hashTree(source); - await cp(source, destination, { recursive: true, filter: includeSource }); - if ((await hashTree(destination)) !== before || (await hashTree(source)) !== before) { - throw new Error( - "Source changed while preparing the benchmark snapshot; retry after edits settle", - ); - } -} diff --git a/packages/eve-bench/src/options.ts b/packages/eve-bench/src/options.ts index 1b5576a1d5..5fe4f97c25 100644 --- a/packages/eve-bench/src/options.ts +++ b/packages/eve-bench/src/options.ts @@ -6,7 +6,6 @@ import { readCohort, readDatasetLock, type DatasetLock } from "./core/dataset.ts import type { Harness } from "./core/harness.ts"; import { createEveHarness } from "./harnesses/eve/index.ts"; import { createOracleHarness } from "./harnesses/oracle.ts"; -import { createE0Harness } from "./harnesses/e0/index.ts"; import { createCliHarness } from "./harnesses/cli/index.ts"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -81,11 +80,8 @@ export function defaultJobName(label: string): string { export function selectHarness( name: string, eve: string, - options: { agent?: string; version?: string; baseUrl?: string; reasoning?: string } = {}, + options: { version?: string; baseUrl?: string; reasoning?: string } = {}, ): Harness { - if (name !== "e0" && options.agent) throw new Error("--agent is only supported by --harness e0"); - if (name === "e0") - return createE0Harness({ agent: options.agent ?? "", reasoning: options.reasoning }); if (name === "pi" || name === "opencode" || name === "codex" || name === "hermes") { return createCliHarness(name, { version: options.version ?? "", diff --git a/packages/eve-bench/src/tools/run_bench.ts b/packages/eve-bench/src/tools/run_bench.ts index 89697ee570..8b63a2c7c4 100644 --- a/packages/eve-bench/src/tools/run_bench.ts +++ b/packages/eve-bench/src/tools/run_bench.ts @@ -13,8 +13,7 @@ export const inputSchema = z cohort: z.string().min(1).optional(), tasks: z.array(z.string().min(1)).min(1).optional(), model: z.string().min(1).optional(), - harness: z.enum(["eve", "e0", "oracle", "pi", "opencode", "codex", "hermes"]).default("eve"), - agent: z.string().min(1).optional(), + harness: z.enum(["eve", "oracle", "pi", "opencode", "codex", "hermes"]).default("eve"), version: z.string().min(1).optional(), baseUrl: z.string().url().optional(), reasoning: z.string().min(1).optional(), @@ -27,9 +26,6 @@ export const inputSchema = z if (input.harness !== "oracle" && !input.model) { ctx.addIssue({ code: "custom", path: ["model"], message: "model is required" }); } - if (input.harness === "e0" && !input.agent) { - ctx.addIssue({ code: "custom", path: ["agent"], message: "agent is required for e0" }); - } if (["pi", "opencode", "codex", "hermes"].includes(input.harness) && !input.version) { ctx.addIssue({ code: "custom", path: ["version"], message: "version is required" }); } diff --git a/packages/eve-bench/src/tools/tools.test.ts b/packages/eve-bench/src/tools/tools.test.ts index 1e5933f587..7def3079d6 100644 --- a/packages/eve-bench/src/tools/tools.test.ts +++ b/packages/eve-bench/src/tools/tools.test.ts @@ -22,11 +22,6 @@ describe("run_bench input", () => { true, ); assert.equal(runBenchInput.safeParse({ harness: "oracle" }).success, true); - assert.equal( - runBenchInput.safeParse({ harness: "e0", agent: "/apps/e0", model: "openai/gpt-5.6-terra" }) - .success, - true, - ); assert.equal( runBenchInput.safeParse({ harness: "pi", @@ -45,6 +40,7 @@ describe("run_bench input", () => { assert.equal( runBenchInput.safeParse({ harness: "e0", model: "openai/gpt-5.6-terra" }).success, false, + "e0 is not a supported harness", ); assert.equal( runBenchInput.safeParse({ harness: "codex", model: "openai/gpt-5.6-terra" }).success, diff --git a/packages/eve-bench/test/scenarios/e0-container.test.ts b/packages/eve-bench/test/scenarios/e0-container.test.ts deleted file mode 100644 index 28afd82ee1..0000000000 --- a/packages/eve-bench/test/scenarios/e0-container.test.ts +++ /dev/null @@ -1,539 +0,0 @@ -import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import { - lstat, - mkdir, - mkdtemp, - open, - readFile, - readdir, - realpath, - rm, - symlink, - writeFile, -} from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { test, type TestContext } from "node:test"; -import { setTimeout as delay } from "node:timers/promises"; -import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; -import type { SandboxBackend, SandboxSession } from "eve/sandbox"; -import { containerBackend } from "../../src/harnesses/e0/container-backend.ts"; - -const MAX_BYTES = 16 * 1024 * 1024; -const exec = promisify(execFile); -const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; -const node = (script: string) => `${quote(process.execPath)} -e ${quote(script)}`; - -function env(t: TestContext, values: Record) { - const previous = Object.fromEntries(Object.keys(values).map((key) => [key, process.env[key]])); - const set = (entries: typeof values) => { - for (const [key, value] of Object.entries(entries)) { - if (value === undefined) delete process.env[key]; - else process.env[key] = value; - } - }; - set(values); - t.after(() => set(previous)); -} - -async function fixture(t: TestContext, commandTimeoutMs = 3_000) { - const dir = await realpath(await mkdtemp(join(tmpdir(), "eve-bench-e0-"))); - const root = join(dir, "task with 'quote"); - const backend: SandboxBackend = containerBackend(root, { commandTimeoutMs }); - const input = { sessionKey: `test:${root}`, templateKey: null, runtimeContext: { appRoot: dir } }; - const handle = await backend.create(input); - t.after(async () => { - await handle.shutdown(); - await rm(dir, { recursive: true, force: true }); - }); - return { dir, root, backend, handle, session: handle.session, input }; -} - -async function collect(stream: ReadableStream) { - const chunks: Uint8Array[] = []; - for await (const chunk of stream) chunks.push(chunk); - return Buffer.concat(chunks).toString("utf8"); -} - -async function ready(root: string) { - for (let i = 0; i < 100; i++) { - try { - return Number(await readFile(join(root, "descendant.pid"), "utf8")); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - await delay(10); - } - throw new Error("Descendant did not start"); -} - -async function stopped(pid: number) { - for (let i = 0; i < 100; i++) { - try { - const { stdout } = await exec("ps", ["-o", "stat=", "-p", String(pid)]); - // Linux init may reap a killed orphan later; a zombie cannot execute work. - if (stdout.trim().startsWith("Z") || !stdout.trim()) return; - } catch (error) { - if ((error as { code?: number }).code === 1) return; - throw error; - } - await delay(10); - } - assert.fail(`Process ${pid} is still running`); -} - -const descendant = "(sleep 0.5; printf leaked > escaped) & echo $! > descendant.pid; wait"; - -test("default root requires workdir and container guard; explicit roots support local tests", async (t) => { - const { root } = await fixture(t); - env(t, { EVE_BENCH_TASK_WORKDIR: undefined, EVE_BENCH_CONTAINER: "1" }); - assert.throws(() => containerBackend(), /EVE_BENCH_TASK_WORKDIR/); - process.env.EVE_BENCH_TASK_WORKDIR = root; - delete process.env.EVE_BENCH_CONTAINER; - assert.throws(() => containerBackend(), /EVE_BENCH_CONTAINER=1/); - assert.throws(() => containerBackend(undefined, {}), /EVE_BENCH_CONTAINER=1/); - assert.doesNotThrow(() => containerBackend(root)); - process.env.EVE_BENCH_CONTAINER = "1"; - assert.doesNotThrow(() => containerBackend()); - for (const invalid of ["", "relative"]) - assert.throws(() => containerBackend(invalid), /absolute task directory/); - for (const commandTimeoutMs of [0, -1, NaN, Infinity, 2_147_483_648]) - assert.throws(() => containerBackend(root, { commandTimeoutMs }), /commandTimeoutMs/); -}); - -test("workspace paths, cwd, actual roots and skill home work without symlinks or shell rewriting", async (t) => { - const { session, backend, dir, root } = await fixture(t); - const home = join(dir, "home"); - env(t, { HOME: home }); - await mkdir(home); - const prewarm = { - templateKey: "test", - runtimeContext: { appRoot: dir }, - seedFiles: [ - { path: "/workspace/nested/seed.txt", content: "seed\n" }, - { path: "$HOME/.agents/skills/test/SKILL.md", content: "skill\n" }, - { path: join(home, ".agents/skills/test/reference.md"), content: Buffer.from("reference\n") }, - ], - }; - assert.deepEqual(await backend.prewarm(prewarm), { reused: false }); - assert.equal(session.resolvePath("/workspace"), root); - assert.equal(session.resolvePath("/workspace/nested"), join(root, "nested")); - assert.equal(session.resolvePath("nested/seed.txt"), join(root, "nested/seed.txt")); - for (const escaped of [ - "/workspace-other", - "/proc/self/environ", - "/installed-agent", - "../../outside", - ]) { - assert.throws(() => session.resolvePath(escaped), /must stay inside/); - await assert.rejects(session.readTextFile({ path: escaped }), /must stay inside/); - } - assert.equal(await session.readTextFile({ path: "/workspace/nested/seed.txt" }), "seed\n"); - await symlink("/proc", join(root, "proc-link")); - await assert.rejects( - session.readTextFile({ path: "/workspace/proc-link/self/environ" }), - /do not follow symbolic links/, - ); - await rm(join(root, "proc-link")); - assert.equal( - await session.readTextFile({ path: "$HOME/.agents/skills/test/SKILL.md" }), - "skill\n", - ); - const shell = await session.run({ - command: 'printf "%s\\n" "$PWD"; cat seed.txt; cat "$HOME/.agents/skills/test/reference.md"', - workingDirectory: "/workspace/nested", - }); - assert.equal(shell.exitCode, 0, shell.stderr); - assert.equal(shell.stdout, `${root}/nested\nseed\nreference\n`); - const literal = await session.run({ command: "printf '%s' '/workspace/literal'" }); - assert.equal(literal.stdout, "/workspace/literal"); - const description = (session as SandboxSession & { description: string }).description; - assert.ok(description.includes(JSON.stringify(root))); - assert.match(description, /shell command text does not/); - assert.equal((await lstat(root)).isSymbolicLink(), false); - assert.deepEqual(await readdir(root), ["nested"]); - await assert.rejects( - backend.prewarm({ ...prewarm, bootstrap: () => assert.fail("must not bootstrap") }), - /Production sandbox bootstrap/, - ); - await assert.rejects( - backend.prewarm({ ...prewarm, seedFiles: [{ path: join(dir, "unexpected"), content: "no" }] }), - /must stay inside/, - ); -}); - -test("shell inherits only operational env, not provider/production secrets or startup hooks", async (t) => { - const { session, dir } = await fixture(t); - const startup = join(dir, "startup.sh"); - await writeFile(startup, "export INJECTED_SECRET=should-not-run\n"); - env(t, { - AI_GATEWAY_API_KEY: "fake-provider", - OPENAI_API_KEY: "fake-openai", - ANTHROPIC_API_KEY: "fake-anthropic", - D0_JUDGE_API_KEY: "fake-judge", - VERCEL_TOKEN: "fake-vercel", - AWS_SECRET_ACCESS_KEY: "fake-aws", - DATABASE_URL: "fake-database", - CUSTOM_PRODUCTION_VALUE: "fake-custom", - GITHUB_TOKEN: "fake-github", - BASH_ENV: startup, - ENV: startup, - NODE_OPTIONS: "--trace-warnings", - LANG: "C", - }); - const result = await session.run({ - command: node("console.log(JSON.stringify(process.env))"), - env: { EVE_CODE_DIAGNOSTICS_REQUEST: "explicit-tool-data" }, - }); - assert.equal(result.exitCode, 0, result.stderr); - const forwarded = JSON.parse(result.stdout) as Record; - for (const key of [ - "AI_GATEWAY_API_KEY", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "D0_JUDGE_API_KEY", - "VERCEL_TOKEN", - "AWS_SECRET_ACCESS_KEY", - "DATABASE_URL", - "CUSTOM_PRODUCTION_VALUE", - "GITHUB_TOKEN", - "BASH_ENV", - "ENV", - "NODE_OPTIONS", - "INJECTED_SECRET", - ]) - assert.equal(forwarded[key], undefined, key); - assert.equal(forwarded.PATH, process.env.PATH); - assert.equal(forwarded.LANG, "C"); - assert.equal(forwarded.EVE_CODE_DIAGNOSTICS_REQUEST, "explicit-tool-data"); - const override = await session.run({ command: 'printf "%s" "$LANG"', env: { LANG: "explicit" } }); - assert.equal(override.stdout, "explicit"); -}); - -test("binary, text and streaming file operations preserve bytes, ranges, encodings and removal", async (t) => { - const { session } = await fixture(t); - for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) - assert.equal(await session[method]({ path: "missing" }), null); - await session.writeTextFile({ path: "/workspace/text", content: "one\n😀two\nthree\n" }); - assert.equal(await session.readTextFile({ path: "text", startLine: 2, endLine: 2 }), "😀two"); - assert.equal(await session.readTextFile({ path: "text", startLine: 3, endLine: 99 }), "three\n"); - await assert.rejects( - async () => session.readTextFile({ path: "text", startLine: 0 }), - /line range/, - ); - await assert.rejects( - async () => session.readTextFile({ path: "text", startLine: 3, endLine: 2 }), - /line range/, - ); - await session.writeTextFile({ path: "latin", content: "café", encoding: "latin1" }); - assert.equal(await session.readTextFile({ path: "latin", encoding: "latin1" }), "café"); - await assert.rejects(async () => session.readTextFile({ path: "latin" }), /encoded data/); - const bytes = Uint8Array.from([0, 255, 128, 1]); - await session.writeBinaryFile({ path: "binary", content: bytes }); - assert.deepEqual( - Buffer.from((await session.readBinaryFile({ path: "binary" }))!), - Buffer.from(bytes), - ); - const stream = await session.readFile({ path: "text" }); - assert.ok(stream); - await session.writeFile({ path: "copy/nested", content: stream }); - assert.equal(await session.readTextFile({ path: "copy/nested" }), "one\n😀two\nthree\n"); - await session.removePath({ path: "/workspace/copy", recursive: true }); - await session.removePath({ path: "missing", force: true }); - assert.equal(await session.readFile({ path: "copy/nested" }), null); -}); - -test("all file surfaces are bounded and oversized streamed writes leave existing files untouched", async (t) => { - const { session, root } = await fixture(t); - const file = await open(join(root, "large"), "w"); - await file.truncate(MAX_BYTES + 1); - await file.close(); - for (const method of ["readFile", "readBinaryFile", "readTextFile"] as const) - await assert.rejects( - async () => session[method]({ path: "large" }), - /File read exceeds 16 MiB/, - ); - await assert.rejects(async () => session.readBinaryFile({ path: "." }), /regular files/); - await session.writeTextFile({ path: "target", content: "unchanged" }); - await assert.rejects( - async () => session.writeBinaryFile({ path: "target", content: new Uint8Array(MAX_BYTES + 1) }), - /File write exceeds 16 MiB/, - ); - await assert.rejects( - async () => session.writeTextFile({ path: "target", content: "😀".repeat(MAX_BYTES / 4 + 1) }), - /File write exceeds 16 MiB/, - ); - let cancelled = false; - const content = new ReadableStream({ - pull(controller) { - controller.enqueue(new Uint8Array(1024 * 1024)); - }, - cancel() { - cancelled = true; - }, - }); - await assert.rejects( - async () => session.writeFile({ path: "target", content }), - /File write exceeds 16 MiB/, - ); - assert.equal(cancelled, true); - assert.equal(await session.readTextFile({ path: "target" }), "unchanged"); - await session.writeBinaryFile({ path: "boundary", content: new Uint8Array(MAX_BYTES) }); - assert.equal((await session.readBinaryFile({ path: "boundary" }))?.byteLength, MAX_BYTES); - assert.equal((await session.run({ command: "mkfifo pipe" })).exitCode, 0); - await assert.rejects(async () => session.readBinaryFile({ path: "pipe" }), /regular files/); - await assert.rejects( - async () => session.writeTextFile({ path: "pipe", content: "no" }), - /ENXIO|regular files/, - ); - await assert.rejects( - async () => session.writeTextFile({ path: "/dev/null", content: "no" }), - /must stay inside/, - ); -}); - -test("aborted file operations and a blocked stream reject without writing", async (t) => { - const { session } = await fixture(t); - const controller = new AbortController(); - const reason = new Error("cancel file"); - const content = new ReadableStream(); - const pending = session.writeFile({ path: "never", content, abortSignal: controller.signal }); - controller.abort(reason); - await assert.rejects( - async () => pending, - (error) => error === reason, - ); - await assert.rejects( - async () => session.readFile({ path: "never", abortSignal: controller.signal }), - (error) => error === reason, - ); - await assert.rejects( - async () => - session.writeBinaryFile({ - path: "never", - content: new Uint8Array(), - abortSignal: controller.signal, - }), - (error) => error === reason, - ); - await assert.rejects( - session.removePath({ path: "never", abortSignal: controller.signal }), - (error) => error === reason, - ); - assert.equal(await session.readFile({ path: "never" }), null); -}); - -test("run and spawn preserve split UTF-8, stderr and native nonzero exits", async (t) => { - const { session } = await fixture(t); - const command = node( - "process.stdout.write(Buffer.from([0xf0, 0x9f])); setTimeout(() => { process.stdout.write(Buffer.from([0x98, 0x80])); process.stderr.write('err'); process.exitCode = 7; }, 20)", - ); - assert.deepEqual(await session.run({ command }), { stdout: "😀", stderr: "err", exitCode: 7 }); - const child = await session.spawn({ command }); - const [stdout, stderr, result] = await Promise.all([ - collect(child.stdout), - collect(child.stderr), - child.wait(), - ]); - assert.deepEqual({ stdout, stderr, ...result }, { stdout: "😀", stderr: "err", exitCode: 7 }); - await child.kill(); - await child.kill(); - await assert.rejects( - async () => session.run({ command: "true", workingDirectory: "absent" }), - /ENOENT/, - ); -}); - -for (const method of ["run", "spawn"] as const) { - test(`${method} timeout kills the entire process group`, async (t) => { - const { session, root } = await fixture(t, 200); - const operation = session[method]({ command: descendant }); - const pid = await ready(root); - if (method === "run") { - const result = (await operation) as Awaited>; - assert.equal(result.exitCode, 124); - assert.match(result.stderr, /process group was stopped/); - } else { - const child = (await operation) as Awaited>; - assert.equal((await child.wait()).exitCode, 124); - } - await stopped(pid); - await delay(550); - assert.equal(await session.readFile({ path: "escaped" }), null); - }); - test(`${method} abort kills descendants and rejects with the abort reason`, async (t) => { - const { session, root } = await fixture(t); - const controller = new AbortController(); - const reason = new Error("cancel command"); - const operation = session[method]({ command: descendant, abortSignal: controller.signal }); - const completion = - method === "run" - ? operation - : Promise.resolve(operation).then((child) => - (child as Awaited>).wait(), - ); - const rejection = assert.rejects( - async () => completion, - (error) => error === reason, - ); - const pid = await ready(root); - controller.abort(reason); - await rejection; - await stopped(pid); - await assert.rejects( - async () => session[method]({ command: "touch never", abortSignal: controller.signal }), - (error) => error === reason, - ); - assert.equal(await session.readFile({ path: "never" }), null); - }); - test(`${method} bounds combined stdout/stderr even when spawn output is not consumed`, async (t) => { - const { session } = await fixture(t); - const command = node( - "const chunk = Buffer.alloc(1024 * 1024, 'x'); setInterval(() => { process.stdout.write(chunk); process.stderr.write(chunk); }, 1)", - ); - if (method === "run") - await assert.rejects(async () => session.run({ command }), /Command output exceeds 16 MiB/); - else { - const child = await session.spawn({ command }); - await assert.rejects(async () => child.wait(), /Command output exceeds 16 MiB/); - await stopped(child.pid!); - } - }); -} - -test("stop/shutdown are idempotent, retain task files and allow reattachment; shell exit kills background work", async (t) => { - const { session, backend, handle, input, root } = await fixture(t); - await session.writeTextFile({ path: "keep", content: "task artifact" }); - for (const stop of [handle.stop, handle.shutdown]) { - const child = await session.spawn({ command: descendant }); - const pid = await ready(root); - await stop(); - await stop(); - await stopped(pid); - assert.equal((await child.wait()).exitCode, 137); - await session.removePath({ path: "descendant.pid" }); - } - assert.deepEqual(await handle.captureState(), { - backendName: backend.name, - metadata: {}, - sessionKey: input.sessionKey, - }); - const reopened = await backend.create(input); - assert.equal(reopened.session.id, session.id); - assert.equal((await reopened.useSessionFn()).id, session.id); - assert.equal(await reopened.session.readTextFile({ path: "keep" }), "task artifact"); - await session.run({ - command: "(sleep 0.5; printf leaked > escaped) >/dev/null 2>&1 & echo $! > descendant.pid", - }); - await stopped(await ready(root)); - await session.setNetworkPolicy("allow-all"); - await assert.rejects(session.setNetworkPolicy("deny-all"), /runner owns network policy/); -}); - -test("installed eve built-in bash/file tools work with workspace, actual root and home paths", async (t) => { - const { session, dir, root } = await fixture(t); - env(t, { HOME: join(dir, "home") }); - const installed = new URL("../../node_modules/eve/dist/src/", import.meta.url); - const { ContextContainer, contextStorage } = await import( - new URL("context/container.js", installed).href - ); - const { executeBashOnSandbox } = await import( - new URL("execution/sandbox/bash-tool.js", installed).href - ); - const { executeReadFileOnSandbox } = await import( - new URL("execution/sandbox/read-file-tool.js", installed).href - ); - const { executeWriteFileOnSandbox } = await import( - new URL("execution/sandbox/write-file-tool.js", installed).href - ); - const { executeGlobOnSandbox } = await import( - new URL("execution/sandbox/glob-tool.js", installed).href - ); - const { executeGrepOnSandbox } = await import( - new URL("execution/sandbox/grep-tool.js", installed).href - ); - await contextStorage.run(new ContextContainer(), async () => { - for (const filePath of ["/workspace/builtin.txt", "$HOME/.agents/skills/example/SKILL.md"]) { - assert.equal( - (await executeWriteFileOnSandbox(session, { filePath, content: "first\n" })).existed, - false, - ); - assert.equal((await executeReadFileOnSandbox(session, { filePath })).content, "1: first"); - assert.equal( - (await executeWriteFileOnSandbox(session, { filePath, content: "second\n" })).existed, - true, - ); - } - const result = await executeBashOnSandbox(session, { command: "cat builtin.txt" }); - assert.equal(result.exitCode, 0); - assert.equal(result.stdout, "second\n"); - const glob = await executeGlobOnSandbox(session, { path: root, pattern: "*.txt" }); - assert.equal(glob.count, 1); - assert.equal(glob.content, join(root, "builtin.txt")); - const grep = await executeGrepOnSandbox(session, { path: root, pattern: "second" }); - assert.equal(grep.matchCount, 1); - assert.ok(grep.content.includes("builtin.txt:1:second")); - }); -}); - -// Optional real extension check: supply the eve-code package path from the e0 -// source checkout. Run on Linux (GNU realpath/stat), with no model or network. -test( - "real e0 code__apply_patch adds, updates, moves and deletes files in the actual task root", - { - skip: !process.env.EVE_BENCH_EVE_CODE_DIR - ? "Set EVE_BENCH_EVE_CODE_DIR to the eve-code package" - : false, - }, - async (t) => { - const { session, root } = await fixture(t); - const { default: tool } = await import( - pathToFileURL(join(process.env.EVE_BENCH_EVE_CODE_DIR!, "extension/tools/apply_patch.ts")) - .href - ); - const initialized = await session.run({ command: "git init -q" }); - assert.equal(initialized.exitCode, 0, initialized.stderr); - const ctx = { getSandbox: async () => session }; - const added = await tool.execute( - { root, patchText: "*** Begin Patch\n*** Add File: hello.txt\n+before\n*** End Patch" }, - ctx, - ); - assert.equal(added.files[0].operation, "add"); - assert.deepEqual(added.diagnostics, []); - const updated = await tool.execute( - { - root, - patchText: - "*** Begin Patch\n*** Update File: hello.txt\n@@\n-before\n+after\n*** End Patch", - }, - ctx, - ); - assert.equal(updated.files[0].operation, "update"); - const moved = await tool.execute( - { - root, - patchText: - "*** Begin Patch\n*** Update File: hello.txt\n*** Move to: nested/moved.txt\n*** End Patch", - }, - ctx, - ); - assert.equal(moved.files[0].operation, "move"); - assert.equal(await session.readTextFile({ path: "/workspace/nested/moved.txt" }), "after\n"); - assert.equal(await session.readTextFile({ path: "hello.txt" }), null); - const deleted = await tool.execute( - { root, patchText: "*** Begin Patch\n*** Delete File: nested/moved.txt\n*** End Patch" }, - ctx, - ); - assert.equal(deleted.files[0].operation, "delete"); - assert.equal(await session.readTextFile({ path: "nested/moved.txt" }), null); - await assert.rejects( - tool.execute( - { root, patchText: "*** Begin Patch\n*** Add File: ../escape\n+no\n*** End Patch" }, - ctx, - ), - /invalid patch path/, - ); - }, -); From a504d198368d8905b189679608a004e3910a93c3 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:31:56 -0400 Subject: [PATCH 5/9] fix: restore docs registry and web template from main The merge commit pre-commit hook reformatted these generated files with a stale local oxfmt, which made check:web-template report drift. Signed-off-by: Rui Conti From 1ffc00416956e9bafd0d52ef25754c29988f6549 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:32:18 -0400 Subject: [PATCH 6/9] fix: restore docs registry and web template from main The pre-commit formatter rewrote these generated files during the main merge, which made check:web-template report drift. Committed without the formatting hook so they match main byte for byte. Signed-off-by: Rui Conti --- apps/docs/registry.json | 802 ++++++++++++++---- .../src/setup/scaffold/create/web-template.ts | 8 +- 2 files changed, 660 insertions(+), 150 deletions(-) diff --git a/apps/docs/registry.json b/apps/docs/registry.json index 62dd8e9c5b..1dd3a85e4b 100644 --- a/apps/docs/registry.json +++ b/apps/docs/registry.json @@ -8,7 +8,9 @@ "type": "registry:item", "title": "Self-modification", "description": "Add a source editing subagent with local edits and official registry installation.", - "dependencies": ["just-bash"], + "dependencies": [ + "just-bash" + ], "meta": { "eve": { "requires": ">=0.62.0", @@ -16,7 +18,10 @@ "pnpm": { "buildScripts": [ { - "packages": ["node-liblzma", "@mongodb-js/zstd"], + "packages": [ + "node-liblzma", + "@mongodb-js/zstd" + ], "optional": true, "recommendedAction": "ignore-optional", "reason": "These optional just-bash compression accelerators are not required by the self-modification agent." @@ -28,7 +33,11 @@ { "package": "eve", "bin": "eve", - "args": ["integration", "setup", "self-modification"] + "args": [ + "integration", + "setup", + "self-modification" + ] } ] } @@ -46,7 +55,9 @@ "type": "registry:item", "title": "Self-modification (Experimental)", "description": "Add an experimental source editing subagent with local edits and official registry installation.", - "dependencies": ["just-bash"], + "dependencies": [ + "just-bash" + ], "meta": { "eve": { "hidden": true, @@ -54,7 +65,10 @@ "pnpm": { "buildScripts": [ { - "packages": ["node-liblzma", "@mongodb-js/zstd"], + "packages": [ + "node-liblzma", + "@mongodb-js/zstd" + ], "optional": true, "recommendedAction": "ignore-optional", "reason": "These optional just-bash compression accelerators are not required by the self-modification agent." @@ -66,7 +80,11 @@ { "package": "eve", "bin": "eve", - "args": ["integration", "setup", "self-modification"] + "args": [ + "integration", + "setup", + "self-modification" + ] } ], "requires": ">=0.62.0" @@ -85,7 +103,9 @@ "type": "registry:item", "title": "agent-browser", "description": "Add browser automation tools backed by agent-browser to an eve agent.", - "dependencies": ["@agent-browser/eve"], + "dependencies": [ + "@agent-browser/eve" + ], "files": [ { "path": "registry/extensions/agent-browser.ts", @@ -99,7 +119,9 @@ "type": "registry:item", "title": "Browserbase", "description": "Search, fetch, and automate the web with Browserbase and Stagehand.", - "dependencies": ["@browserbasehq/eve"], + "dependencies": [ + "@browserbasehq/eve" + ], "envVars": { "BROWSERBASE_API_KEY": "" }, @@ -116,7 +138,9 @@ "type": "registry:item", "title": "BlitzReels", "description": "Turn long videos into short clips, generate media, repair edits, and export.", - "dependencies": ["@blitzreels/eve"], + "dependencies": [ + "@blitzreels/eve" + ], "envVars": { "BLITZREELS_API_KEY": "" }, @@ -133,7 +157,10 @@ "type": "registry:item", "title": "GitHub Tools", "description": "Add scoped GitHub tools with Vercel Connect authentication and approval rules.", - "dependencies": ["@github-tools/eve-extension", "@vercel/connect"], + "dependencies": [ + "@github-tools/eve-extension", + "@vercel/connect" + ], "envVars": { "GITHUB_TOKEN": "" }, @@ -150,7 +177,9 @@ "type": "registry:item", "title": "Jetty", "description": "Grade agent turns, compare experiments, and store durable evaluation trajectories.", - "dependencies": ["@jetty/eve"], + "dependencies": [ + "@jetty/eve" + ], "envVars": { "JETTY_API_TOKEN": "", "JETTY_COLLECTION": "" @@ -168,7 +197,9 @@ "type": "registry:item", "title": "KERNEL", "description": "Let your eve agent use the Internet with KERNEL browser infrastructure, observability, and stealth.", - "dependencies": ["@onkernel/eve-extension"], + "dependencies": [ + "@onkernel/eve-extension" + ], "envVars": { "KERNEL_API_KEY": "" }, @@ -190,7 +221,11 @@ "setup": { "package": "eve", "bin": "eve", - "args": ["integration", "setup", "file-memory"] + "args": [ + "integration", + "setup", + "file-memory" + ] }, "requires": ">=0.49.1", "docs": "/docs/memory/file" @@ -209,7 +244,9 @@ "type": "registry:item", "title": "Supermemory", "description": "Give your agents long-term memory, user profiles, and SuperRAG across conversations and context.", - "dependencies": ["@supermemory/eve"], + "dependencies": [ + "@supermemory/eve" + ], "envVars": { "SUPERMEMORY_API_KEY": "" }, @@ -231,7 +268,10 @@ "type": "registry:item", "title": "Upstash AgentKit", "description": "Give your agents ranked recall and automatic capture on Upstash Redis, or a Redis backend for file memory.", - "dependencies": ["@upstash/agentkit-eve", "@upstash/redis"], + "dependencies": [ + "@upstash/agentkit-eve", + "@upstash/redis" + ], "envVars": { "UPSTASH_REDIS_REST_URL": "", "UPSTASH_REDIS_REST_TOKEN": "" @@ -254,7 +294,9 @@ "type": "registry:item", "title": "Kybernesis Arcana", "description": "Give your agents workspace-scoped long-term memory with automatic recall and deliberate storage.", - "dependencies": ["@kybernesis/arcana"], + "dependencies": [ + "@kybernesis/arcana" + ], "envVars": { "ARCANA_API_KEY": "", "ARCANA_WORKSPACE": "" @@ -277,7 +319,9 @@ "type": "registry:item", "title": "Hindsight", "description": "Recall relevant context before every turn and retain each exchange automatically.", - "dependencies": ["@vectorize-io/hindsight-eve"], + "dependencies": [ + "@vectorize-io/hindsight-eve" + ], "envVars": { "HINDSIGHT_API_KEY": "", "HINDSIGHT_API_URL": "", @@ -492,7 +536,9 @@ "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/vercel.ts", @@ -512,12 +558,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "linear", "mcp.linear.app", "linear"] + "args": [ + "integration", + "connect", + "linear", + "mcp.linear.app", + "linear" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/linear.ts", @@ -537,12 +591,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "notion", "mcp.notion.com", "notion"] + "args": [ + "integration", + "connect", + "notion", + "mcp.notion.com", + "notion" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/notion.ts", @@ -562,12 +624,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "datadog", "mcp.datadoghq.com", "datadog"] + "args": [ + "integration", + "connect", + "datadog", + "mcp.datadoghq.com", + "datadog" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/datadog.ts", @@ -587,12 +657,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "honeycomb", "mcp.honeycomb.io", "honeycomb"] + "args": [ + "integration", + "connect", + "honeycomb", + "mcp.honeycomb.io", + "honeycomb" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/honeycomb.ts", @@ -627,8 +705,12 @@ "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], - "devDependencies": ["vercel@>=58.7.0"], + "dependencies": [ + "@vercel/connect" + ], + "devDependencies": [ + "vercel@>=58.7.0" + ], "files": [ { "path": "registry/connections/agentcard.ts", @@ -648,12 +730,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "airtable", "airtable", "airtable"] + "args": [ + "integration", + "connect", + "airtable", + "airtable", + "airtable" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/airtable.ts", @@ -673,12 +763,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "bitly", "bitly", "bitly"] + "args": [ + "integration", + "connect", + "bitly", + "bitly", + "bitly" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/bitly.ts", @@ -698,12 +796,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "brex", "brex", "brex"] + "args": [ + "integration", + "connect", + "brex", + "brex", + "brex" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/brex.ts", @@ -723,12 +829,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "candid", "candid", "candid"] + "args": [ + "integration", + "connect", + "candid", + "candid", + "candid" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/candid.ts", @@ -748,12 +862,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "clickhouse", "clickhouse", "clickhouse"] + "args": [ + "integration", + "connect", + "clickhouse", + "clickhouse", + "clickhouse" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/clickhouse.ts", @@ -773,12 +895,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "cloudinary", "cloudinary", "cloudinary"] + "args": [ + "integration", + "connect", + "cloudinary", + "cloudinary", + "cloudinary" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/cloudinary.ts", @@ -798,12 +928,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "coda", "coda", "coda"] + "args": [ + "integration", + "connect", + "coda", + "coda", + "coda" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/coda.ts", @@ -823,12 +961,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "context", "mcp.context.dev", "context"] + "args": [ + "integration", + "connect", + "context", + "mcp.context.dev", + "context" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/context.ts", @@ -848,12 +994,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "egnyte", "egnyte", "egnyte"] + "args": [ + "integration", + "connect", + "egnyte", + "egnyte", + "egnyte" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/egnyte.ts", @@ -873,12 +1027,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "embat", "embat", "embat"] + "args": [ + "integration", + "connect", + "embat", + "embat", + "embat" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/embat.ts", @@ -898,12 +1060,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "hugging-face", "hugging-face", "hugging-face"] + "args": [ + "integration", + "connect", + "hugging-face", + "hugging-face", + "hugging-face" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/hugging-face.ts", @@ -923,12 +1093,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "local-falcon", "local-falcon", "local-falcon"] + "args": [ + "integration", + "connect", + "local-falcon", + "local-falcon", + "local-falcon" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/local-falcon.ts", @@ -948,12 +1126,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "make", "make", "make"] + "args": [ + "integration", + "connect", + "make", + "make", + "make" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/make.ts", @@ -973,12 +1159,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "manufact", "manufact", "manufact"] + "args": [ + "integration", + "connect", + "manufact", + "manufact", + "manufact" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/manufact.ts", @@ -998,12 +1192,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "mem0", "mem0", "mem0"] + "args": [ + "integration", + "connect", + "mem0", + "mem0", + "mem0" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/mem0.ts", @@ -1023,12 +1225,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "miro", "miro", "miro"] + "args": [ + "integration", + "connect", + "miro", + "miro", + "miro" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/miro.ts", @@ -1048,12 +1258,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "mixpanel", "mixpanel", "mixpanel"] + "args": [ + "integration", + "connect", + "mixpanel", + "mixpanel", + "mixpanel" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/mixpanel.ts", @@ -1073,12 +1291,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "natural", "mcp.natural.com", "natural"] + "args": [ + "integration", + "connect", + "natural", + "mcp.natural.com", + "natural" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/natural.ts", @@ -1098,12 +1324,22 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "neon", "neon", "neon", "--principal-type", "app"] + "args": [ + "integration", + "connect", + "neon", + "neon", + "neon", + "--principal-type", + "app" + ] }, "requires": ">=0.48.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/neon.ts", @@ -1123,12 +1359,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "netlify", "netlify", "netlify"] + "args": [ + "integration", + "connect", + "netlify", + "netlify", + "netlify" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/netlify.ts", @@ -1148,12 +1392,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "oreilly", "oreilly", "oreilly"] + "args": [ + "integration", + "connect", + "oreilly", + "oreilly", + "oreilly" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/oreilly.ts", @@ -1173,12 +1425,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "planetscale", "planetscale", "planetscale"] + "args": [ + "integration", + "connect", + "planetscale", + "planetscale", + "planetscale" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/planetscale.ts", @@ -1198,12 +1458,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "posthog", "posthog", "posthog"] + "args": [ + "integration", + "connect", + "posthog", + "posthog", + "posthog" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/posthog.ts", @@ -1223,12 +1491,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "postman", "postman", "postman"] + "args": [ + "integration", + "connect", + "postman", + "postman", + "postman" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/postman.ts", @@ -1248,12 +1524,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "razorpay", "razorpay", "razorpay"] + "args": [ + "integration", + "connect", + "razorpay", + "razorpay", + "razorpay" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/razorpay.ts", @@ -1273,12 +1557,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "sentry", "sentry", "sentry"] + "args": [ + "integration", + "connect", + "sentry", + "sentry", + "sentry" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/sentry.ts", @@ -1298,12 +1590,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "similarweb", "similarweb", "similarweb"] + "args": [ + "integration", + "connect", + "similarweb", + "similarweb", + "similarweb" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/similarweb.ts", @@ -1323,7 +1623,11 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "shopify"] + "args": [ + "integration", + "setup", + "shopify" + ] }, "requires": ">=0.46.1" } @@ -1347,12 +1651,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "stripe", "stripe", "stripe"] + "args": [ + "integration", + "connect", + "stripe", + "stripe", + "stripe" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/stripe.ts", @@ -1372,12 +1684,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "supabase", "supabase", "supabase"] + "args": [ + "integration", + "connect", + "supabase", + "supabase", + "supabase" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/supabase.ts", @@ -1397,12 +1717,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "ticket-tailor", "ticket-tailor", "ticket-tailor"] + "args": [ + "integration", + "connect", + "ticket-tailor", + "ticket-tailor", + "ticket-tailor" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/ticket-tailor.ts", @@ -1422,12 +1750,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "ticktick", "ticktick", "ticktick"] + "args": [ + "integration", + "connect", + "ticktick", + "ticktick", + "ticktick" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/ticktick.ts", @@ -1460,7 +1796,9 @@ "requires": ">=0.48.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/tinybird.ts", @@ -1480,12 +1818,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "todoist", "todoist", "todoist"] + "args": [ + "integration", + "connect", + "todoist", + "todoist", + "todoist" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/todoist.ts", @@ -1505,12 +1851,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "webflow", "webflow", "webflow"] + "args": [ + "integration", + "connect", + "webflow", + "webflow", + "webflow" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/webflow.ts", @@ -1530,12 +1884,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "wix", "wix", "wix"] + "args": [ + "integration", + "connect", + "wix", + "wix", + "wix" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/wix.ts", @@ -1555,12 +1917,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "zapier", "zapier", "zapier"] + "args": [ + "integration", + "connect", + "zapier", + "zapier", + "zapier" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/zapier.ts", @@ -1580,12 +1950,20 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "connect", "zomato", "zomato", "zomato"] + "args": [ + "integration", + "connect", + "zomato", + "zomato", + "zomato" + ] }, "requires": ">=0.29.0" } }, - "dependencies": ["@vercel/connect"], + "dependencies": [ + "@vercel/connect" + ], "files": [ { "path": "registry/connections/zomato.ts", @@ -1605,28 +1983,40 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "slack"] + "args": [ + "integration", + "setup", + "slack" + ] }, "requires": ">=0.33.0", "docs": "/docs/channels/slack", "implementation": "native" } }, - "dependencies": ["@vercel/connect@>=0.4.2"] + "dependencies": [ + "@vercel/connect@>=0.4.2" + ] }, { "name": "channel/discord", "type": "registry:item", "title": "Discord", "description": "Connect an eve agent to Discord with guided connector and slash-command setup.", - "dependencies": ["@vercel/connect@>=0.6.0"], + "dependencies": [ + "@vercel/connect@>=0.6.0" + ], "meta": { "eve": { "setup": { "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "discord"] + "args": [ + "integration", + "setup", + "discord" + ] }, "requires": ">=0.33.0", "docs": "/docs/channels/discord", @@ -1645,15 +2035,23 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "teams"] + "args": [ + "integration", + "setup", + "teams" + ] }, "requires": ">=0.53.0", "docs": "/docs/channels/teams", "implementation": "native" } }, - "dependencies": ["@vercel/connect@>=2.1.0"], - "devDependencies": ["vercel@>=58.5.1"] + "dependencies": [ + "@vercel/connect@>=2.1.0" + ], + "devDependencies": [ + "vercel@>=58.5.1" + ] }, { "name": "channel/telegram", @@ -1705,7 +2103,9 @@ "type": "registry:item", "title": "Blooio", "description": "Send and receive iMessage, RCS, and SMS through Blooio, with reactions, typing indicators, read receipts, polls, groups, capability checks, and history.", - "dependencies": ["eve-channel-blooio"], + "dependencies": [ + "eve-channel-blooio" + ], "envVars": { "BLOOIO_API_KEY": "", "BLOOIO_WEBHOOK_SECRET": "" @@ -1735,15 +2135,23 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "github"] + "args": [ + "integration", + "setup", + "github" + ] }, "requires": ">=0.33.0", "docs": "/docs/channels/github", "implementation": "native" } }, - "dependencies": ["@vercel/connect@>=0.6.0"], - "devDependencies": ["vercel@>=58.5.1"] + "dependencies": [ + "@vercel/connect@>=0.6.0" + ], + "devDependencies": [ + "vercel@>=58.5.1" + ] }, { "name": "channel/linear", @@ -1756,15 +2164,23 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "linear"] + "args": [ + "integration", + "setup", + "linear" + ] }, "requires": ">=0.33.0", "docs": "/docs/channels/linear", "implementation": "native" } }, - "dependencies": ["@vercel/connect"], - "devDependencies": ["vercel@>=58.5.1"] + "dependencies": [ + "@vercel/connect" + ], + "devDependencies": [ + "vercel@>=58.5.1" + ] }, { "name": "channel/web", @@ -1777,7 +2193,11 @@ "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "web"] + "args": [ + "integration", + "setup", + "web" + ] }, "requires": ">=0.66.1", "docs": "/docs/channels/eve", @@ -2030,7 +2450,11 @@ "type": "registry:item", "title": "Google Chat", "description": "Google Chat spaces and DMs via the Chat SDK.", - "dependencies": ["chat", "@chat-adapter/gchat", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@chat-adapter/gchat", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-gchat.ts", @@ -2053,7 +2477,11 @@ "type": "registry:item", "title": "WhatsApp", "description": "Customer messaging through WhatsApp Business Cloud via the Chat SDK.", - "dependencies": ["chat", "@chat-adapter/whatsapp", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@chat-adapter/whatsapp", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-whatsapp.ts", @@ -2079,7 +2507,11 @@ "type": "registry:item", "title": "X", "description": "Public mentions and DMs on X via the Chat SDK.", - "dependencies": ["chat", "@chat-adapter/x", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@chat-adapter/x", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-x.ts", @@ -2103,7 +2535,11 @@ "type": "registry:item", "title": "Messenger", "description": "Facebook Messenger bots with templates, buttons, and reactions via the Chat SDK.", - "dependencies": ["chat", "@chat-adapter/messenger", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@chat-adapter/messenger", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-messenger.ts", @@ -2128,7 +2564,11 @@ "type": "registry:item", "title": "Zernio", "description": "Reach seven social and messaging platforms through one Zernio integration.", - "dependencies": ["chat", "@zernio/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@zernio/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "ZERNIO_API_KEY": "", "ZERNIO_WEBHOOK_SECRET": "" @@ -2152,7 +2592,11 @@ "type": "registry:item", "title": "Velt", "description": "Add agents to anchored comments across documents, canvases, PDFs, and video.", - "dependencies": ["chat", "@veltdev/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@veltdev/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "VELT_API_KEY": "", "VELT_WEBHOOK_SECRET": "" @@ -2176,7 +2620,11 @@ "type": "registry:item", "title": "Sendblue", "description": "Send and receive iMessage, SMS, and RCS through Sendblue.", - "dependencies": ["chat", "chat-adapter-sendblue", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "chat-adapter-sendblue", + "@chat-adapter/state-memory" + ], "envVars": { "SENDBLUE_API_KEY": "", "SENDBLUE_API_SECRET": "", @@ -2201,7 +2649,11 @@ "type": "registry:item", "title": "Novu", "description": "Reach Slack, Teams, WhatsApp, Telegram, and email through Novu.", - "dependencies": ["chat", "@novu/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@novu/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-novu.ts", @@ -2225,7 +2677,11 @@ "type": "registry:item", "title": "Liveblocks", "description": "Bring your agent into Liveblocks comment threads, mentions, and reactions.", - "dependencies": ["chat", "@liveblocks/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@liveblocks/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "LIVEBLOCKS_SECRET_KEY": "", "LIVEBLOCKS_WEBHOOK_SECRET": "" @@ -2249,15 +2705,23 @@ "type": "registry:item", "title": "Linq", "description": "Connect an eve agent to iMessage and SMS through Linq with guided Connect or portable setup.", - "dependencies": ["@vercel/connect@>=0.9.0"], - "devDependencies": ["vercel@>=58.5.1"], + "dependencies": [ + "@vercel/connect@>=0.9.0" + ], + "devDependencies": [ + "vercel@>=58.5.1" + ], "meta": { "eve": { "setup": { "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "linq"] + "args": [ + "integration", + "setup", + "linq" + ] }, "requires": ">=0.41.0", "docs": "/docs/channels/linq", @@ -2270,7 +2734,11 @@ "type": "registry:item", "title": "Kapso", "description": "Managed WhatsApp conversations, media, buttons, and history through Kapso.", - "dependencies": ["chat", "@kapso/chat-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@kapso/chat-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "KAPSO_API_KEY": "", "KAPSO_PHONE_NUMBER_ID": "", @@ -2295,14 +2763,20 @@ "type": "registry:item", "title": "Photon iMessage", "description": "Connect an eve agent to iMessage through Photon with guided project and phone setup.", - "dependencies": ["@vercel/connect@>=0.5.0"], + "dependencies": [ + "@vercel/connect@>=0.5.0" + ], "meta": { "eve": { "setup": { "command": "eve", "package": "eve", "bin": "eve", - "args": ["integration", "setup", "photon"] + "args": [ + "integration", + "setup", + "photon" + ] }, "requires": ">=0.33.0", "docs": "/docs/channels/photon", @@ -2315,7 +2789,11 @@ "type": "registry:item", "title": "Dial", "description": "Give your agent a phone number for SMS, MMS, iMessage, and voice transcripts.", - "dependencies": ["chat", "@getdial/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@getdial/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "DIAL_API_KEY": "", "DIAL_FROM_NUMBER_ID": "", @@ -2340,7 +2818,11 @@ "type": "registry:item", "title": "AgentPhone", "description": "SMS, MMS, iMessage, and voice conversations through AgentPhone.", - "dependencies": ["chat", "@agentphone/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@agentphone/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "AGENTPHONE_API_KEY": "", "AGENTPHONE_AGENT_ID": "", @@ -2365,7 +2847,11 @@ "type": "registry:item", "title": "Lark / Feishu", "description": "Lark and Feishu chats with native card streaming via the Chat SDK.", - "dependencies": ["chat", "@larksuite/vercel-chat-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@larksuite/vercel-chat-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "LARK_APP_ID": "", "LARK_APP_SECRET": "" @@ -2389,7 +2875,11 @@ "type": "registry:item", "title": "Beeper", "description": "Matrix rooms and bridged messaging networks through Beeper.", - "dependencies": ["chat", "@beeper/chat-adapter-matrix", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@beeper/chat-adapter-matrix", + "@chat-adapter/state-memory" + ], "files": [ { "path": "registry/channels/chat-sdk-beeper.ts", @@ -2413,7 +2903,11 @@ "type": "registry:item", "title": "Resend", "description": "Send and receive threaded email through Resend via the Chat SDK.", - "dependencies": ["chat", "@resend/chat-sdk-adapter", "@chat-adapter/state-memory"], + "dependencies": [ + "chat", + "@resend/chat-sdk-adapter", + "@chat-adapter/state-memory" + ], "envVars": { "RESEND_API_KEY": "", "RESEND_WEBHOOK_SECRET": "", @@ -2438,7 +2932,11 @@ "type": "registry:item", "title": "Gmail", "description": "Turn labelled Gmail threads into agent conversations via the Chat SDK.", - "dependencies": ["chat", "@chat-adapter/gmail", "@chat-adapter/state-redis"], + "dependencies": [ + "chat", + "@chat-adapter/gmail", + "@chat-adapter/state-redis" + ], "envVars": { "REDIS_URL": "", "GMAIL_MAILBOX": "", @@ -2475,7 +2973,9 @@ "type": "registry:item", "title": "Braintrust", "description": "Trace eve agent sessions in Braintrust, including turns, steps, tool calls, and subagent interactions.", - "dependencies": ["braintrust@^3.29.0"], + "dependencies": [ + "braintrust@^3.29.0" + ], "envVars": { "BRAINTRUST_API_KEY": "" }, @@ -2516,7 +3016,9 @@ "type": "registry:item", "title": "Sentry", "description": "Send agent traces to Sentry's OTLP endpoint for tracing and debugging.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "envVars": { "SENTRY_OTLP_TRACES_ENDPOINT": "", "SENTRY_PUBLIC_KEY": "" @@ -2534,7 +3036,9 @@ "type": "registry:item", "title": "Datadog", "description": "Export agent traces to Datadog APM alongside the rest of your stack.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "envVars": { "DATADOG_OTLP_TRACES_ENDPOINT": "", "DD_API_KEY": "" @@ -2552,7 +3056,9 @@ "type": "registry:item", "title": "Honeycomb", "description": "Send OpenTelemetry traces to Honeycomb and query every agent turn.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "envVars": { "HONEYCOMB_API_KEY": "" }, @@ -2569,7 +3075,9 @@ "type": "registry:item", "title": "Arize", "description": "Export traces to Arize AX for LLM observability and evaluation.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "envVars": { "ARIZE_SPACE_ID": "", "ARIZE_API_KEY": "" @@ -2587,7 +3095,9 @@ "type": "registry:item", "title": "Raindrop", "description": "Send agent traces to Raindrop to detect and debug AI product issues.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "envVars": { "RAINDROP_WRITE_KEY": "" }, @@ -2604,7 +3114,9 @@ "type": "registry:item", "title": "Jaeger", "description": "Trace your agent with a local or self-hosted Jaeger OTLP backend.", - "dependencies": ["@vercel/otel"], + "dependencies": [ + "@vercel/otel" + ], "files": [ { "path": "registry/instrumentation/jaeger.ts", diff --git a/packages/eve/src/setup/scaffold/create/web-template.ts b/packages/eve/src/setup/scaffold/create/web-template.ts index 6217be7d1d..8d4f19463f 100644 --- a/packages/eve/src/setup/scaffold/create/web-template.ts +++ b/packages/eve/src/setup/scaffold/create/web-template.ts @@ -9,7 +9,7 @@ export const WEB_APP_TEMPLATE_FILES = { "app/apple-icon.tsx": 'import { ImageResponse } from "next/og";\n\nexport const size = {\n width: 180,\n height: 180,\n};\n\nexport const contentType = "image/png";\n\nexport default function AppleIcon() {\n return new ImageResponse(\n \n \n \n ,\n size,\n );\n}\n', "app/eve-agent.ts": - "/** Named workspace agent selected by the Web Chat installer. */\nexport const WEB_CHAT_AGENT: string | undefined = undefined;\n", + '/** Named workspace agent selected by the Web Chat installer. */\nexport const WEB_CHAT_AGENT: string | undefined = undefined;\n', "app/globals.css": '@import "tailwindcss";\n@source "../node_modules/streamdown/dist/*.js";\n\n@theme inline {\n --color-background: var(--background);\n --color-foreground: var(--foreground);\n --color-card: var(--card);\n --color-card-foreground: var(--card-foreground);\n --color-popover: var(--popover);\n --color-popover-foreground: var(--popover-foreground);\n --color-primary: var(--primary);\n --color-primary-foreground: var(--primary-foreground);\n --color-secondary: var(--secondary);\n --color-secondary-foreground: var(--secondary-foreground);\n --color-muted: var(--muted);\n --color-muted-foreground: var(--muted-foreground);\n --color-accent: var(--accent);\n --color-accent-foreground: var(--accent-foreground);\n --color-destructive: var(--destructive);\n --color-border: var(--border);\n --color-input: var(--input);\n --color-ring: var(--ring);\n --radius-sm: calc(var(--radius) - 4px);\n --radius-md: calc(var(--radius) - 2px);\n --radius-lg: var(--radius);\n --radius-xl: calc(var(--radius) + 4px);\n --font-sans: "Geist", "Geist Fallback", ui-sans-serif, system-ui, sans-serif;\n --font-mono: "Geist Mono", "Geist Mono Fallback", ui-monospace, monospace;\n}\n\n:root {\n color-scheme: light;\n /* Soft neutral page with white elevated surfaces so cards/composer pop. */\n --background: oklch(0.971 0 0);\n --foreground: oklch(0.16 0 0);\n --card: oklch(1 0 0);\n --card-foreground: oklch(0.16 0 0);\n --popover: oklch(1 0 0);\n --popover-foreground: oklch(0.16 0 0);\n --primary: oklch(0.19 0 0);\n --primary-foreground: oklch(0.985 0 0);\n --secondary: oklch(0.94 0 0);\n --secondary-foreground: oklch(0.19 0 0);\n --muted: oklch(0.94 0 0);\n --muted-foreground: oklch(0.6 0 0);\n --accent: oklch(0.94 0 0);\n --accent-foreground: oklch(0.19 0 0);\n --destructive: oklch(0.577 0.245 27.325);\n --border: oklch(0.916 0 0);\n --input: oklch(0.916 0 0);\n --ring: oklch(0.708 0 0);\n --radius: 0.625rem;\n}\n\n@media (prefers-color-scheme: dark) {\n :root {\n color-scheme: dark;\n --background: oklch(0.145 0 0);\n --foreground: oklch(0.985 0 0);\n --card: oklch(0.205 0 0);\n --card-foreground: oklch(0.985 0 0);\n --popover: oklch(0.205 0 0);\n --popover-foreground: oklch(0.985 0 0);\n --primary: oklch(0.922 0 0);\n --primary-foreground: oklch(0.205 0 0);\n --secondary: oklch(0.269 0 0);\n --secondary-foreground: oklch(0.985 0 0);\n --muted: oklch(0.269 0 0);\n --muted-foreground: oklch(0.708 0 0);\n --accent: oklch(0.269 0 0);\n --accent-foreground: oklch(0.985 0 0);\n --destructive: oklch(0.704 0.191 22.216);\n --border: oklch(1 0 0 / 10%);\n --input: oklch(1 0 0 / 15%);\n --ring: oklch(0.556 0 0);\n }\n}\n\n* {\n border-color: var(--border);\n}\n\nhtml {\n height: 100%;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\nbody {\n min-height: 100%;\n margin: 0;\n background: var(--background);\n font-family: var(--font-sans);\n}\n\nbutton,\ninput,\ntextarea {\n font: inherit;\n}\n', "app/icon.svg": @@ -88,10 +88,8 @@ export const WEB_APP_TEMPLATE_FILES = { } as const; export const WEB_CHANNEL_TEMPLATES = { - default: - 'import { eveChannel } from "eve/channels/eve";\nimport { localDev, placeholderAuth, vercelOidc } from "eve/channels/auth";\n\nexport default eveChannel({\n auth: [\n // Lets the eve TUI and your Vercel deployments reach the deployed agent.\n vercelOidc(),\n // Open on localhost for `eve dev` and the REPL; ignored in production.\n localDev(),\n // This placeholder will not allow browser requests in production.\n // Replace it with your app\'s auth provider, like Auth.js or Clerk,\n // or use none() for a public demo.\n placeholderAuth(),\n ],\n});\n', - "sign-in-with-vercel": - 'import { eveChannel } from "eve/channels/eve";\nimport { localDev, type AuthFn, vercelOidc } from "eve/channels/auth";\nimport { auth } from "@/lib/auth";\n\nconst betterAuthSession: AuthFn = async (request) => {\n const session = await auth.api.getSession({ headers: request.headers });\n if (!session) return null;\n\n const attributes: Record = {\n email: session.user.email,\n name: session.user.name,\n };\n if (session.user.image) {\n attributes.picture = session.user.image;\n }\n\n return {\n attributes,\n authenticator: "better-auth:vercel",\n principalId: session.user.id,\n principalType: "user",\n };\n};\n\nexport default eveChannel({\n auth: [betterAuthSession, vercelOidc(), localDev()],\n});\n', + default: 'import { eveChannel } from "eve/channels/eve";\nimport { localDev, placeholderAuth, vercelOidc } from "eve/channels/auth";\n\nexport default eveChannel({\n auth: [\n // Lets the eve TUI and your Vercel deployments reach the deployed agent.\n vercelOidc(),\n // Open on localhost for `eve dev` and the REPL; ignored in production.\n localDev(),\n // This placeholder will not allow browser requests in production.\n // Replace it with your app\'s auth provider, like Auth.js or Clerk,\n // or use none() for a public demo.\n placeholderAuth(),\n ],\n});\n', + "sign-in-with-vercel": 'import { eveChannel } from "eve/channels/eve";\nimport { localDev, type AuthFn, vercelOidc } from "eve/channels/auth";\nimport { auth } from "@/lib/auth";\n\nconst betterAuthSession: AuthFn = async (request) => {\n const session = await auth.api.getSession({ headers: request.headers });\n if (!session) return null;\n\n const attributes: Record = {\n email: session.user.email,\n name: session.user.name,\n };\n if (session.user.image) {\n attributes.picture = session.user.image;\n }\n\n return {\n attributes,\n authenticator: "better-auth:vercel",\n principalId: session.user.id,\n principalType: "user",\n };\n};\n\nexport default eveChannel({\n auth: [betterAuthSession, vercelOidc(), localDev()],\n});\n', } as const; export const WEB_APP_SIGN_IN_WITH_VERCEL_TEMPLATE_FILES = { From 857d622d6f5e4ba01c62e129d60dae4fad9d8bc6 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:33:42 -0400 Subject: [PATCH 7/9] fix(eve-bench): keep the unit tier independent of built eve The root test-unit job does not build eve. Typecheck runs in the typecheck job, which builds dependencies first, and the tool tests that import eve's exports now run with the integration tier. Signed-off-by: Rui Conti --- packages/eve-bench/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index 47e2ce2426..c41d63a168 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -9,10 +9,10 @@ "type": "module", "scripts": { "bench": "node src/cli.ts", - "test": "pnpm run test:unit && pnpm run test:integration", - "test:unit": "pnpm run typecheck && node --test src/core/*.test.ts src/harnesses/**/*.test.ts src/tools/*.test.ts", - "test:integration": "node --test test/scenarios/*.test.ts", - "test:scenario": "node --test test/scenarios/*.test.ts", + "test": "pnpm run typecheck && pnpm run test:unit && pnpm run test:integration", + "test:unit": "node --test src/core/*.test.ts src/harnesses/*/*.test.ts", + "test:integration": "node --test test/scenarios/*.test.ts src/tools/*.test.ts", + "test:scenario": "node --test test/scenarios/*.test.ts src/tools/*.test.ts", "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, From 23f2bc5d13cf64592cef6fb016bb2f88dce7ece9 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:40:29 -0400 Subject: [PATCH 8/9] fix(eve-bench): run integration tests after eve builds The integration tier imports eve's built exports. Without a turbo dependency it raced a concurrent eve build that cleans dist, matching packages/eve-code. Signed-off-by: Rui Conti --- packages/eve-bench/turbo.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 packages/eve-bench/turbo.json diff --git a/packages/eve-bench/turbo.json b/packages/eve-bench/turbo.json new file mode 100644 index 0000000000..d77ff0acd1 --- /dev/null +++ b/packages/eve-bench/turbo.json @@ -0,0 +1,11 @@ +{ + "extends": ["//"], + "tasks": { + "test:integration": { + "dependsOn": ["eve#build"] + }, + "test:scenario": { + "dependsOn": ["^build"] + } + } +} From b997916bcd94427ba2571dec636c1fd924eacb55 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Fri, 25 Sep 2026 00:52:33 -0400 Subject: [PATCH 9/9] fix(eve-bench): skip POSIX-only integration tests on Windows eve-bench drives Linux task containers and its integration tests stub POSIX tools, so they never passed on the Windows integration job. Signed-off-by: Rui Conti --- packages/eve-bench/package.json | 4 ++-- packages/eve-bench/test/posix-only.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 packages/eve-bench/test/posix-only.ts diff --git a/packages/eve-bench/package.json b/packages/eve-bench/package.json index c41d63a168..eb052a852d 100644 --- a/packages/eve-bench/package.json +++ b/packages/eve-bench/package.json @@ -11,8 +11,8 @@ "bench": "node src/cli.ts", "test": "pnpm run typecheck && pnpm run test:unit && pnpm run test:integration", "test:unit": "node --test src/core/*.test.ts src/harnesses/*/*.test.ts", - "test:integration": "node --test test/scenarios/*.test.ts src/tools/*.test.ts", - "test:scenario": "node --test test/scenarios/*.test.ts src/tools/*.test.ts", + "test:integration": "node test/posix-only.ts test/scenarios/*.test.ts src/tools/*.test.ts", + "test:scenario": "node test/posix-only.ts test/scenarios/*.test.ts src/tools/*.test.ts", "test:stress": "node --test test/stress/*.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, diff --git a/packages/eve-bench/test/posix-only.ts b/packages/eve-bench/test/posix-only.ts new file mode 100644 index 0000000000..a8bc93183c --- /dev/null +++ b/packages/eve-bench/test/posix-only.ts @@ -0,0 +1,12 @@ +// eve-bench drives Linux task containers, and its integration tests stub POSIX +// tools (sh, shebang scripts, process groups), so they cannot run on Windows hosts. +import { spawnSync } from "node:child_process"; + +if (process.platform === "win32") { + console.log("Skipping eve-bench integration tests: they require a POSIX host."); + process.exit(0); +} +const { status } = spawnSync(process.execPath, ["--test", ...process.argv.slice(2)], { + stdio: "inherit", +}); +process.exit(status ?? 1);