feat(eve-bench): add zero-dependency benchmark runner

.github/workflows/bench.yml

@@ -0,0 +1,159 @@
+name: eve Benchmarks
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - "packages/eve-bench/**"
+      - research/eve-bench-runner.md
+  workflow_dispatch:
+    inputs:
+      model:
+        description: Model to benchmark
+        required: false
+        default: zai/glm-5.2
+        type: string
+      cohort:
+        description: Task cohort to run
+        required: false
+        default: smoke
+        type: string
+      base_job_artifact:
+        description: Optional artifact containing a base job to compare
+        required: false
+        type: string
+
+concurrency:
+  group: eve-bench-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  oracle-smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: packages/eve-bench
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+        working-directory: ${{ github.workspace }}
+
+      - name: Run tests
+        run: pnpm run test
+
+      - name: Sync benchmark tasks
+        run: pnpm bench tasks sync --json
+
+      - name: Run oracle smoke benchmark
+        run: pnpm bench run --harness oracle --cohort smoke --concurrency 5 --job oracle-smoke --format junit > junit.xml
+
+      - name: Print oracle smoke report
+        run: pnpm bench report oracle-smoke
+
+      - name: Upload oracle smoke artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: eve-bench-oracle-smoke
+          path: |
+            packages/eve-bench/.generated/jobs/oracle-smoke
+            packages/eve-bench/junit.xml
+          include-hidden-files: true
+          if-no-files-found: warn
+
+      - name: Require every oracle attempt to resolve
+        run: |
+          node -e '
+            const result = require("./.generated/jobs/oracle-smoke/result.json");
+            const { attempts, resolved } = result.summary;
+            if (resolved < attempts) {
+              console.error(`Oracle resolved ${resolved} of ${attempts} attempts`);
+              process.exit(1);
+            }
+          '
+
+  # AI_GATEWAY_API_KEY is available to pull requests from branches in this repository, not forks.
+  eve-smoke:
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bench:eve') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+    defaults:
+      run:
+        working-directory: packages/eve-bench
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+        working-directory: ${{ github.workspace }}
+
+      - name: Build eve package
+        run: pnpm --filter eve run build
+        working-directory: ${{ github.workspace }}
+
+      - name: Run eve smoke benchmark
+        env:
+          MODEL: ${{ inputs.model || 'zai/glm-5.2' }}
+          COHORT: ${{ inputs.cohort || 'smoke' }}
+        run: pnpm bench run --harness eve --model "$MODEL" --cohort "$COHORT" --attempts 1 --concurrency 5 --job eve-smoke
+
+      - name: Download base job artifact
+        if: ${{ inputs.base_job_artifact }}
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: ${{ inputs.base_job_artifact }}
+          path: packages/eve-bench/.generated/base-job
+
+      - name: Compare with base job
+        if: ${{ inputs.base_job_artifact }}
+        run: |
+          pnpm bench diff .generated/base-job .generated/jobs/eve-smoke | tee diff.txt
+          {
+            echo "## Benchmark diff"
+            echo '```text'
+            cat diff.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload eve smoke artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: eve-bench-eve-smoke
+          path: packages/eve-bench/.generated/jobs/eve-smoke
+          include-hidden-files: true
+          if-no-files-found: warn

.gitignore

@@ -19,6 +19,7 @@ coverage/
 .extension-contracts-cache/
 packages/eve/.workflow-vitest/
 packages/eve/.generated/
+packages/eve-bench/.generated/
 playwright-report/
 test-results/
 

packages/eve-bench/README.md

@@ -0,0 +1,171 @@
+# @eve/bench
+
+Terminal-Bench harness benchmarks for eve, run by a zero-dependency Node runner.
+
+The package owns the harness under evaluation (`agent/`), pinned dataset locks
+and task cohorts (`datasets/`), and the runner (`src/`). Tasks keep the
+Terminal-Bench layout and reward contract, so results stay comparable with
+published runs. See `research/eve-bench-runner.md` for the design.
+
+## Prerequisites
+
+- Node.js 24 and pnpm
+- Docker
+- model-provider credentials in the environment, such as `AI_GATEWAY_API_KEY`
+- a built local eve package when using `--eve local`; run `pnpm build` from the
+  repository root first
+
+Everything below runs from this directory with `pnpm bench` (alias for
+`node src/cli.ts`).
+
+## Run
+
+```bash
+pnpm bench tasks sync                                   # fetch pinned datasets into .generated/datasets
+pnpm bench tasks list --cohort smoke                    # print a cohort's task names
+
+AI_GATEWAY_API_KEY=... pnpm bench run --model zai/glm-5.2 --cohort smoke --attempts 3 --concurrency 8
+pnpm bench run --harness oracle --cohort smoke           # reference solutions; validates the runner, no model
+pnpm bench run --model zai/glm-5.2 --task fix-git --eve 0.35.0   # benchmark a published eve release
+pnpm bench run --harness oracle --task-dir ./path/to/task # run a local Terminal-Bench task
+
+# Build and cache a local e0 snapshot without calling a model.
+pnpm bench prepare --harness e0 --agent /path/to/internal-agents/agents/e0 \
+  --model openai/gpt-5.6-terra
+
+# Run one e0 smoke task. The source app must have installed workspace dependencies.
+AI_GATEWAY_API_KEY=... pnpm bench run --harness e0 \
+  --agent /path/to/internal-agents/agents/e0 \
+  --model deepseek/deepseek-v4-pro --cohort smoke --task fix-git --job e0-deepseek
+
+pnpm bench report <job> --format junit --out junit.xml
+pnpm bench diff <base-job> <candidate-job>
+```
+
+Every command accepts `--json` and never prompts. Jobs live in
+`.generated/jobs/<job>/`. Re-running the same job resumes only when the model,
+harness, bundle provenance, dataset, tasks, and attempt count match. Invalid or
+mismatched trial records fail closed instead of being overwritten. Ctrl-C
+aborts in-flight trials and removes their containers.
+
+Pass `--task-dir <path>` more than once to run one or more local Terminal-Bench
+task directories. A local-only run does not sync a dataset and records its
+dataset as `local`.
+
+On an ARM host, an amd64-only prebuilt task image can make native CLI agents run
+through emulation. Pass `--native-build` to build the selected task's reviewed
+`environment/Dockerfile` for the host architecture instead. Use the flag for
+every comparison arm, and do not compare its scores or latency directly with
+prebuilt-image runs. `job.json` records `local-dockerfile` instead of the
+prebuilt image name.
+
+## How a trial runs
+
+1. Pull the task's prebuilt image (or build `environment/Dockerfile` once).
+2. Start one container, upload the harness bundle to `/installed-agent`, and
+   run the harness command under the task's agent timeout.
+3. Upload `tests/` and run `tests/test.sh` under the verifier timeout. The
+   verifier always runs, so the reward reflects what the agent left behind.
+4. Read `/logs/verifier/reward.txt`, copy `/logs/agent` and `/logs/verifier`
+   to the trial directory, remove the container, and write `trial.json`.
+
+Each runner-owned log (`container.log`, `agent.log`, and `verifier.log`) is
+capped at 64 MiB. The runner appends a truncation notice when a log exceeds the
+cap.
+
+Containers carry the labels `eve-bench=1`, `eve-bench.job=<job>`, and
+`eve-bench.task=<task>`. Find any leftovers with:
+
+```bash
+docker ps -a --filter label=eve-bench=1
+```
+
+At the start of a job, the runner removes stale containers with the same
+`eve-bench.job` label before starting new trials.
+
+## Stress tests
+
+Run the Docker-backed stress suite separately from the default test suite:
+
+```bash
+pnpm run test:stress
+```
+
+The suite requires Docker and takes about 30 seconds. It covers the happy path,
+agent and verifier timeouts, a bad image, SIGINT abort and resume, and concurrent
+trials.
+
+## Harnesses
+
+- `eve` (default): builds `agent/` against the local checkout (`--eve local`)
+  or an exact published version, then packages the Nitro output and a small
+  runner that drives the server over HTTP. Build eve from the repository root
+  with `pnpm build` before using `--eve local`. For cache lookup, `prepare()`
+  derives the local eve identity from `packages/eve/dist` instead of packing
+  first. The cache lives in `.generated/cache/`. Each trial uploads the bundle
+  and only the Linux Node binary matching the task container's architecture,
+  for about 128 MB total.
+
+  The agent's tools operate on the task working directory through
+  `EVE_BENCH_TASK_WORKDIR`; Docker is the isolation boundary, so they bypass
+  eve's nested sandbox on purpose.
+
+- `e0`: snapshots an installed local e0 app and its source-backed `eve-code`
+  extension. The bundle records source, dependency, and installed eve hashes.
+  It preserves e0's instructions, coding capabilities, tools, skills, and
+  authored worker model. The adapter replaces production sandbox and transport
+  integrations with the task-container backend and a local eve channel. It
+  removes production Connect configuration, channels, connections, schedules,
+  instrumentation, and sandbox bootstrap. `--model` changes the root model,
+  not e0's authored worker model.
+
+- `pi`, `opencode`, and `codex`: prepare explicitly versioned CLI bundles on the
+  host and route full provider-qualified model IDs through an HTTPS API base.
+  Pass an exact `--version`. The default base is Vercel AI Gateway. Codex uses
+  the Responses protocol; pi and OpenCode use chat completions. Hermes fails
+  explicitly until a pinned Python/uv bundle and verified custom-provider
+  routing are implemented.
+
+- `oracle`: runs each task's `solution/solve.sh`. Use it to check the runner
+  and as the reward ceiling.
+
+Adding a harness means implementing `src/core/harness.ts` (prepare a
+directory, return a command and env) and registering it in `src/cli.ts`.
+
+## Agent tools
+
+`src/tools/` exposes `run_bench`, `get_bench_result`, `diff_bench`, and
+`list_bench_tasks` as eve `defineTool` definitions over the same core used by
+the CLI. Their results use the same `JobResult`, `TaskDelta`, and task-list JSON
+as the CLI. Add a file to an agent's `tools/` directory that re-exports a
+definition by relative path, or copy the definition there.
+
+## Datasets and cohorts
+
+`datasets/<name>-<version>.json` pins a git URL, commit, and the task paths in
+it. `datasets/cohorts/<name>.json` names a reviewed task subset and the reason
+it exists. Compare base and candidate runs with the same cohort, model,
+attempt count, and eve source.
+
+## CI
+
+`.github/workflows/bench.yml` runs the oracle smoke cohort on pull requests that
+touch this package or its design document and requires every task attempt to
+resolve. It uploads the job directory and `junit.xml` as artifacts.
+
+The real-model `eve-smoke` job runs through `workflow_dispatch` or when the
+`bench:eve` label is added to a pull request. It accepts `model` and `cohort`
+inputs for manual runs and requires `AI_GATEWAY_API_KEY`, which is unavailable
+to pull requests from forks. It uploads its job directory and can optionally
+diff the result against a downloaded base-job artifact.
+
+## Layout
+
+- `agent/`: harness behavior under evaluation (model binding, instructions, tools)
+- `src/core/`: runner library with no dependency on eve or third parties
+- `src/harnesses/`: `eve`, `e0`, CLI competitors, and `oracle`
+- `src/tools/`: eve tool definitions over the runner core
+- `src/options.ts`: run options shared by the CLI and tools
+- `src/cli.ts`: `eve-bench` CLI
+- `datasets/`: locks and cohorts
+- `.generated/`: datasets, bundle cache, and job output; never committed

packages/eve-bench/agent/agent.ts

@@ -0,0 +1,10 @@
+import { defineAgent } from "eve";
+
+const model = process.env.EVE_BENCH_MODEL;
+if (model === undefined || model.length === 0) {
+  throw new Error("EVE_BENCH_MODEL must be set by the eve-bench harness.");
+}
+
+export default defineAgent({
+  model,
+});

packages/eve-bench/agent/instructions.md

@@ -0,0 +1,11 @@
+You are an autonomous terminal task agent.
+
+Work until the requested task is complete or conclusively cannot be completed.
+
+- Inspect relevant files before modifying them.
+- Use the available terminal and file tools to work directly in the task environment.
+- Make the smallest correct changes that satisfy the task.
+- Run relevant validation after making changes.
+- Do not ask the user questions.
+- Do not use network access unless the task explicitly requires it.
+- Finish with a concise description of the completed work and validation.

packages/eve-bench/agent/task-path.ts

@@ -0,0 +1,18 @@
+import { resolve } from "node:path";
+
+export function taskRoot(): string {
+  const root = process.env.EVE_BENCH_TASK_WORKDIR;
+  if (root === undefined || root.length === 0) {
+    throw new Error("EVE_BENCH_TASK_WORKDIR must be set by the eve-bench harness.");
+  }
+  return resolve(root);
+}
+
+export function resolveTaskPath(path = "."): string {
+  const root = taskRoot();
+  const target = resolve(root, path);
+  if (target !== root && !target.startsWith(`${root}/`)) {
+    throw new Error("Task paths must stay inside the task working directory.");
+  }
+  return target;
+}

packages/eve-bench/agent/tools/ask_question.ts

@@ -0,0 +1,3 @@
+import { disableTool } from "eve/tools";
+
+export default disableTool();