chore: full-matrix benchmark run (do not merge)

.changeset/eval-artifact-timing.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`eve eval` artifacts now record each eval's `startedAt` and `completedAt` in `summary.json`, `results.jsonl`, and the per-eval detail file.

.github/scripts/discover-e2e-fixtures.mjs

@@ -146,7 +146,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
   else process.stdout.write(result.lines);
 }
 
-function discoverEvalIds(root) {
+export function discoverEvalIds(root) {
   const evalIds = [];
   const visit = (directory) => {
     for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) =>

.github/scripts/stage-benchmark-leg.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Stages one eval-benchmark leg for upload as $RUNNER_TEMP/leg:
+#   meta.json      leg identity and runner context
+#   expected.json  `eve eval --list --json` output, when the run got that far
+#   run/           the leg's single `.eve/evals/<timestamp>` tree, when one exists
+# The publish job (scripts/eval-benchmark/records.mjs) reads this layout.
+set -euo pipefail
+
+leg="$RUNNER_TEMP/leg"
+rm -rf "$leg"
+mkdir -p "$leg"
+
+jq -n \
+  --arg leg_id "$LEG_ID" \
+  --arg sha "$PLANNED_SHA" \
+  --arg model_id "$MODEL_ID" \
+  --arg world "$WORLD" \
+  --argjson attempt "$ATTEMPT" \
+  --arg runner_os "$RUNNER_OS_NAME" \
+  '{leg_id: $leg_id, sha: $sha, model_id: $model_id, world: $world, attempt: $attempt, runner_os: $runner_os}' \
+  >"$leg/meta.json"
+
+if [ -s "$RUNNER_TEMP/expected.json" ]; then
+  cp "$RUNNER_TEMP/expected.json" "$leg/expected.json"
+fi
+
+shopt -s nullglob
+runs=("$FIXTURE_DIR"/.eve/evals/*/)
+if [ "${#runs[@]}" -gt 1 ]; then
+  echo "::error::Expected one .eve/evals tree in $FIXTURE_DIR, found ${#runs[@]}."
+  exit 1
+fi
+if [ "${#runs[@]}" -eq 1 ]; then
+  cp -R "${runs[0]}" "$leg/run"
+fi

.github/workflows/ci.yml

@@ -98,6 +98,9 @@ jobs:
       - name: Test workflow helpers
         run: node --test .github/scripts/*.test.mjs
 
+      - name: Test eval benchmark scripts
+        run: pnpm test:eval-benchmark
+
   test-integration:
     name: test-integration (${{ matrix.os }})
     runs-on: ${{ matrix.os }}