diff --git a/.github/actions/bencher-track/action.yml b/.github/actions/bencher-track/action.yml index 498f19aff..33198baf8 100644 --- a/.github/actions/bencher-track/action.yml +++ b/.github/actions/bencher-track/action.yml @@ -12,7 +12,7 @@ inputs: description: Bencher testbed slug. required: true workload: - description: Workload key for the `bencher-thresholds-reset-` tag (e.g. ix-compile, aiur). + description: Workload key for the `bencher-thresholds-reset-` tag (the backend testbed minus its runner-arch suffix, e.g. zisk-check-execute). required: true file: description: Bencher Metric Format JSON file to upload. diff --git a/.github/actions/install-sp1/action.yml b/.github/actions/install-sp1/action.yml new file mode 100644 index 000000000..91ad4d6bd --- /dev/null +++ b/.github/actions/install-sp1/action.yml @@ -0,0 +1,31 @@ +name: Install SP1 +description: >- + Install the system build deps and the SP1 zkVM toolchain (sp1up) needed to + build and run the SP1 host. Assumes a Rust toolchain is already set up. + +runs: + using: composite + steps: + # The shared zkVM apt superset (the ZisK book's full Ubuntu list — the + # prebuilt cargo tooling and proofman's C++ link OpenMPI/OpenMP/GMP/ + # nlohmann-json/nasm/secp256k1/… — plus pkg-config + libssl-dev for SP1's + # host crates). The Nix shells provided this; a bare runner doesn't. + - name: Install system build deps + shell: bash + run: | + # Some warpbuild images ship an unreachable azure mirror that hangs + # `apt-get update`; drop it first (no-op elsewhere). + sudo sed -i '/azure\.archive\.ubuntu\.com/d' /etc/apt/apt-mirrors.txt 2>/dev/null || true + sudo apt-get update + sudo apt-get install -y \ + xz-utils jq curl build-essential qemu-system libomp-dev libgmp-dev \ + nlohmann-json3-dev protobuf-compiler uuid-dev libgrpc++-dev \ + libsecp256k1-dev libsodium-dev libpqxx-dev nasm libopenmpi-dev \ + openmpi-bin openmpi-common libclang-dev clang gcc-riscv64-unknown-elf \ + pkg-config libssl-dev + - name: Install SP1 toolchain (sp1up, latest) + shell: bash + run: | + curl -L https://sp1up.succinct.xyz | bash + ~/.sp1/bin/sp1up + echo "$HOME/.sp1/bin" >> "$GITHUB_PATH" diff --git a/.github/actions/install-zisk/action.yml b/.github/actions/install-zisk/action.yml new file mode 100644 index 000000000..604fcec82 --- /dev/null +++ b/.github/actions/install-zisk/action.yml @@ -0,0 +1,95 @@ +name: Install Zisk +description: >- + Install the system build deps, the ZisK zkVM toolchain (ziskup, CPU build), + and — unless `proving-key: false` — the fork-matching proving key needed to + RUN the Zisk host. Execute needs the key too (zisk-host's `client.setup()` + loads the circuit's const-tree files before either the execute or the prove + branch), but BUILDING the host does not, so build-only callers skip the + ~3 GB download + const-tree regeneration. Assumes a Rust toolchain is + already set up. + +inputs: + proving-key: + description: >- + Install the fork-matching proving key (required to execute or prove; + not needed to build). Set false for build-only jobs. + required: false + default: "true" + +runs: + using: composite + steps: + # The shared zkVM apt superset — the ZisK book's full Ubuntu list (prebuilt + # cargo-zisk + proofman's C++ link OpenMPI/OpenMP/GMP/nlohmann-json/nasm/ + # secp256k1/…), kept identical to install-sp1 so a host that links both is + # covered. The Nix shells provided this; a bare runner doesn't. + - name: Install system build deps + shell: bash + run: | + # Some warpbuild images ship an unreachable azure mirror that hangs + # `apt-get update`; drop it first (no-op elsewhere). + sudo sed -i '/azure\.archive\.ubuntu\.com/d' /etc/apt/apt-mirrors.txt 2>/dev/null || true + sudo apt-get update + sudo apt-get install -y \ + xz-utils jq curl build-essential qemu-system libomp-dev libgmp-dev \ + nlohmann-json3-dev protobuf-compiler uuid-dev libgrpc++-dev \ + libsecp256k1-dev libsodium-dev libpqxx-dev nasm libopenmpi-dev \ + openmpi-bin openmpi-common libclang-dev clang gcc-riscv64-unknown-elf \ + pkg-config libssl-dev + # `--version 0.18.0` pins the toolchain to match our deps. Our host links the + # argumentcomputer/zisk `blake3-precompile` fork, which is based on v0.18.0 + # (its cargo-zisk has `check-setup`, used below to regenerate the key's + # const-trees). Without the pin, ziskup installs `releases/latest`, which + # resolves to upstream `v1.0.0-alpha` — a different circuit whose cargo-zisk + # dropped the `check-setup` subcommand, breaking the key step. `--cpu` picks + # the CPU build (no GPU on the runner) and `--nokey` skips ziskup's key + # install — both avoid its interactive /dev/tty prompts. We keep `--nokey` + # because the upstream `zisk-setup` bucket only carries the upstream circuit's + # key; our fork has a different circuit (extra Blake3f AIR), so we restore the + # fork-matching key from our own S3 in the next step. `--prefix $HOME/.zisk` + # pins the install where cargo-zisk's ZiskPaths fallback looks (the runner + # sets XDG_CONFIG_HOME, which would otherwise relocate it). + - name: Install Zisk toolchain (ziskup, pinned v0.18.0) + shell: bash + run: | + curl -L https://raw.githubusercontent.com/0xPolygonHermez/zisk/main/ziskup/install.sh \ + | bash -s -- --cpu --nokey -y --version 0.18.0 --prefix "$HOME/.zisk" + echo "$HOME/.zisk/bin" >> "$GITHUB_PATH" + # Pre-build the proofman C++ sys crate ALONE so its build script runs + # exactly once before any parallel zisk-host build. zisk-host pulls + # zisk-sdk as both a dependency and a build-dependency, so cargo compiles + # proofman-starks-lib-c as two units whose build scripts can run + # CONCURRENTLY — and both run `make` inside the SHARED + # ~/.cargo/git/checkouts/pil2-proofman-* source dir. On a cold runner the + # Makefile stamp is absent, so both units take the `make clean` + + # `make -j` path and race: one unit's clean deletes build/ while the + # other's g++ is mid-compile ("opening dependency file ….d: No such file + # or directory"). Building the crate solo writes the stamp; the second + # unit then skips the clean and its `make -j` is a no-op. + # (Proper fix is an flock in pil2-proofman's build.rs — upstream.) + - name: Pre-build proofman-starks-lib-c (serialize the shared make) + shell: bash + run: cargo build --release -p proofman-starks-lib-c + working-directory: zisk + # Execute still needs a proving key present: zisk-host calls `client.setup()` + # (which the SDK runs before the execute branch), and that loads the circuit's + # const-tree files. We host the fork-matching key in a public S3 bucket + # WITHOUT the const-trees — exactly like Zisk's released + # `zisk-provingkey-*.tar.gz` on `storage.googleapis.com/zisk-setup` — and + # regenerate them here with `cargo-zisk check-setup -a`, which is how `ziskup` + # itself populates them. That keeps the artifact ~3 GB (gzip) instead of + # ~48 GB. The object name carries the fork rev so a circuit change can't + # silently reuse a stale key. Public bucket → plain curl, no AWS creds. + - name: Restore Zisk proving key (fork circuit) from S3 + if: inputs.proving-key == 'true' + shell: bash + run: | + mkdir -p "$HOME/.zisk" + curl -fSL --retry 3 \ + https://argument-zisk-setup.s3.amazonaws.com/zisk-provingkey-blake3-8f9e24d5-cpu.tar.gz \ + -o /tmp/zisk-provingkey.tar.gz + tar -C "$HOME/.zisk" -xzf /tmp/zisk-provingkey.tar.gz + rm -f /tmp/zisk-provingkey.tar.gz + # Regenerate the const-tree files omitted from the artifact (CPU build, so + # no --gpu). This is the "may take a while" step ziskup prints. + cargo-zisk check-setup --proving-key "$HOME/.zisk/provingKey" -a diff --git a/.github/scripts/watchdog.sh b/.github/scripts/watchdog.sh new file mode 100755 index 000000000..528445dc1 --- /dev/null +++ b/.github/scripts/watchdog.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Hard memory cap: run in a systemd user scope with cgroup-v2 +# memory.max = and swap disabled. The kernel OOM-kills at +# the cap — SIGKILL, exit 137, which the orchestrator (`ix bench run`) +# reads as the in-flight constant's OOM. No sampler to race, nothing to +# sum: the cgroup charges the whole tree's resident memory, and cached +# allocator reservations don't count. Validated on ubuntu-latest and +# warp runners (see ix-cpu-info's cgroup-memcap.yml). +# +# watchdog.sh [args...] +# +# Needs a user systemd instance: the linger call boots one on CI +# (passwordless sudo); it no-ops locally, where a desktop session +# already provides the user manager. +set -u +ceiling_gb=${1:?ceiling_gb} +shift +[ $# -ge 1 ] || { echo "watchdog: no command given" >&2; exit 2; } + +sudo -n loginctl enable-linger "${USER:-$(id -un)}" 2>/dev/null || true +export XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$(id -u)} + +# memory.oom.group=1: on breach the kernel kills the WHOLE scope (exit +# 137 -> oom row), not just its biggest process. Without it, Zisk's ASM +# service gets singled out and the surviving host converts the memory +# kill into a clean exit 1 — which the orchestrator must treat as a +# deterministic failure. The scope's cgroup is user-delegated, so the +# write needs no sudo; if it fails, exit 2 rather than run with wrong +# kill semantics. +exec systemd-run --user --scope --quiet \ + -p MemoryMax="${ceiling_gb}G" -p MemorySwapMax=0 \ + bash -c 'echo 1 > "/sys/fs/cgroup$(cut -d: -f3- /proc/self/cgroup)/memory.oom.group" \ + || { echo "watchdog: cannot set memory.oom.group" >&2; exit 2; } + exec "$@"' watchdog "$@" diff --git a/.github/workflows/bench-main.yml b/.github/workflows/bench-main.yml index 957b51b38..4904c63ba 100644 --- a/.github/workflows/bench-main.yml +++ b/.github/workflows/bench-main.yml @@ -1,48 +1,61 @@ -name: Aiur benchmarks +name: Benchmark main -# One workflow, two benchmarks per library env, on every push to main: -# 1. compile job — `ix compile` the Lean env to a `.ixe` (compile-throughput -# metrics) and cache the `.ixe`. -# 2. prove job — restore that `.ixe` from the cache (no recompile) and -# STARK-check selected constants over it via bench-typecheck -# (Aiur execute + prove metrics). -# The prove job reuses the exact `.ixe` the compile job built, so the compiler -# runs once. Compile and prove report to separate bencher testbeds so each one's -# `--thresholds-reset` only touches its own measures. +# Benchmarks tracked on Bencher on every push to main, all reusing the one +# compiled `.ixe` so the compiler runs once. Every measurement is a single +# `ix bench run` cell (backend × env × mode) — the same orchestrator local +# runs and the !benchmark PR workflow use — converted for upload by +# `ix bench bmf`: +# 1. compile — `ix bench run --backend compile`: serialize the Lean env +# to a `.ixe` (compile-throughput metrics) and cache the +# `.ixe`. +# 2. prove — restore that `.ixe` (no recompile) and STARK-check selected +# constants over it via bench-typecheck (Aiur execute + prove). +# 3. zkvm-execute — restore that `.ixe` and execute the same constants through +# the zkVM hosts (deterministic cycle counts + +# time/throughput/RAM; proving needs a GPU, so execute-only). +# 4. ooc-check — restore that `.ixe` and run the out-of-circuit Rust kernel +# (the same kernel, out-of-circuit and parallel — far faster) +# over the whole env, tracking throughput. +# Each job reports to its own bencher testbed/workload so a threshold reset only +# touches its own measures. `ix bench run` exits 3 when the kernel REJECTS a +# constant — the red X lands on that step (no output scraping), while the clean +# rows still upload (`ix bench bmf` drops every non-ok row, so a rejected or +# OOM'd constant never becomes a bencher data point). on: push: - branches: main + branches: [main] workflow_dispatch: permissions: contents: read checks: write +# No concurrency group: push-to-main and manual dispatch only — every merged +# commit gets benchmarked; a later merge must never cancel or queue behind an +# in-flight run. + env: COMPILE_DIR: Benchmarks/Compile jobs: # Build + stage the `ix` and `bench-typecheck` binaries once, then restore - # them on the (more expensive) matrix runners. + # them on the matrix runners. `-Ctarget-cpu=native`: every job that RUNS + # these binaries must also be a warp host. build: - runs-on: ubuntu-latest + runs-on: warp-ubuntu-latest-x64-32x steps: - uses: actions/checkout@v6 # Pinned Rust toolchain + cargo cache (~/.cargo + target/, via the action's # built-in rust-cache), so `lake build`'s cargo step doesn't recompile the # Plonky3/multi-stark deps from scratch on every run. - uses: actions-rust-lang/setup-rust-toolchain@v1 - # `.cargo/config.toml` sets `-Ctarget-cpu=native`, and the binary is built - # once here (then only restored on the warp runners), so this build host's - # CPU fixes the instruction set baked in — incl. AVX-512, a big Plonky3 - # speedup. Log it so a benchmark shift can be traced to a build-CPU change. + # Log the build CPU so a benchmark shift can be traced to a host change. - name: Log build CPU run: | lscpu - grep -qw avx512f /proc/cpuinfo \ - && echo "AVX-512F: present (compiled into the binary)" \ - || echo "AVX-512F: absent" + flags=$(grep -m1 -oE 'avx2|avx512[a-z0-9_]*' /proc/cpuinfo | sort -u | tr '\n' ' ') + echo "AVX flags: ${flags:-absent}" - uses: leanprover/lean-action@v1 with: auto-config: false @@ -57,26 +70,72 @@ jobs: - uses: actions/cache/save@v5 with: path: ~/.local/bin - key: aiur-bench-bins-${{ github.sha }} + key: bench-bins-${{ github.sha }} + + # Derive the job matrices from the registry (Ix/Cli/BenchCmd.lean) + # via `ix bench ci matrix`, so the benched-env / enabled-backend fan-out is + # single-sourced instead of hand-copied per job. Runs beside the compile + # job; only the downstream matrix jobs wait on it. + plan: + needs: build + # `ci matrix` is a pure-Lean subcommand (AVX-512 lives only in the + # binary's Rust objects, which it never executes) — cheap host is fine. + # Move to a warp host if `ix` startup or this subcommand ever touches + # the FFI. + runs-on: ubuntu-latest + outputs: + bench-envs: ${{ steps.matrix.outputs.bench-envs }} + zkvm-cells: ${{ steps.matrix.outputs.zkvm-cells }} + steps: + - uses: actions/checkout@v6 + - uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ github.sha }} + fail-on-cache-miss: true + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + # Provision the toolchain so `ix` finds libleanshared (no package build). + - uses: leanprover/lean-action@v1 + with: + auto-config: false + build: false + use-github-cache: false + - id: matrix + run: | + ix bench ci matrix --kind cells > cells.json + cat cells.json + # bench-envs: the benched env names — an env name (e.g. InitStd) + # is the single identifier: `ix bench run --env`, the `.ixe` + # filename, the cache-key suffix, and the env-keyed bencher + # benchmark name. + # zkvm-cells: the zkVM subset of the cells (zisk today; sp1 comes + # back by re-enabling it in the registry). + echo "bench-envs=$(jq -c '[.[].env] | unique' cells.json)" >> "$GITHUB_OUTPUT" + echo "zkvm-cells=$(jq -c 'map(select(.backend == "zisk" or .backend == "sp1"))' cells.json)" >> "$GITHUB_OUTPUT" # Compile each library env to a `.ixe` and track compile throughput. Caches - # the `.ixe` (keyed by sha + matrix job) for the prove job to consume. + # the `.ixe` (keyed by sha + env slug) for the prove job to consume. compile: + # Explicit names on the matrix jobs: the default would append every + # matrix value (slugs, flags) to the job title. + name: compile-${{ matrix.env }} needs: build runs-on: warp-ubuntu-latest-x64-32x strategy: fail-fast: false matrix: - bench: [InitStd, Lean, Mathlib, FLT] # Add FC if updated to latest Lean + # env = the registry key (Ix/Cli/BenchCmd.lean): the `ix bench run --env` + # token, `.ixe` filename, lake target suffix (Compile), + # cache-key suffix, and bencher row name, all one spelling. + # Deliberately wider than the benched set: Lean/FLT + # compile-throughput is tracked even though no downstream job proves + # them. Add FC if updated to latest Lean. include: - - bench: Mathlib - mathlib: true - - bench: FLT - cache_pkg: flt - mathlib: true - # - bench: FC - # cache_pkg: formal_conjectures - # mathlib: true + - { env: InitStd } + - { env: Lean } + - { env: Mathlib, mathlib: true } + - { env: FLT, cache_pkg: flt, mathlib: true } + # - { env: FC, cache_pkg: formal_conjectures, mathlib: true } steps: - uses: actions/checkout@v6 with: @@ -85,11 +144,11 @@ jobs: - uses: actions/cache/restore@v5 with: path: ~/.local/bin - key: aiur-bench-bins-${{ github.sha }} + key: bench-bins-${{ github.sha }} - run: echo "$HOME/.local/bin" >> $GITHUB_PATH # FC's library env lives in a sibling `${COMPILE_DIR}FC` package dir, so # point COMPILE_DIR there for the FC matrix job. - # - if: matrix.bench == 'FC' + # - if: matrix.env == 'FC' # run: echo "COMPILE_DIR=${{ env.COMPILE_DIR }}FC" | tee -a $GITHUB_ENV # Download elan; fetch the mathlib cache only for benches that import # Mathlib (Mathlib, FLT) — InitStd/Lean would otherwise pull it for @@ -108,41 +167,55 @@ jobs: key: ${{ matrix.cache_pkg }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}/lean-toolchain', env.COMPILE_DIR)) }}-${{ hashFiles(format('{0}/lake-manifest.json', env.COMPILE_DIR)) }} # No `--wfail` here: formal-conjectures (FC) emits a copyright-notice # warning that must not fail the build. - - run: lake build Compile${{ matrix.bench }} + - run: lake build Compile${{ matrix.env }} working-directory: ${{ env.COMPILE_DIR }} - # Serialize the env to a `.ixe` and emit the `##benchmark##` line. - - name: Run ix compile + # Serialize the env to a `.ixe` and measure the compile via `ix bench + # run` — for the `compile` backend the compile IS the benchmark (always + # fresh; `--ixe` is ignored by design). Writes `.ixe` at the + # workspace root and one results row keyed by the CamelCase env slug — + # the same driver the !benchmark PR path uses; bmf wraps it for bencher. + - name: Run ix compile benchmark run: | - ix compile ${{ env.COMPILE_DIR }}/Compile${{ matrix.bench }}.lean \ - --out ${{ matrix.bench }}.ixe 2>&1 | tee output.txt - # Cache the `.ixe` for the prove job (reused, never recompiled there). - # Only the matrix jobs the prove job consumes, to stay under the repo cache limit. - - if: matrix.bench == 'InitStd' || matrix.bench == 'Mathlib' + ix bench run --backend compile --env ${{ matrix.env }} --out bench.json + ix bench bmf --in bench.json --out benchmark.json + cat benchmark.json + # Gate for the two steps below: is this env benched (registry + # `benched: true`)? Derived from the registry via the in-job `ix` + # instead of hand-copying the env list into the conditionals. + - id: benched + name: Is this env benched? + run: | + echo "yes=$(ix bench ci matrix --kind envs \ + | jq 'index("${{ matrix.env }}") != null')" >> "$GITHUB_OUTPUT" + # Pre-cut the zisk closure-shard artifacts for the heavy primaries + # (`ix shard extract` → `ix profile` → `ix shard`, via `ix bench + # shard` — the same code path the zisk cells run lazily when the + # artifacts are absent). Done here, next to the fresh `.ixe` with + # `ix` + the Lean toolchain on hand, so the zkvm job just restores the + # dir. + - if: steps.benched.outputs.yes == 'true' + name: Cut closure shards for heavy primaries + run: ix bench shard --env ${{ matrix.env }} --ixe ${{ matrix.env }}.ixe + # Cache the `.ixe` + closure-shard artifacts for the prove/zkvm jobs + # (reused, never regenerated there). Only the benched envs those + # consume, to stay under the repo cache limit. NB: every restore of + # this key must list the SAME paths — actions/cache versions the entry + # by its path list. + - if: steps.benched.outputs.yes == 'true' uses: actions/cache/save@v5 with: - path: ${{ matrix.bench }}.ixe - key: aiur-ixe-${{ github.sha }}-${{ matrix.bench }} - - name: Generate compile benchmark JSON - run: | - line=$(grep '^##benchmark##' output.txt) - elapsed_s=$(echo "$line" | awk '{printf "%.3f", $2 / 1000}') - bytes=$(echo "$line" | awk '{print $3}') - constants=$(echo "$line" | awk '{print $4}') - throughput=$(echo "$line" | awk '{if ($2 > 0) printf "%.2f", $4 * 1000 / $2; else print 0}') - cat > benchmark.json <> $GITHUB_PATH # Provision the toolchain so the bench-typecheck binary finds libleanshared # (no package build). use-github-cache off: nothing to cache here, and @@ -197,59 +281,48 @@ jobs: build: false use-github-cache: false # Pull the `.ixe` the compile job built — do NOT recompile it here. + # (The path list must match the compile job's save exactly.) - uses: actions/cache/restore@v5 with: - path: ${{ matrix.bench }}.ixe - key: aiur-ixe-${{ github.sha }}-${{ matrix.bench }} + path: | + ${{ matrix.bench }}.ixe + zkshards-${{ matrix.bench }} + key: bench-ixe-${{ github.sha }}-${{ matrix.bench }} fail-on-cache-miss: true - # Run each constant in its own process so a clean failure or timeout drops - # only that constant from the report. NB: a constant heavy enough to OOM - # the runner host still cancels the whole job (an OOM SIGKILL of the host - # is uncatchable here), so every listed constant must fit in runner RAM. - # RAM is tracked via tracing-texray's machine-readable streaming lines - # (`[texray] peak-rss-bytes= ()`, emitted as each - # span closes): we parse the raw byte integer (awk's `$2+0` stops at - # the first non-digit, ignoring the parenthesized human suffix) and - # fold the max over spans in as `peak-rss` (bytes), the proving RSS - # high-water mark. + # Exit 3 = the kernel REJECTED a constant (a correctness regression, not + # a benchmark blip): the job reddens right here, with the rows on disk + # for the upload below. - name: Run Aiur typecheck benchmark run: | - measure() { - local c="$1" rss - timeout 20m bench-typecheck --ixe ${{ matrix.bench }}.ixe "$c" \ - --json "res-$c.json" --texray 2>"tx-$c.log" \ - || echo "warning: $c failed (OOM/timeout); dropping it from this report" - rss=$(awk -F'peak-rss-bytes=' 'NF>1 && $2+0>max {max=$2+0} END {if (max>0) print max}' "tx-$c.log") - if [ -f "res-$c.json" ] && [ -n "$rss" ] && [ "$rss" -gt 0 ]; then - jq --argjson rss "$rss" 'map_values(. + {"peak-rss": $rss})' \ - "res-$c.json" > "res-$c.json.tmp" && mv "res-$c.json.tmp" "res-$c.json" || true - fi - } - for c in ${{ matrix.consts }}; do measure "$c"; done - # Merge the per-constant results; if none produced anything, emit `{}`. - jq -s 'reduce .[] as $o ({}; . + $o)' res-*.json > results.json 2>/dev/null \ - || echo '{}' > results.json - [ -s results.json ] || echo '{}' > results.json - # Wrap each metric value as { "value": v } for Bencher Metric Format. - # bench-typecheck already emits slug keys (constants, fft-cost, - # execute-time, prove-time, throughput = constants/prove-time); peak-rss - # is injected above. - jq ' - map_values(to_entries | map({(.key): {value: .value}}) | add) - ' results.json > aiur.json + ix bench run --backend aiur --env ${{ matrix.bench }} \ + --mode ${{ matrix.mode }} --ixe ${{ matrix.bench }}.ixe --out bench.json + # Upload whatever clean rows exist even when the run step reddened the + # job — bmf drops every non-ok row, so a rejected or OOM'd constant + # never reaches bencher. + - name: Convert to Bencher Metric Format + id: bmf + if: ${{ !cancelled() }} + run: | + ix bench bmf --in bench.json --out aiur.json cat aiur.json # Upload Aiur metrics. Every measure shares the per-workload baseline - # window (data points since the aiur reset tag). constants is deterministic - # → pinned exactly (0/0). fft-cost is deterministic but only ever drops on - # a real Aiur win, so it rides an upper-only 5% bound (flag a regression, - # let wins through) rather than a hard pin. prove-time/execute-time, - # peak-rss (texray's proving RSS high-water mark), and throughput - # (constants/prove-time, where a drop is the regression) are noisy - # wall-clock and ride percentage bounds. + # window (data points since the workload's reset tag). constants is + # deterministic → pinned exactly (0/0). fft-cost is deterministic but + # only ever drops on a real Aiur win, so it rides an upper-only 5% + # bound (flag a regression, let wins through) rather than a hard pin. + # The wall-clock/RAM/rate measures ride percentage bounds; peak-rss + # and throughput are phase-scoped by the CELL (execute vs prove + # testbed), and prove-time/verify-time/proof-size exist only on the + # prove testbed — a threshold naming an absent measure just sits + # empty there. The per-phase `phase:` measures (witness gen, + # stage commits, quotient, …) are uploaded for trend visibility but + # intentionally left un-thresholded (noisy and dynamically named; + # the PR-comment drill-down does the phase-level comparison). - uses: ./.github/actions/bencher-track + if: ${{ !cancelled() && steps.bmf.outcome == 'success' }} with: - testbed: aiur-typecheck-x64-32x - workload: aiur + testbed: aiur-check-${{ matrix.mode }}-x64-32x + workload: aiur-check-${{ matrix.mode }} file: aiur.json key: ${{ secrets.BENCHER_API_KEY }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -263,6 +336,12 @@ jobs: --threshold-measure prove-time --threshold-test percentage --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 --threshold-lower-boundary _ + --threshold-measure verify-time --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 + --threshold-lower-boundary _ + --threshold-measure proof-size --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.05 + --threshold-lower-boundary _ --threshold-measure execute-time --threshold-test percentage --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 --threshold-lower-boundary _ @@ -272,3 +351,189 @@ jobs: --threshold-measure throughput --threshold-test percentage --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary _ --threshold-lower-boundary 0.10 + + # Execute the same constants through the zkVM hosts and track cycles / + # execute-time / throughput / peak-rss (and shards / max-shard-cycles + # for the closure-sharded heavy primaries). Reuses the compile job's cached + # `.ixe` + pre-cut zkshards-/ artifacts; `ix bench run` builds the Rust + # host itself and runs each heavy primary as its shard-manifest partition + # when the artifacts are present. zkVM proving needs a GPU (absent here), so + # this is execute-only. `ix` orchestrates the cell, so the staged binaries + + # Lean toolchain (libleanshared) are restored here too. + zkvm-execute: + name: ${{ matrix.cell.backend }}-${{ matrix.cell.env }}-execute + needs: [compile, plan] + runs-on: warp-ubuntu-latest-x64-32x + # Per-constant loop (heavy primaries ride the RAM watchdog to OOM rows) — + # `ix bench run` persists completed rows incrementally, so even a job-level + # timeout would keep them on disk, but the bencher upload needs the job + # alive. + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + # Enabled zkVM backends × benched envs, from the registry. sp1 is + # disabled in the registry (execute too slow for a per-push job); + # re-enabling it there restores its cells — also restore an + # `if: matrix.cell.backend == 'sp1'` install-sp1 step then. + cell: ${{ fromJson(needs.plan.outputs.zkvm-cells) }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + fetch-tags: true # bencher-track reads the bencher-thresholds-reset tag + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: ${{ matrix.cell.backend }} + - name: Install Zisk + if: matrix.cell.backend == 'zisk' + uses: ./.github/actions/install-zisk + - uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ github.sha }} + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + # Provision the toolchain so `ix` finds libleanshared (no package build). + - uses: leanprover/lean-action@v1 + with: + auto-config: false + build: false + use-github-cache: false + # Pull the `.ixe` + the pre-cut closure-shard artifacts the compile job + # built — `--ixe` means no recompile, and `ix bench run` skips + # re-cutting shards it finds in zkshards-/. + - uses: actions/cache/restore@v5 + with: + path: | + ${{ matrix.cell.env }}.ixe + zkshards-${{ matrix.cell.env }} + key: bench-ixe-${{ github.sha }}-${{ matrix.cell.env }} + fail-on-cache-miss: true + # Exit 3 = kernel rejection → red X here; clean rows still upload below. + - name: Run ${{ matrix.cell.backend }} execute benchmark + run: | + # ZisK's ASM microservices mmap with MAP_LOCKED: raise the memlock + # hard limit in this shell so the tools `ix bench run` spawns + # inherit it. + if [ "${{ matrix.cell.backend }}" = zisk ]; then + sudo prlimit --pid $$ --memlock=unlimited:unlimited + fi + ix bench run --backend ${{ matrix.cell.backend }} \ + --env ${{ matrix.cell.env }} --mode execute \ + --ixe ${{ matrix.cell.env }}.ixe --out bench.json + # Upload whatever clean rows exist even when the run step reddened the + # job — bmf drops every non-ok row. + - name: Convert to Bencher Metric Format + id: bmf + if: ${{ !cancelled() }} + run: | + ix bench bmf --in bench.json --out bench-bmf.json + cat bench-bmf.json + # cycles / shards / max-shard-cycles are deterministic per guest ELF, but + # a real guest / packer improvement legitimately drops them — upper-only + # 0% bound (flag regressions, let wins through), like `fft-cost` on the + # aiur job. execute-time / peak-rss / throughput are noisy wall-clock → + # percentage bounds (throughput's regression is a drop). + - uses: ./.github/actions/bencher-track + if: ${{ !cancelled() && steps.bmf.outcome == 'success' }} + with: + testbed: ${{ matrix.cell.backend }}-check-${{ matrix.cell.mode }}-x64-32x + workload: ${{ matrix.cell.backend }}-check-${{ matrix.cell.mode }} + file: bench-bmf.json + key: ${{ secrets.BENCHER_API_KEY }} + github-token: ${{ secrets.GITHUB_TOKEN }} + thresholds: | + --threshold-measure cycles --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0 + --threshold-lower-boundary _ + --threshold-measure shards --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0 + --threshold-lower-boundary _ + --threshold-measure max-shard-cycles --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0 + --threshold-lower-boundary _ + --threshold-measure execute-time --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 + --threshold-lower-boundary _ + --threshold-measure peak-rss --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 + --threshold-lower-boundary _ + --threshold-measure throughput --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary _ + --threshold-lower-boundary 0.10 + + # Out-of-circuit Rust kernel typecheck — the same kernel as the zkVM guest, + # but run out-of-circuit and in parallel, so far faster than proving. + # `ix bench run --backend ooc` checks the whole env (one row keyed by the + # env slug) plus one full-closure row per primary constant, for an + # apples-to-apples baseline next to the zkVM cells. Reuses the compile job's + # cached `.ixe` and the staged `ix` binary — no recompile. + ooc-check: + name: ooc-${{ matrix.bench }} + needs: [compile, plan] + runs-on: warp-ubuntu-latest-x64-32x + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + bench: ${{ fromJson(needs.plan.outputs.bench-envs) }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + fetch-tags: true # bencher-track reads the bencher-thresholds-reset tag + - uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ github.sha }} + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + # Provision the toolchain so `ix` finds libleanshared (no package build). + - uses: leanprover/lean-action@v1 + with: + auto-config: false + build: false + use-github-cache: false + # (The path list must match the compile job's save exactly.) + - uses: actions/cache/restore@v5 + with: + path: | + ${{ matrix.bench }}.ixe + zkshards-${{ matrix.bench }} + key: bench-ixe-${{ github.sha }}-${{ matrix.bench }} + fail-on-cache-miss: true + # Exit 3 = kernel rejection → red X here; clean rows still upload below. + - name: Run out-of-circuit kernel check + run: | + ix bench run --backend ooc --env ${{ matrix.bench }} --mode execute \ + --ixe ${{ matrix.bench }}.ixe --out bench.json + # Upload whatever clean rows exist even when the run step reddened the + # job — bmf drops every non-ok row. + - name: Convert to Bencher Metric Format + id: bmf + if: ${{ !cancelled() }} + run: | + ix bench bmf --in bench.json --out bench-bmf.json + cat bench-bmf.json + # constants is deterministic → pinned (0/0); check-time / throughput / + # peak-rss are noisy parallel wall-clock → percentage bounds. + - uses: ./.github/actions/bencher-track + if: ${{ !cancelled() && steps.bmf.outcome == 'success' }} + with: + testbed: ooc-check-x64-32x + workload: ooc-check + file: bench-bmf.json + key: ${{ secrets.BENCHER_API_KEY }} + github-token: ${{ secrets.GITHUB_TOKEN }} + thresholds: | + --threshold-measure constants --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0 + --threshold-lower-boundary 0 + --threshold-measure check-time --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 + --threshold-lower-boundary _ + --threshold-measure throughput --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary _ + --threshold-lower-boundary 0.10 + --threshold-measure peak-rss --threshold-test percentage + --threshold-max-sample-size __WINDOW__ --threshold-upper-boundary 0.10 + --threshold-lower-boundary _ diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index a2a1823c9..ca37bba3e 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -1,4 +1,30 @@ -# Creates a PR benchmark comment with a comparison to main +# `!benchmark` PR command: run the curated constant set (Benchmarks/Vectors.csv) +# through chosen prover backend(s) and post a main-vs-PR comparison table. +# +# !benchmark ([aiur] [zisk] [sp1] [ooc] [compile] | all) [execute] +# (sp1 is disabled in the registry (Ix/Cli/BenchCmd.lean) — the parser skips it +# with a note in the config summary) +# BENCH_ENVS=InitStd,Mathlib # which compiled envs (default InitStd; case-insensitive) +# BENCH_FULL=1 # run the full curated set, not just primary +# BENCH_SHARD=1 # restrict to the multi-shard target constants +# RUST_LOG=info # passthrough env (allowlisted) +# +# Mode defaults per backend (the registry's defaultMode): `aiur` runs +# `prove` — the real-workload simulation, whose report also carries the +# Phase-1 columns `fft-cost` / `execute-time` measured en route; `zisk` / +# `sp1` / `ooc` run `execute`; `compile` runs `ix compile .lean → +# .ixe` (the same cell bench-main.yml uploads under testbed +# `ix-compile-*`). The optional bare `execute` token flips `aiur` to +# execute-only (Phase 1, skipping the prove); bench-main runs both aiur +# modes as separate cells on separate testbeds, so either kind of cell +# fetches a cached main-side baseline from bencher. +# +# Each matrix cell is one `ix bench run` per side, driven by the PR's `ix` +# (`--repo` points it at the checkout to measure). main's numbers come from +# bencher.dev (`ix bench fetch-main`); the workflow re-runs the base SHA +# locally only when bencher can't supply them: the base SHA isn't ingested yet +# (freshly-pushed main whose CI is still running), or the PR's Vectors.csv +# selects constants main was never benched on (constants the PR itself adds). name: Benchmark pull requests on: @@ -11,145 +37,580 @@ permissions: pull-requests: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.issue.number }} cancel-in-progress: true jobs: setup: - name: Comparative PR benchmark comment - if: - github.event.issue.pull_request - && github.event.issue.state == 'open' - && (contains(github.event.comment.body, '!benchmark')) - && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') + name: Parse !benchmark comment + if: >- + github.event.issue.pull_request && + github.event.issue.state == 'open' && + contains(github.event.comment.body, '!benchmark') && + (github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'OWNER') runs-on: ubuntu-latest outputs: - benches: ${{ steps.bench-params.outputs.benches }} - env-vars: ${{ steps.bench-params.outputs.env-vars }} + base-sha: ${{ steps.shas.outputs.base }} + head-sha: ${{ steps.shas.outputs.head }} steps: - uses: actions/checkout@v6 - - name: Parse PR comment body - id: bench-params + # issue_comment doesn't carry the PR's base/head; this action looks + # them up. + - uses: xt0rted/pull-request-comment-branch@v3 + id: comment-branch + - name: Resolve base/head SHAs + id: shas run: | - # Parse `issue_comment` body - printf '${{ github.event.comment.body }}' > comment.txt - BENCH_COMMAND=$(head -n 1 comment.txt | tr -d '\r') - echo "$BENCH_COMMAND" - - BENCHES=$(echo $BENCH_COMMAND | awk -F'!benchmark ' '{ print $2 }') - # Set default benches to run if none specified - BENCHES=${BENCHES:-"bench-aiur"} - echo "BENCHES:" - echo "$BENCHES" - JSON=$(echo $BENCHES | jq -R -c 'split(" ")') - - echo "JSON:" - echo "$JSON" - - echo "benches=$JSON" | tee -a $GITHUB_OUTPUT + echo "base=${{ steps.comment-branch.outputs.base_sha }}" >> "$GITHUB_OUTPUT" + echo "head=${{ steps.comment-branch.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + # Build the PR's `ix` + `bench-typecheck` once (they embed the IxVM kernel + # and the Aiur prover), stage under ~/.local/bin, and cache by head SHA — + # the matrix cells restore instead of re-running the full Lean build per + # cell, and re-running !benchmark on the same commit skips the build + # entirely. Built on warp, mirroring bench-main.yml's build job, so PR + # binaries carry the same instruction-set provenance (uniform AVX-512 + # fleet) as the binaries behind bencher's main-side numbers — and every + # job that RUNS them must be a warp host too, or a non-AVX-512 GitHub + # host SIGILLs. The job ends by parsing the !benchmark command with + # the freshly built `ix bench ci parse` — the registry lives in Lean, so the + # matrix can only exist post-build. + build: + needs: setup + runs-on: warp-ubuntu-latest-x64-32x + outputs: + matrix: ${{ steps.parse.outputs.matrix }} + envs: ${{ steps.parse.outputs.envs }} + shard: ${{ steps.parse.outputs.shard }} + full: ${{ steps.parse.outputs.full }} + passthrough-env: ${{ steps.parse.outputs.passthrough-env }} + config-summary: ${{ steps.parse.outputs.config-summary }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.setup.outputs.head-sha }} + # The job builds and runs PR code; never leave the token in .git. + persist-credentials: false + - id: bins + uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ needs.setup.outputs.head-sha }} + - if: steps.bins.outputs.cache-hit != 'true' + uses: actions-rust-lang/setup-rust-toolchain@v1 + # `.cargo/config.toml` sets `-Ctarget-cpu=native`; log the build CPU so a + # benchmark shift can be traced to an instruction-set change. + - name: Log build CPU + if: steps.bins.outputs.cache-hit != 'true' + run: | + lscpu + flags=$(grep -m1 -oE 'avx2|avx512[a-z0-9_]*' /proc/cpuinfo | sort -u | tr '\n' ' ') + echo "AVX flags: ${flags:-absent}" + # One lean-action for both paths: cache miss → build the binaries; + # cache hit → provision the toolchain only (the restored `ix` still + # needs libleanshared to run the parse step below). + - uses: leanprover/lean-action@v1 + with: + auto-config: false + build: ${{ steps.bins.outputs.cache-hit != 'true' }} + build-args: "ix bench-typecheck" + use-github-cache: false + - if: steps.bins.outputs.cache-hit != 'true' + run: | + mkdir -p ~/.local/bin + cp .lake/build/bin/ix .lake/build/bin/bench-typecheck ~/.local/bin/ + chmod +x ~/.local/bin/ix ~/.local/bin/bench-typecheck + - if: steps.bins.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ needs.setup.outputs.head-sha }} + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + # Parse the !benchmark command from an env var (never + # inline-interpolated); unknown tokens fall off the allowlist. + - name: Parse command + id: parse + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: ix bench ci parse - # Can't persist env vars between jobs, so we pass them as an output and set them in the next job - echo "env-vars=$(tail -n +2 comment.txt | tr -d '\r' | tr '\n' ' ')" | tee -a $GITHUB_OUTPUT + # ONE measured `ix compile` per requested env, before the cells: it + # publishes the `.ixe` the prover cells restore (no per-cell compile + # races), and its results row doubles as the compile cell's PR side — + # requesting the compile backend never compiles the env twice. + compile: + name: compile-${{ matrix.env }} + needs: [setup, build] + if: needs.build.outputs.envs != '[]' + runs-on: warp-ubuntu-latest-x64-32x + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + env: ${{ fromJSON(needs.build.outputs.envs) }} + steps: + - name: Checkout PR + uses: actions/checkout@v6 + with: + ref: ${{ needs.setup.outputs.head-sha }} + # The job runs PR code; never leave the token in .git. + persist-credentials: false + # Re-running !benchmark on the same commit: the .ixe is already + # published — nothing to do. + - name: Check for published .ixe + id: pr-ixe + uses: actions/cache/restore@v5 + with: + path: ${{ matrix.env }}.ixe + key: bench-pr-ixe-${{ needs.setup.outputs.head-sha }}-${{ matrix.env }} + lookup-only: true + - name: Restore PR binaries + if: steps.pr-ixe.outputs.cache-hit != 'true' + uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ needs.setup.outputs.head-sha }} + fail-on-cache-miss: true + - if: steps.pr-ixe.outputs.cache-hit != 'true' + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Provision Lean toolchain + if: steps.pr-ixe.outputs.cache-hit != 'true' + uses: leanprover/lean-action@v1 + with: + lake-package-directory: . + auto-config: false + build: false + use-github-cache: false + use-mathlib-cache: ${{ matrix.env == 'Mathlib' && 'true' || 'false' }} + - name: Compile ${{ matrix.env }}.ixe + if: steps.pr-ixe.outputs.cache-hit != 'true' + run: ix bench run --backend compile --env ${{ matrix.env }} --out compile.json + - name: Publish .ixe + if: steps.pr-ixe.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: ${{ matrix.env }}.ixe + key: bench-pr-ixe-${{ needs.setup.outputs.head-sha }}-${{ matrix.env }} + # The measured row: the compile cell reuses it as its PR side (same + # runner class, same binaries, same command it would run itself). + - name: Publish compile row + if: steps.pr-ixe.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: compile.json + key: bench-pr-row-${{ needs.setup.outputs.head-sha }}-${{ matrix.env }} benchmark: - needs: [ setup ] - runs-on: warp-ubuntu-latest-x64-16x + # Explicit name: the default would append EVERY matrix value (backend, + # env, mode, runner, label); the label already is the cell id. + name: ${{ matrix.cell.label }} + needs: [setup, build, compile] + # A compile failure stops the cells (they would just re-fail the same + # compile lazily); the skipped case can't arise (envs is never empty) + # but is tolerated for robustness. + if: ${{ !cancelled() && needs.build.result == 'success' && needs.compile.result != 'failure' && needs.compile.result != 'cancelled' }} + runs-on: ${{ matrix.cell.runner }} + # Wide enough for the zisk cell's worst case: per-constant loop + PR-side + # `ix profile` when a heavy primary's closure shards must be cut fresh. + timeout-minutes: 180 strategy: + fail-fast: false matrix: - # Runs a job for each benchmark specified in the `issue_comment` input - bench: ${{ fromJSON(needs.setup.outputs.benches) }} + cell: ${{ fromJSON(needs.build.outputs.matrix) }} + env: + BACKEND: ${{ matrix.cell.backend }} + BENV: ${{ matrix.cell.env }} + MODE: ${{ matrix.cell.mode }} + LABEL: ${{ matrix.cell.label }} + BASE_SHA: ${{ needs.setup.outputs.base-sha }} + HEAD_SHA: ${{ needs.setup.outputs.head-sha }} + SHARD: ${{ needs.build.outputs.shard }} + FULL: ${{ needs.build.outputs.full }} steps: - - name: Set env vars - run: | - # Overrides default env vars with those specified in the `issue_comment` input if identically named - for var in ${{ needs.setup.outputs.env-vars }} - do - echo "$var" | tee -a $GITHUB_ENV - done - - uses: actions/checkout@v6 - # Get base branch of the PR - - uses: xt0rted/pull-request-comment-branch@v3 - id: comment-branch - - name: Checkout base branch + # PR checked out at the workspace root so the local install actions + # resolve; base (bencher-miss fallback only) goes under base/. + - name: Checkout PR uses: actions/checkout@v6 with: - ref: ${{ steps.comment-branch.outputs.base_sha }} - path : ${{ github.workspace }}/base - - name: Run `lake build` on base branch + ref: ${{ env.HEAD_SHA }} + # The job runs PR code; never leave the token in .git. + persist-credentials: false + # Allowlisted KEY=VALUE lines from the !benchmark comment (often + # empty). Delivered via an env var — inline `${{ }}` inside a heredoc + # both risks injection and breaks the quoted terminator's required + # column-0 position under YAML block indentation. + - name: Apply passthrough env + env: + PTENV: ${{ needs.build.outputs.passthrough-env }} + run: printf '%s\n' "$PTENV" | sed '/^[[:space:]]*$/d' >> "$GITHUB_ENV" + # Restore the once-built PR binaries (see the build job) into the PR + # tree's own bin dir: `ix bench run` resolves the measured tools from + # /.lake/build/bin first, then PATH, so staging in-tree keeps the + # PR and base sides cleanly separated (~/.local/bin stays free for the + # base side's cache restore below). The same dir goes on PATH — the + # PR's `ix bench` orchestrates BOTH sides, with `--repo` selecting + # whose tools get measured. + - name: Restore PR binaries + uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ env.HEAD_SHA }} + fail-on-cache-miss: true + - run: | + mkdir -p .lake/build/bin + mv ~/.local/bin/ix ~/.local/bin/bench-typecheck .lake/build/bin/ + echo "$PWD/.lake/build/bin" >> "$GITHUB_PATH" + # Toolchain provisioning only (no package build): the restored binaries + # link against the toolchain's libleanshared. Mathlib olean cache only + # for the mathlib env (its `ix compile` needs the oleans). + - name: Provision Lean toolchain uses: leanprover/lean-action@v1 with: - lake-package-directory: ${{ github.workspace }}/base - test: false - - name: Run bench on base branch + lake-package-directory: . + auto-config: false + build: false + use-github-cache: false + use-mathlib-cache: ${{ matrix.cell.env == 'Mathlib' && 'true' || 'false' }} + # The compiled env is identical across every cell of the same PR commit — + # the compile job published it; cells pass it via `--ixe` instead of + # recompiling. + - name: Restore PR .ixe + id: pr-ixe + uses: actions/cache/restore@v5 + with: + path: ${{ matrix.cell.env }}.ixe + key: bench-pr-ixe-${{ env.HEAD_SHA }}-${{ matrix.cell.env }} + # Compile cells reuse the compile job's measured row as their PR side + # (a miss just means this cell compiles and measures itself). + - name: Restore compile row + if: matrix.cell.backend == 'compile' + uses: actions/cache/restore@v5 + with: + path: compile.json + key: bench-pr-row-${{ env.HEAD_SHA }}-${{ matrix.cell.env }} + # zkVM cells additionally need the Rust toolchain + the backend's toolchain + # and system deps (the shared composite install actions). + - name: Set up zkVM Rust toolchain + if: matrix.cell.backend == 'zisk' || matrix.cell.backend == 'sp1' + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: ${{ matrix.cell.backend }} + # sp1 is disabled in the registry (execute too slow for CI); + # re-enable it there and uncomment this install step to restore it. + # - name: Install SP1 + # if: matrix.cell.backend == 'sp1' + # uses: ./.github/actions/install-sp1 + - name: Install Zisk + if: matrix.cell.backend == 'zisk' + uses: ./.github/actions/install-zisk + + # ---------- PR side ---------- + # The PR side runs FIRST: `ix bench run` does its own constant selection + # from Vectors.csv, so pr.json's row names are this cell's canonical + # name list — the main-side bencher fetch below filters on them. + - name: Run backend on PR → pr.json + id: pr-run + # Exit 3 = the kernel rejected a constant on the PR side (rows on + # disk). Don't stop the job here — the compare table and PR comment + # must still render the ❌ row; the failure is re-raised LOUDLY at + # the end of the job, after the table upload. + continue-on-error: true run: | - if $(lake run get-exe-targets | grep -q ${{ matrix.bench }}); then - lake exe ${{ matrix.bench }} - else - echo "No matching bench target found on base branch" + if [ "$BACKEND" = zisk ]; then + # ZisK's ASM microservices mmap with MAP_LOCKED: raise the memlock + # hard limit in this shell so the spawned tools inherit it. + sudo prlimit --pid $$ --memlock=unlimited:unlimited fi - working-directory: ${{ github.workspace }}/base - - name: Checkout PR branch + if [ "$BACKEND" = compile ] && [ -f "$GITHUB_WORKSPACE/compile.json" ]; then + # The compile job already measured this env's compile (same + # runner class, binaries, and command) — reuse its row instead + # of compiling a second time. + cp "$GITHUB_WORKSPACE/compile.json" "$GITHUB_WORKSPACE/pr.json" + exit 0 + fi + flags="" + [ "$FULL" = 1 ] && flags="$flags --full" + [ "$SHARD" = 1 ] && flags="$flags --shard-only" + # The compile job published the .ixe; if it is somehow absent, + # ix bench run compiles it fresh (no flag). + [ -f "$BENV.ixe" ] && flags="$flags --ixe $BENV.ixe" + ix bench run --backend "$BACKEND" --env "$BENV" --mode "$MODE" \ + --out "$GITHUB_WORKSPACE/pr.json" $flags + # First cell to compile the PR env publishes it for the others (racing + # saves are fine — the first wins, the rest fail gracefully). hashFiles + # guard: the run step is continue-on-error, so it may have died before + # producing the `.ixe` — saving a missing path would fail this step. + - name: Save PR .ixe + if: steps.pr-ixe.outputs.cache-hit != 'true' && hashFiles(format('{0}.ixe', matrix.cell.env)) != '' + uses: actions/cache/save@v5 + with: + path: ${{ matrix.cell.env }}.ixe + key: bench-pr-ixe-${{ env.HEAD_SHA }}-${{ matrix.cell.env }} + + # ---------- main side ---------- + # Try bencher.dev first (bench-main.yml has uploaded main's numbers). + # fetch-main's exit codes are load-bearing: 3 = transient (base SHA not + # ingested yet) → fall back to a local base run; anything else (2 = + # backend/mode has no main testbed — a registry / + # bench-main.yml drift) is a permanent misconfiguration that a local + # rebuild can never fix, so fail the cell loudly instead of silently + # paying the fallback on every future run. + # + # Partial miss (exit 0 + non-empty missing.txt): the PR's Vectors.csv + # selects names main was never benched on — typically constants the PR + # itself adds. Bencher's numbers stand for the covered set; the base run + # below fills the gaps and the merge keeps bencher canonical. `run-base` + # gates every base-side step. + - name: Fetch main from bencher + id: bencher + run: | + # This cell's names = pr.json's row keys (empty when the PR run + # produced nothing — fetch-main then exits 3 and the base run + # supplies the whole main side). + NAMES=$(jq -r 'keys | join(",")' "$GITHUB_WORKSPACE/pr.json" 2>/dev/null || true) + set +e + ix bench fetch-main \ + --sha "$BASE_SHA" --backend "$BACKEND" --mode "$MODE" --env "$BENV" \ + --consts "$NAMES" \ + --missing-out "$GITHUB_WORKSPACE/missing.txt" \ + --out "$GITHUB_WORKSPACE/main.json" + rc=$? + set -e + case $rc in + 0) if [ -s "$GITHUB_WORKSPACE/missing.txt" ]; then + echo "source=bencher @ ${BASE_SHA::7} + base run ($(wc -l < "$GITHUB_WORKSPACE/missing.txt") new)" >> "$GITHUB_OUTPUT" + echo "run-base=true" >> "$GITHUB_OUTPUT" + else + echo "source=bencher @ ${BASE_SHA::7}" >> "$GITHUB_OUTPUT" + echo "run-base=false" >> "$GITHUB_OUTPUT" + fi ;; + 3) echo "source=base run @ ${BASE_SHA::7} (not on bencher)" >> "$GITHUB_OUTPUT" + echo "run-base=true" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::fetch-main: permanent config error (exit $rc) — check backendSpecs testbeds vs bench-main.yml"; exit "$rc" ;; + esac + - name: Checkout base (bencher data missing or partial) + if: steps.bencher.outputs.run-base == 'true' uses: actions/checkout@v6 with: - path: ${{ github.workspace }}/pr - ref: ${{ steps.comment-branch.outputs.head_sha }} - - name: Run `lake build` on PR branch + ref: ${{ env.BASE_SHA }} + path: base + persist-credentials: false + # bench-main.yml's build/compile jobs already cached the base SHA's + # binaries and `.ixe` on the push to main — restore both before paying + # for a from-scratch base build. + - name: Restore base binaries (bench-main build cache) + if: steps.bencher.outputs.run-base == 'true' + id: base-bins + uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ env.BASE_SHA }} + - name: Restore base .ixe (bench-main compile cache) + if: steps.bencher.outputs.run-base == 'true' + id: base-ixe + uses: actions/cache/restore@v5 + with: + # Path list must match bench-main's compile-job save (actions/cache + # versions entries by path list); the key suffix is the env name + # (bench-main's matrix.env). + path: | + ${{ matrix.cell.env }}.ixe + zkshards-${{ matrix.cell.env }} + key: bench-ixe-${{ env.BASE_SHA }}-${{ matrix.cell.env }} + # Cached base binaries are usable only when the two `lean-toolchain` + # files are identical (plain `cmp`), and — for the mathlib env — only + # when the `.ixe` was also restored (otherwise base's `ix compile` needs + # mathlib oleans that only the full build fetches). Usable binaries are + # staged into base/.lake/build/bin, where `ix bench run --repo base` + # resolves them (a from-scratch base build lands there natively). + - name: Resolve base binaries + if: steps.bencher.outputs.run-base == 'true' + id: base-src + run: | + cached=false + if [ "${{ steps.base-bins.outputs.cache-hit }}" = true ] \ + && cmp -s lean-toolchain base/lean-toolchain; then + if [ "$BENV" != Mathlib ] || [ "${{ steps.base-ixe.outputs.cache-hit }}" = true ]; then + mkdir -p base/.lake/build/bin + mv ~/.local/bin/ix ~/.local/bin/bench-typecheck base/.lake/build/bin/ 2>/dev/null || true + [ -x base/.lake/build/bin/ix ] && [ -x base/.lake/build/bin/bench-typecheck ] && cached=true + fi + fi + echo "cached=$cached" >> "$GITHUB_OUTPUT" + echo "base binaries: $([ "$cached" = true ] && echo restored from bench-main cache || echo building from source)" + - name: Build base (ix, bench-typecheck) + if: steps.bencher.outputs.run-base == 'true' && steps.base-src.outputs.cached != 'true' uses: leanprover/lean-action@v1 with: - lake-package-directory: ${{ github.workspace }}/pr - test: false - - name: Copy base benchmarks into PR dir for comparison + lake-package-directory: base + auto-config: false + build: true + build-args: "ix bench-typecheck" + use-github-cache: false + use-mathlib-cache: ${{ matrix.cell.env == 'Mathlib' && 'true' || 'false' }} + # NOTE: the PR's `ix bench` orchestrates the base run (--repo base); the + # MEASURED tools resolve from base/.lake/build/bin. When a PR changes + # the benchmark CLIs themselves, the base tools may reject the new + # flags and every constant drops — compare then renders the loud + # "main produced no results" note instead of a silent all-n/a table. + # + # Partial bencher miss (missing.txt non-empty): the base side runs JUST + # the uncovered names (passed via `--consts`) — typically constants the PR + # itself adds to Vectors.csv, which base's Vectors.csv doesn't list, so + # `--csv` points at the PR's copy. Full miss: the whole cell runs on + # base with the normal CSV selection. + - name: Run backend on base → merge into main.json + if: steps.bencher.outputs.run-base == 'true' run: | - BENCH_DIR_PR=${{ github.workspace }}/pr/.lake/benches - BENCH_DIR_BASE=${{ github.workspace }}/base/.lake/benches - mkdir -p $BENCH_DIR_PR - [ -d "$BENCH_DIR_BASE" ] && cp -r $BENCH_DIR_BASE/. $BENCH_DIR_PR/ - ls $BENCH_DIR_PR - - name: Run bench on PR branch and generate comparison report + if [ "${{ steps.base-ixe.outputs.cache-hit }}" = true ]; then + mv "${{ matrix.cell.env }}.ixe" "base/${{ matrix.cell.env }}.ixe" + # bench-main's pre-cut closure shards ride the same cache entry; + # in place, `ix bench run` skips re-cutting them for the base tree. + mv "zkshards-${{ matrix.cell.env }}" "base/zkshards-${{ matrix.cell.env }}" 2>/dev/null || true + fi + if [ "$BACKEND" = zisk ]; then + sudo prlimit --pid $$ --memlock=unlimited:unlimited + fi + flags="" + [ "$FULL" = 1 ] && flags="$flags --full" + [ "$SHARD" = 1 ] && flags="$flags --shard-only" + if [ -s "$GITHUB_WORKSPACE/main.json" ] && [ -s "$GITHUB_WORKSPACE/missing.txt" ]; then + flags="$flags --consts $(paste -sd, "$GITHUB_WORKSPACE/missing.txt")" + fi + # A base-side rejection or empty run must not redden the PR's cell: + # whatever rows landed are merged and the table says the rest. + set +e + # Cached base .ixe when the restore above hit; else the base run + # compiles it fresh (no flag). + [ -f "base/$BENV.ixe" ] && flags="$flags --ixe base/$BENV.ixe" + ix bench run --backend "$BACKEND" --env "$BENV" --mode "$MODE" \ + --repo base \ + --csv "$GITHUB_WORKSPACE/Benchmarks/Vectors.csv" \ + --out "$GITHUB_WORKSPACE/base.json" $flags + echo "base-side ix bench run exit $?" + set -e + # Partial fallback: bencher already supplied main.json for the + # covered names; fill only the gaps from the base run. Bencher wins + # any overlap (e.g. ooc's always-emitted whole-env row) — it is the + # canonical main-side number. Full fallback: main.json doesn't exist + # (fetch-main exits 3 before writing), so the base run IS main.json. + if [ -s "$GITHUB_WORKSPACE/base.json" ]; then + if [ -s "$GITHUB_WORKSPACE/main.json" ]; then + jq -s '.[0] + .[1]' "$GITHUB_WORKSPACE/base.json" "$GITHUB_WORKSPACE/main.json" \ + > "$GITHUB_WORKSPACE/main.merged" \ + && mv "$GITHUB_WORKSPACE/main.merged" "$GITHUB_WORKSPACE/main.json" + else + mv "$GITHUB_WORKSPACE/base.json" "$GITHUB_WORKSPACE/main.json" + fi + fi + + # ---------- compare ---------- + - name: Build comparison table run: | - BENCH_REPORT=1 lake exe ${{ matrix.bench }} - working-directory: ${{ github.workspace }}/pr - - name: Get env for PR body + mkdir -p out + ix bench compare \ + --backend "$BACKEND" --env "$BENV" --mode "$MODE" \ + --main "$GITHUB_WORKSPACE/main.json" --pr "$GITHUB_WORKSPACE/pr.json" \ + --main-source "${{ steps.bencher.outputs.source }}" \ + --out "out/table-$LABEL.md" + cat "out/table-$LABEL.md" + + - name: Upload table if: always() + uses: actions/upload-artifact@v4 + with: + name: table-${{ env.LABEL }} + path: out/table-${{ env.LABEL }}.md + if-no-files-found: warn + # A PR-side rejection is a correctness regression: re-raise the run + # step's failure LOUDLY (red X on the PR) — after the table upload, so + # the comment still posts with the constant's ❌ row and note. + - name: Fail on PR-side benchmark failure + if: steps.pr-run.outcome != 'success' + run: | + echo "::error::ix bench run failed on the PR side (kernel rejection or empty run) — see the 'Run backend on PR' step" + exit 1 + + # Assemble the comment body from the per-cell tables. Deliberately + # UNPRIVILEGED: this job checks out and executes PR-derived code (the + # PR-built `ix bench report`), so it must never share a job with the App + # token — the posting job below only downloads the finished body as an + # artifact and never touches PR code (CodeQL: untrusted checkout / + # execution in a privileged issue_comment workflow). + assemble: + needs: [setup, build, benchmark] + # Assemble even when benchmark cells failed (the tables carry the ❌ + # rows); the build job must have produced the binaries. + if: always() && needs.setup.result == 'success' && needs.build.result == 'success' + # `bench report` is a pure-Lean subcommand (AVX-512 lives only in the + # binary's Rust objects, which it never executes) — cheap host is fine. + # Move to a warp host if `ix` startup or this subcommand ever touches + # the FFI. + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.setup.outputs.head-sha }} + persist-credentials: false + - uses: actions/cache/restore@v5 + with: + path: ~/.local/bin + key: bench-bins-${{ needs.setup.outputs.head-sha }} + fail-on-cache-miss: true + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + # Provision the toolchain so `ix` finds libleanshared (no package build). + - uses: leanprover/lean-action@v1 + with: + auto-config: false + build: false + use-github-cache: false + - name: Download tables + uses: actions/download-artifact@v4 + with: + path: tables + pattern: table-* + merge-multiple: true + - name: Build comment body + env: + SUMMARY: ${{ needs.build.outputs.config-summary }} + HEAD_SHA: ${{ needs.setup.outputs.head-sha }} run: | - SHORT_SHA_PR=$(git rev-parse --short HEAD) - REPO_URL=${{ github.server_url }}/${{ github.repository }} - echo "COMMIT_LINK=[\`$SHORT_SHA_PR\`]($REPO_URL/commit/${{ steps.comment-branch.outputs.head_sha }})" | tee -a $GITHUB_ENV - echo "WORKFLOW_LINK=[Workflow logs]($REPO_URL/actions/runs/${{ github.run_id }})" | tee -a $GITHUB_ENV - working-directory: ${{ github.workspace }}/pr + mkdir -p tables # absent when no cell uploaded a table + ix bench report \ + --tables tables --summary "$SUMMARY" \ + --head "$HEAD_SHA" \ + --repo-url "${{ github.server_url }}/${{ github.repository }}" \ + --run-id "${{ github.run_id }}" --out comment-body.md + - uses: actions/upload-artifact@v4 + with: + name: comment-body + path: comment-body.md + + # Post the assembled body. This is the ONLY job with access to the App + # token, and it runs no checkout and no repo-derived code — just an + # artifact download and two actions. + comment: + needs: [setup, assemble] + if: always() && needs.assemble.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Download comment body + uses: actions/download-artifact@v4 + with: + name: comment-body - name: Generate token to write PR comment - uses: actions/create-github-app-token@v3 - if: always() id: app-token + uses: actions/create-github-app-token@v3 with: - app-id: ${{ secrets.TOKEN_APP_ID }} + # `client-id` accepts the numeric App ID too (the JWT issuer claim + # takes either), so the existing secret works unchanged. + client-id: ${{ secrets.TOKEN_APP_ID }} private-key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }} - - name: Build benchmark comment body - if: success() - run: | - { - echo '## Benchmark for `${{ matrix.bench }}` at ${{ env.COMMIT_LINK }}'; - echo ""; - for file in .lake/benches/*/report.md; do - [ -f "$file" ] && cat "$file" && echo "" - done - echo "${{ env.WORKFLOW_LINK }}"; - } > ${{ github.workspace }}/comment-body.md - working-directory: ${{ github.workspace }}/pr - - name: Comment on successful run - if: success() + - name: Post / update comment uses: peter-evans/create-or-update-comment@v5 with: token: ${{ steps.app-token.outputs.token }} issue-number: ${{ github.event.issue.number }} - body-path: 'comment-body.md' - - name: Comment on failing run - if: failure() - uses: peter-evans/create-or-update-comment@v5 - with: - token: ${{ steps.app-token.outputs.token }} - issue-number: ${{ github.event.issue.number }} - body: | - ## Benchmark for `${{ matrix.bench }}` at ${{ env.COMMIT_LINK }} failed :x: - - [Workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + body-path: comment-body.md diff --git a/.github/workflows/bencher-thresholds-reset.yml b/.github/workflows/bencher-thresholds-reset.yml index 37a0213c7..80cab6c87 100644 --- a/.github/workflows/bencher-thresholds-reset.yml +++ b/.github/workflows/bencher-thresholds-reset.yml @@ -17,7 +17,9 @@ name: Bencher thresholds reset # `bencher-thresholds-reset:` label on the PR, whatever added it — so a # Triage+ collaborator can queue a reset by applying the label directly, and # cancel by removing it before merge. Naming convention: one label per token, -# `bencher-thresholds-reset:` where is `ix-compile`, `aiur`, or +# `bencher-thresholds-reset:` where is a workload (a backend +# testbed in Ix/Cli/BenchCmd.lean (backendSpecs) minus its runner-arch suffix: +# `ix-compile`, `aiur-check-execute`, `aiur-check-prove`, `zisk-check-execute`, `sp1-check-execute`, `ooc-check`) or # `all` (the merge step expands an `all` label into every workload). Labeling # requires Triage+, so PR authors from forks cannot self-queue a reset. The # label shares the command/workflow name; the git tag it moves is the same @@ -36,7 +38,10 @@ on: description: Workload baseline to reset required: true type: choice - options: [ix-compile, aiur, all] + # GitHub requires literal choice options, so this list stays static: + # keep it (and the jobs' valid= lists below) in sync with the + # backend testbeds in Ix/Cli/BenchCmd.lean (backendSpecs). + options: [ix-compile, aiur-check-execute, aiur-check-prove, zisk-check-execute, sp1-check-execute, ooc-check, all] sha: description: "Commit to anchor to (default: HEAD)" required: false @@ -65,7 +70,11 @@ jobs: MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} steps: - run: | - valid="ix-compile aiur" + # Workload registry: the backend testbeds in Ix/Cli/BenchCmd.lean + # (backendSpecs) minus the runner-arch suffix. Static because this + # job runs on a cheap runner with no built `ix`; keep in sync when + # adding a backend. + valid="aiur-check-execute aiur-check-prove ix-compile ooc-check sp1-check-execute zisk-check-execute" if [ "$EVENT" = workflow_dispatch ]; then # Reset the chosen workload(s) at the given commit; no PR scan. sha="${INPUT_SHA:-$HEAD_SHA}" @@ -118,8 +127,11 @@ jobs: steps: - run: | # Accepted command tokens — applied verbatim as labels (incl. `all`, - # which the merge job expands into every workload). - accepted="ix-compile aiur all" + # which the merge job expands into every workload). Same static + # list as the reset job; keep both in sync with backendSpecs in + # Ix/Cli/BenchCmd.lean. + valid="aiur-check-execute aiur-check-prove ix-compile ooc-check sp1-check-execute zisk-check-execute" + accepted="$valid all" # Parse the workload token(s) after the command, lowercased. workloads=$(printf '%s' "$BODY" \ | grep -oiE '!bencher-thresholds-reset[[:space:]]+[a-z0-9 -]+' \ @@ -141,6 +153,7 @@ jobs: gh pr comment "$PR" --repo "$REPO" \ --body "♻️ Baseline reset queued for:$ok — will anchor to the merge commit when this PR merges." else + expected=$(printf '`%s`, ' $valid) gh pr comment "$PR" --repo "$REPO" \ - --body "⚠️ Reset command matched no known workload (expected \`ix-compile\`, \`aiur\`, or \`all\`). Nothing will reset on merge." + --body "⚠️ Reset command matched no known workload (expected ${expected}or \`all\`). Nothing will reset on merge." fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68f5d4514..b3276de80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,3 +62,56 @@ jobs: uses: EmbarkStudios/cargo-deny-action@v2 with: rust-version: ${{ env.RUST_VERSION }} + + # zkVM host build gate: do the Zisk/SP1 hosts (and their guest ELFs, via + # each workspace's build.rs) still compile? rust-test doesn't build these + # workspaces (special toolchains), and bench-main.yml's zkvm-execute job — + # which runs real executions — tolerates per-constant failures by design + # (dropped rows, OOM sentinels), so it never turns red on a breakage. These + # jobs are the red-X signal, kept cheap: build-only, no execution, and + # therefore no Zisk proving key (the key is loaded at runtime by + # `client.setup()`, never at build time — skipping it saves a ~3 GB download + # + const-tree regeneration per run). SP1 and Zisk build as independent jobs + # so they parallelize; each installs only its own toolchain via sp1up / + # ziskup (prebuilt binaries). The apt list inside the install actions is the + # shared superset both backends need (proofman's C++ links + # OpenMPI/OpenMP/GMP/…; SP1's host crates need pkg-config + libssl-dev). + sp1-build: + name: SP1 host build + runs-on: warp-ubuntu-latest-x64-16x + steps: + - uses: actions/checkout@v6 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: sp1 + - uses: ./.github/actions/install-sp1 + # The precompile-aware SP1 runner-binary is auto-built from the fork git + # dep by `sp1-core-executor-runner`'s build script — no manual override. + # `cargo test` reuses the build's dep graph and runs the host's unit + # tests — the clap surface `ix bench run` drives (`--consts` comma-splitting, + # `--consts-file` union/dedup), so a CLI regression reds this gate too. + - name: Build + test sp1-host (guest ELF via build.rs) + run: | + cd sp1 + cargo build --release --bin sp1-host + cargo test --release --bin sp1-host + + zisk-build: + name: Zisk host build + runs-on: warp-ubuntu-latest-x64-16x + steps: + - uses: actions/checkout@v6 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: zisk + - uses: ./.github/actions/install-zisk + with: + proving-key: false + # Unit tests: the clap surface `ix bench run` drives, plus the closure auditor + # (closure_detects_missing_dep self-skips without an IX_TEST_IXE + # fixture — this gate has no Lean build to produce one). + - name: Build + test zisk-host (guest ELF via build.rs) + run: | + cd zisk + cargo build --release --bin zisk-host + cargo test --release --bin zisk-host diff --git a/.github/workflows/ignored.yml b/.github/workflows/ignored.yml index 5e92186a4..997517468 100644 --- a/.github/workflows/ignored.yml +++ b/.github/workflows/ignored.yml @@ -8,9 +8,9 @@ on: permissions: contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true +# No concurrency group: push-to-main and manual dispatch only — every merged +# commit runs the extended tests; a later merge must never cancel an in-flight +# run. jobs: ignored-test: diff --git a/.github/workflows/riscv-bench.yml b/.github/workflows/riscv-bench.yml deleted file mode 100644 index d9ec22b7e..000000000 --- a/.github/workflows/riscv-bench.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: RISC-V bench - -# zkVM execute is ~10+ min (toolchain installs + host builds + emulation), so it -# is kept off the per-PR path: this workflow runs only on pushes to main (and on -# manual dispatch). It compiles the `minimal.ixe` fixture, then executes the -# kernel typecheck of one constant in the SP1 and Zisk VMs — in parallel jobs. -on: - push: - branches: main - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Compile a tiny env once (ix is already built here) and hand it to the zkVM - # execute jobs via artifact, so those jobs stay Lean-free. - compile-fixture: - name: Compile zkVM fixture (minimal.ixe) - runs-on: warp-ubuntu-latest-x64-16x - steps: - - uses: actions/checkout@v6 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - uses: leanprover/lean-action@v1 - with: - build-args: "--wfail -v" - - name: Compile zkVM test fixture (minimal.ixe) - run: lake exe ix compile Tests/MinimalDefs.lean --out minimal.ixe - - uses: actions/upload-artifact@v4 - with: - name: minimal-ixe - path: minimal.ixe - if-no-files-found: error - - # Execute the kernel typecheck of the `minimal.ixe` fixture natively (no Nix, - # no proof, no GPU). SP1 and Zisk run as independent jobs so they parallelize; - # each installs only its own toolchain via sp1up / ziskup (prebuilt binaries) - # and downloads the shared fixture. minimal.ixe carries the full Init closure, - # so we scope execution with `--constant myReflEq --skip-deps`: that - # subject-only-typechecks just the named constant, trusting its Init - # dependencies as Claim assumptions, instead of typechecking all of Init (which - # never finishes in the emulator). Each host bails non-zero on any typecheck - # failure; we also assert the `failures: 0` line. - # - # The apt list is the shared superset both backends need: the ZisK book's full - # Ubuntu list (its prebuilt cargo-zisk and proofman's C++ link OpenMPI, OpenMP, - # GMP, nlohmann-json, nasm, secp256k1, …) plus pkg-config + libssl-dev for - # SP1's host crates (openssl/bindgen). The Nix shells provided all this; a bare - # runner doesn't. Must precede the toolchain install (it runs cargo-zisk). - sp1-execute: - name: SP1 zkVM Execute - needs: compile-fixture - runs-on: warp-ubuntu-latest-x64-16x - steps: - - uses: actions/checkout@v6 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache-workspaces: sp1 - - name: Install system build deps - run: | - sudo apt-get update - sudo apt-get install -y \ - xz-utils jq curl build-essential qemu-system libomp-dev libgmp-dev \ - nlohmann-json3-dev protobuf-compiler uuid-dev libgrpc++-dev \ - libsecp256k1-dev libsodium-dev libpqxx-dev nasm libopenmpi-dev \ - openmpi-bin openmpi-common libclang-dev clang gcc-riscv64-unknown-elf \ - pkg-config libssl-dev - - uses: actions/download-artifact@v4 - with: - name: minimal-ixe - - name: Install SP1 toolchain (sp1up, latest) - run: | - curl -L https://sp1up.succinct.xyz | bash - ~/.sp1/bin/sp1up - echo "$HOME/.sp1/bin" >> "$GITHUB_PATH" - # The precompile-aware SP1 runner-binary is auto-built from the fork git - # dep by `sp1-core-executor-runner`'s build script — no manual override. - - name: SP1 — execute minimal.ixe (assert failures == 0) - run: | - cd sp1 - cargo run --bin sp1-host -- --execute --ixe ../minimal.ixe --constant myReflEq --skip-deps | tee only.txt - grep -qE "failures: 0\b" only.txt - - zisk-execute: - name: Zisk zkVM Execute - needs: compile-fixture - runs-on: warp-ubuntu-latest-x64-16x - steps: - - uses: actions/checkout@v6 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache-workspaces: zisk - - name: Install system build deps - run: | - sudo apt-get update - sudo apt-get install -y \ - xz-utils jq curl build-essential qemu-system libomp-dev libgmp-dev \ - nlohmann-json3-dev protobuf-compiler uuid-dev libgrpc++-dev \ - libsecp256k1-dev libsodium-dev libpqxx-dev nasm libopenmpi-dev \ - openmpi-bin openmpi-common libclang-dev clang gcc-riscv64-unknown-elf \ - pkg-config libssl-dev - - uses: actions/download-artifact@v4 - with: - name: minimal-ixe - - name: Install Zisk toolchain (ziskup, pinned v0.18.0) - # `--version 0.18.0` pins the toolchain to match our deps. Our host links - # the argumentcomputer/zisk `blake3-precompile` fork, which is based on - # v0.18.0 (its cargo-zisk has `check-setup`, used below to regenerate the - # key's const-trees). Without the pin, ziskup installs `releases/latest`, - # which resolves to upstream `v1.0.0-alpha` — a different circuit whose - # cargo-zisk dropped the `check-setup` subcommand, breaking the key step. - # `--cpu` picks the CPU build (no GPU on the runner) and `--nokey` skips - # ziskup's key install — both avoid its interactive /dev/tty prompts. We - # keep `--nokey` because the upstream `zisk-setup` bucket only carries the - # upstream circuit's key; our fork has a different circuit (extra Blake3f - # AIR), so we restore the fork-matching key from our own S3 in the next - # step. `--prefix $HOME/.zisk` pins the install where cargo-zisk's - # ZiskPaths fallback looks (the runner sets XDG_CONFIG_HOME, which would - # otherwise relocate it). - run: | - curl -L https://raw.githubusercontent.com/0xPolygonHermez/zisk/main/ziskup/install.sh \ - | bash -s -- --cpu --nokey -y --version 0.18.0 --prefix "$HOME/.zisk" - echo "$HOME/.zisk/bin" >> "$GITHUB_PATH" - # Execute still needs a proving key present: zisk-host calls - # `client.setup()` (which the SDK runs before the execute branch), and that - # loads the circuit's const-tree files. We host the fork-matching key in a - # public S3 bucket WITHOUT the const-trees — exactly like Zisk's released - # `zisk-provingkey-*.tar.gz` on `storage.googleapis.com/zisk-setup` — and - # regenerate them here with `cargo-zisk check-setup -a`, which is how - # `ziskup` itself populates them. That keeps the artifact ~3 GB (gzip) - # instead of ~48 GB. The object name carries the fork rev so a circuit - # change can't silently reuse a stale key. Public bucket → plain curl, no - # AWS creds. - - name: Restore Zisk proving key (fork circuit) from S3 - run: | - mkdir -p "$HOME/.zisk" - curl -fSL --retry 3 \ - https://argument-zisk-setup.s3.amazonaws.com/zisk-provingkey-blake3-8f9e24d5-cpu.tar.gz \ - -o /tmp/zisk-provingkey.tar.gz - tar -C "$HOME/.zisk" -xzf /tmp/zisk-provingkey.tar.gz - rm -f /tmp/zisk-provingkey.tar.gz - # Regenerate the const-tree files omitted from the artifact (CPU build, - # so no --gpu). This is the "may take a while" step ziskup prints. - cargo-zisk check-setup --proving-key "$HOME/.zisk/provingKey" -a - - name: Zisk — execute minimal.ixe (assert failures == 0) - run: | - cd zisk - # ZisK's ASM microservices mmap the ROM with MAP_LOCKED, which needs - # unlimited locked memory — the Zisk book's "Critical Memory - # Configuration" prescribes DefaultLimitMEMLOCK=infinity. The runner - # caps the memlock hard limit (so a bare `ulimit -l unlimited` can't - # raise it) and we can't reboot it, so raise the limit in-session as - # root via prlimit; the cargo child (and the ASM services it spawns) - # inherit it. Without this the services die with - # `mmap(rom) errno=11` / "shmem creation ... failed". - sudo prlimit --pid $$ --memlock=unlimited:unlimited - cargo run --bin zisk-host -- --execute --ixe ../minimal.ixe --constant myReflEq --skip-deps | tee only.txt - grep -qE "failures: 0\b" only.txt diff --git a/Benchmarks/Typecheck.lean b/Benchmarks/Typecheck.lean index 047cbec76..fbb20effa 100644 --- a/Benchmarks/Typecheck.lean +++ b/Benchmarks/Typecheck.lean @@ -5,6 +5,8 @@ import Ix.Aiur.Compiler import Ix.Aiur.Statistics import Ix.TracingTexray import Ix.Benchmark.Bench +import Ix.Benchmark.Results +import Ix.Cli.ConstsFile import Ix.Cli.NameResolve /-! @@ -15,28 +17,36 @@ kernel's `verify_claim` entrypoint, run over constants from a serialized `Ixon.Env` (`.ixe`). Reading the env from a `.ixe` keeps this on Ix's critical path (same load `ix check --ixe` uses) and avoids importing Lean modules at runtime. Useful standalone (per-constant timeline + RAM breakdown via -tracing-texray) and as a machine source (neutral results JSON). +tracing-texray) and as a machine source (benchmark results JSON). ``` -lake exe bench-typecheck --ixe [names…] [flags] +lake exe bench-typecheck --ixe --consts [--consts-file

] [flags] - --ixe serialized `Ixon.Env`, e.g. from `ix compile Foo.lean` - (writes `foo.ixe`). Required. - [names…] zero or more fully-qualified constant names to benchmark, - e.g. `Nat.add_comm String.append`. - --manifest additionally read names from a file: one per line, blank - lines and `#` comments ignored. Unions with [names…]. + --ixe serialized `Ixon.Env`, e.g. from `ix compile Foo.lean` + (writes `foo.ixe`). Required. + --consts comma-separated fully-qualified constant names to + benchmark (e.g. `Nat.add_comm,String.append`). Same + flag/shape as `ix check --consts`, `zisk-host --consts`, + and `sp1-host --consts`. + --consts-file additionally read names from a file: one per line, blank + lines and `#` comments ignored. Unions with --consts. - Names (from either source) resolve against the env's named map via + Names (from any source) resolve against the env's named map via `String.toName` plus a `toString` fallback (mirrors `ix check --ixe`), so numeric / private components round-trip (`Foo.0.Bar`, `_private.M.0.foo`). - Pass at least one name or a `--manifest`. + Pass at least one name via --consts or --consts-file. + --skip-deps check only each target itself (verify_const, trusting its + deps) instead of its whole transitive closure (verify_claim, + the default). Same flag as `zisk-host --skip-deps`; reserved + for targets too expensive to full-closure-check. --json write per-constant results JSON to . Off by default: normal CLI usage prints only the human-readable summary. - --texray force the tracing-texray timeline + RAM breakdown on. - --no-texray force it off. Default: on iff `--json` was NOT given, so a - plain local run gets the breakdown while a JSON run stays quiet. + --texray enable the tracing-texray timeline + RAM breakdown. With + --json , per-phase span timings are also written to + .spans (JSON Lines) for the CI drill-down. Off by default. + --execute-only run only Phase 1 (constants / fft-cost / execute-time) and skip + proving — the fast `execute`-mode signal. ``` For each constant the harness STARK-checks `Ix.Claim.check addr none` (the full @@ -47,8 +57,11 @@ transitive typecheck) in two phases: (Σ width·height·log2(height) over circuits — the proving-cost proxy), and `execute-time`. 2. **Prove** (cheap→expensive by measured fft-cost): the end-to-end STARK prove, - recording `prove-time`. With texray on, each prove emits a per-span timeline - (`aiur/execute`, `aiur/witness`, `stark/...`) with RAM Δ/peak to stderr. + recording `prove-time`, the serialized `proof-size` (bytes), and + `verify-time` (verifying the fresh proof) — prover changes can trade speed + against proof size or verification cost, so all three are tracked. With + texray on, each prove emits a per-span timeline (`aiur/execute`, + `aiur/witness`, `stark/...`) with RAM Δ/peak to stderr. When `--json` is set the file is rewritten after every prove, so an external `timeout` still leaves a complete file of the results collected so far (cheapest @@ -56,10 +69,16 @@ first). A name absent from the env or whose execution errors is skipped with a warning, so a single bad name never fails the run. The harness imposes no time limit; bound a run with an external `timeout` if needed. -The JSON is a neutral, flat shape (`{ "": { "constants": …, "fft-cost": …, -"execute-time": …, "prove-time": …, "throughput": … } }`, where `prove-time` and -`throughput` appear only for proven constants); any bencher-specific reshaping -is the caller's job (see `.github/workflows/bench-main.yml`). +The JSON is a flat shape (`{ "": { "constants": …, "fft-cost": …, +"execute-time": …, "prove-time": …, "proof-size": …, "verify-time": …, +"throughput": …, "peak-rss": … } }`). `peak-rss` and `throughput` are +phase-scoped by MODE: an `--execute-only` row carries the Phase-1 RSS +high-water and constants/sec over the execute; a prove row carries the +prover's high-water and constants/sec over the prove (with `prove-time`, +`proof-size`, `verify-time` present only once proven). The two modes are +stored on separate bencher testbeds, so the shared names never collide. Any +bencher-specific reshaping is the caller's job (see +`.github/workflows/bench-main.yml`). -/ open Lean (Json Name) @@ -77,15 +96,6 @@ def friParameters : Aiur.FriParameters := { queryProofOfWorkBits := 0 } -/-- Manifest lines as raw strings: one name per line. Everything from a `#` to - end of line is a comment (whole-line or inline); blank lines are dropped. - `#` never appears in a Lean name, so splitting on it is safe. Resolution - against the env happens later (so the `toString` fallback can see the - displayed form the user wrote). -/ -def parseManifest (contents : String) : Array String := - (contents.splitOn "\n").filterMap (fun line => - let s := ((line.splitOn "#").head?.getD "").trimAscii - if s.isEmpty then none else some s.toString) |>.toArray /-- Per-constant measurements. `proveSec` is `none` when the constant was @@ -95,28 +105,83 @@ structure Result where constants : Nat fftCost : Float executeSec : Float + /-- The kernel REJECTED the constant (Phase-1 check error). The JSON entry + is `{"status": "rejected"}` (see `Ix.Benchmark.Results`) — a rejected + constant is a correctness signal, not a benchmark datum — Phase 2 + skips it, and the run exits with the reserved rejection code. -/ + failed : Bool := false proveSec : Option Float := none + /-- Serialized proof size in bytes (`Aiur.Proof.toBytes`). Tracked because + prover changes can trade speed against proof size. -/ + proofSize : Option Nat := none + /-- Wall time of `AiurSystem.verify` over the fresh proof — the other side + of the same trade-off. `none` if verification failed (reported loudly). -/ + verifySec : Option Float := none + /-- The constant's prove-phase RSS high-water in bytes (tracing-texray + tree sampler; the window resets per prove). -/ + peakRss : Option Nat := none + /-- The constant's own Phase-1 (execute) RSS high-water mark — the sampler + window resets per constant. Emitted as `peak-rss` in execute-only + mode; a prove row's `peak-rss` is the prover's window instead (the + prover would dwarf an execute peak, so the modes never share a row — + or a testbed). -/ + executePeakRss : Option Nat := none deriving Inhabited -/-- Round a Float to `d` decimal places, to keep the emitted JSON readable. -/ -def roundTo (d : Nat) (f : Float) : Float := +/-- A `Json` number with at most `d` decimal places, rendered decimally. + `Float`'s own `ToJson` prints the full binary representation + (`0.02602000000000000146…`), so build the `JsonNumber` (mantissa · + 10⁻ᵈ) directly from the rounded value instead. -/ +def jsonRound (d : Nat) (f : Float) : Json := let scale := (10.0 : Float) ^ d.toFloat - (f * scale).round / scale + let scaled := f * scale + let m : Int := + if scaled < 0 then -Int.ofNat (-scaled).round.toUInt64.toNat + else Int.ofNat scaled.round.toUInt64.toNat + Json.num ⟨m, d⟩ -/-- Neutral, flat results object: `name → { constants, fft-cost, execute-time, - prove-time?, throughput? }`. No bencher-specific shaping. -/ -def Result.toJsonEntry (r : Result) : String × Json := +/-- Flat results object: `name → { constants, fft-cost, execute-time, … }`. + No bencher-specific shaping. + + `peak-rss` and `throughput` are PHASE-SCOPED BY MODE, not by name: an + execute-only run's row carries the Phase-1 peak and constants/sec over + the execute; a prove run's row carries the prove-phase peak and + constants/sec over the prove. The two modes upload to separate bencher + testbeds (aiur-check-execute-* / aiur-check-prove-*), so the shared names never + collide — run the execute cell when you want execute-side numbers. -/ +def Result.toJsonEntry (executeOnly : Bool) (r : Result) : String × Json := + if r.failed then + (r.name, Json.mkObj [("status", Json.str "rejected")]) else let base : List (String × Json) := - [ ("constants", Lean.toJson r.constants) - , ("fft-cost", Lean.toJson (roundTo 0 r.fftCost)) - , ("execute-time", Lean.toJson (roundTo 6 r.executeSec)) ] - -- prove-time and the derived proving throughput (constants/prove-time, the - -- proving analog of compile's constants/sec) are present only once proven. - let fields := match r.proveSec with - | some p => base ++ [ ("prove-time", Lean.toJson (roundTo 6 p)) - , ("throughput", Lean.toJson (roundTo 2 (r.constants.toFloat / p))) ] - | none => base - (r.name, Json.mkObj fields) + [ ("status", Json.str "ok") + , ("constants", Lean.toJson r.constants) + , ("fft-cost", jsonRound 0 r.fftCost) + , ("execute-time", jsonRound 6 r.executeSec) ] + if executeOnly then + let fields := if r.executeSec > 0 then + base ++ [ ("throughput", jsonRound 2 (r.constants.toFloat / r.executeSec)) ] + else base + let fields := match r.executePeakRss with + | some n => fields ++ [ ("peak-rss", Lean.toJson n) ] + | none => fields + (r.name, Json.mkObj fields) + else + -- prove-time, the proving throughput, and the prove-phase peak are + -- present only once proven. + let fields := match r.proveSec with + | some p => base ++ [ ("prove-time", jsonRound 6 p) + , ("throughput", jsonRound 2 (r.constants.toFloat / p)) ] + | none => base + let fields := match r.peakRss with + | some n => fields ++ [ ("peak-rss", Lean.toJson n) ] + | none => fields + let fields := match r.proofSize with + | some n => fields ++ [ ("proof-size", Lean.toJson n) ] + | none => fields + let fields := match r.verifySec with + | some v => fields ++ [ ("verify-time", jsonRound 6 v) ] + | none => fields + (r.name, Json.mkObj fields) /-- Time a thunk, returning its value and the elapsed seconds. The result is forced by `blackBoxIO` so a pure computation isn't optimized away. -/ @@ -130,37 +195,42 @@ def runTypecheckCmd (p : Cli.Parsed) : IO UInt32 := do let some ixeArg := p.flag? "ixe" | IO.eprintln "error: --ixe is required"; return 1 let ixePath := ixeArg.as! String - -- Names come from the variadic positional args and/or a `--manifest` file. - let cliNames := p.variableArgsAs! String - let fileNames ← match p.flag? "manifest" with - | some f => pure (parseManifest (← IO.FS.readFile (f.as! String))) - | none => pure #[] - -- Union, preserving first-seen order, so the same const isn't proven twice. - let nameArgs := Id.run do - let mut seen : Std.HashSet String := {} - let mut acc : Array String := #[] - for n in cliNames ++ fileNames do - if !seen.contains n then seen := seen.insert n; acc := acc.push n - return acc + -- `--consts` comma-list ∪ `--consts-file`, shared grammar + dedup + -- (Ix.Cli.ConstsFile — same parser as `ix check-rs`). Raw strings: + -- resolution against the env happens later (so the `toString` fallback can + -- see the displayed form the user wrote). + let nameArgs ← Ix.Cli.ConstsFile.gather p if nameArgs.isEmpty then - IO.eprintln "error: provide one or more constant names and/or --manifest " + IO.eprintln "error: provide at least one constant via --consts and/or --consts-file " return 1 let jsonOut : Option String := (p.flag? "json").map (·.as! String) - -- subject-only: check just the target (`verify_const`, trusting its deps) + -- skip-deps: check just the target (`verify_const`, trusting its deps) -- instead of re-checking the whole transitive closure (`verify_claim`). - let subjectOnly := p.hasFlag "subject-only" - -- Default: trace iff we're not in JSON/bencher mode. - let useTexray := - if p.hasFlag "texray" then true - else if p.hasFlag "no-texray" then false - else jsonOut.isNone + let skipDeps := p.hasFlag "skip-deps" + -- Execute-only: run just Phase 1 (constants / fft-cost / execute-time) and + -- skip the Phase 2 prove loop. + let executeOnly := p.hasFlag "execute-only" + -- Off by default; CI passes --texray explicitly. + let useTexray := p.hasFlag "texray" + -- Start the process-tree RSS sampler so each Result's peak-rss reflects the + -- true high-water mark. With --texray, install the streaming subscriber up + -- front: every phase span — aiur/execute_ixvm in Phase 1 included — + -- renders its duration and RAM Δ/peak through the one shared channel + -- instead of per-benchmark arithmetic. With --json too, the per-span sink + -- also lands the timings at `.spans` as JSON Lines for the CI + -- comparison. + TracingTexray.startSampler + if useTexray then TracingTexray.init {} + match useTexray, jsonOut with + | true, some path => TracingTexray.jsonSink s!"{path}.spans" + | _, _ => pure () -- Compile the IxVM kernel once; build the prover system once. let .ok toplevel := IxVM.ixVM | throw (IO.userError "Merging IxVM kernel failed") let .ok compiled := toplevel.compile | throw (IO.userError "Compilation of IxVM kernel failed") - let entrypoint := if subjectOnly then `verify_const else `verify_claim + let entrypoint := if skipDeps then `verify_const else `verify_claim let some funIdx := compiled.getFuncIdx entrypoint | throw (IO.userError s!"{entrypoint} entrypoint missing") let aiurSystem := Aiur.AiurSystem.build compiled.bytecode commitmentParameters @@ -190,51 +260,91 @@ def runTypecheckCmd (p : Cli.Parsed) : IO UInt32 := do -- Phase 1: execute every constant (cheap, deterministic structural metrics). -- For full-closure check claims, use `checkAddrWithEnv` against the - -- shared `envHandle`. For `--subject-only` (`buildVerifyConst`), the + -- shared `envHandle`. For `--skip-deps` (`buildVerifyConst`), the -- witness is a small subject-only blob — keep Lean witness + -- `executeIxVM`. IO.println "── Phase 1: execute (witness generation) ──" let mut execed : Array (Result × Address) := #[] + let mut execIdx := 0 for (label, addr) in targets do + execIdx := execIdx + 1 try + -- Announce BEFORE the execute (flushed): a kill mid-execute must + -- leave the in-flight constant's name in the log. + IO.println s!" [{execIdx}/{targets.size}] executing {label} …" + (← IO.getStdout).flush + -- Windowed high-water: reset per constant so each row's + -- execute peak is its own, not the loop's running maximum. + TracingTexray.resetPeakTreeRss let (res, execSec) ← timed fun _ => - if subjectOnly then + if skipDeps then let witness := IxVM.ClaimHarness.buildVerifyConst ixonEnv addr compiled.bytecode.executeIxVM funIdx witness.input witness.inputIOBuffer else compiled.bytecode.checkAddrWithEnv funIdx envHandle addr.hash + let execPeak ← TracingTexray.peakTreeRssBytes match res with - | .error e => IO.eprintln s!" execute {label} failed: {e}" + | .error e => + IO.eprintln s!" ❌ {label} FAILED TO TYPECHECK: {e}" + execed := execed.push + ({ name := label, constants := 0, fftCost := 0, executeSec := 0, + failed := true }, addr) | .ok (_, _, queryCounts) => let stats := Aiur.computeStats compiled queryCounts let constants := (IxVM.ClaimHarness.closureFrom ixonEnv addr).size + -- Throughput via the shared benchmark framework; duration/RAM per + -- phase stream from texray's `aiur/execute_ixvm` span line. + let thrpt := (Throughput.Elements constants.toUInt64 "consts").formatRate + (execSec * 1e9) IO.println s!" {label}: constants={constants} fft-cost={stats.totalFftCost} \ - execute={execSec}s" + execute={execSec}s thrpt={thrpt}" execed := execed.push - ({ name := label, constants, fftCost := stats.totalFftCost, executeSec := execSec }, addr) + ({ name := label, constants, fftCost := stats.totalFftCost, + executeSec := execSec, executePeakRss := some execPeak }, addr) catch e => IO.eprintln s!" execute {label} threw: {e}" - -- Write the neutral results JSON, but only when `--json` was given. Rewritten - -- after each prove so a `timeout` kill still leaves a complete file. + -- Persist rows when `--json` was given, MERGING into the file (the shared + -- results-row contract): rows land after each result, so a kill leaves the + -- completed ones, and per-constant processes can share one file with the + -- orchestrator and each other. let writeJson (results : Array Result) : IO Unit := match jsonOut with | some path => - IO.FS.writeFile path (Json.mkObj (results.map Result.toJsonEntry).toList).pretty + results.forM fun r => + let (name, row) := Result.toJsonEntry executeOnly r + Ix.Benchmark.Results.writeEntry path name row | none => pure () - -- Phase 2: prove cheap→expensive. Refine each entry with its prove-time as it - -- lands. Install texray first so the prove spans (timeline + RAM Δ/peak) render. - if useTexray then TracingTexray.init {} + -- `--execute-only`: stop after Phase 1; the results JSON (if requested) is + -- already complete with the execute metrics. + if executeOnly then + writeJson (execed.map (·.1)) + match jsonOut with + | some path => IO.println s!"wrote {execed.size} execute-only benchmarks to {path}" + | none => IO.println s!"executed {execed.size} constants (--execute-only); pass --json to emit results" + return if execed.any (·.1.failed) then Ix.Benchmark.Results.exitRejected else 0 + + -- Phase 2: prove cheap→expensive. Refine each entry with its prove-time as + -- it lands (texray was installed before Phase 1, so the prove spans render + -- the same way the execute ones did). IO.println "── Phase 2: prove ──" let mut ordered := execed.qsort (·.1.fftCost < ·.1.fftCost) writeJson (ordered.map (·.1)) let mut spent : Float := 0.0 for i in [:ordered.size] do let (r, addr) := ordered[i]! + if r.failed then continue try + -- Announce BEFORE the prove (flushed): when a watchdog kill or OOM + -- lands mid-prove, the log must already say which constant died. + IO.println s!" [{i + 1}/{ordered.size}] proving {r.name} (fft-cost={r.fftCost}) …" + (← IO.getStdout).flush + -- Windowed high-water: each prove's peak is its own window, not + -- the run's cumulative maximum (mirrors the Phase-1 resets). + TracingTexray.resetPeakTreeRss let (proveRes, proveSec) ← timed fun _ => - if subjectOnly then + if skipDeps then let witness := IxVM.ClaimHarness.buildVerifyConst ixonEnv addr let (claim, proof, ioBuf) := aiurSystem.proveIxVM friParameters funIdx witness.input witness.inputIOBuffer @@ -243,41 +353,55 @@ def runTypecheckCmd (p : Cli.Parsed) : IO UInt32 := do else match aiurSystem.proveAddrWithEnv friParameters funIdx envHandle addr.hash with | .error e => .error e - | .ok (_claimBytes, proof, ioBuf) => - -- The shared envHandle path doesn't return an `Array G` - -- claim — adapt to the existing benchmark return shape - -- by recomputing the claim digest from the witness's - -- input (Phase 2 doesn't read it). - .ok (#[], proof, ioBuf) + | .ok (claimBytes, proof, ioBuf) => + -- The envHandle path returns the SERIALIZED `Ix.Claim`; rebuild + -- the Array-G claim `verify` takes — `verify_claim`'s input is + -- the 32-G blake3 digest of those bytes (same recipe as + -- `ix verify`). + let digest := Address.blake3 claimBytes + let claim := + Aiur.buildClaim funIdx (digest.hash.data.map .ofUInt8) #[] + .ok (claim, proof, ioBuf) match (proveRes : Except String (Array Aiur.G × Aiur.Proof × Aiur.IOBuffer)) with | .error e => IO.eprintln s!" prove {r.name} failed: {e}"; continue - | .ok _ => pure () - spent := spent + proveSec - IO.println s!" {r.name}: prove={proveSec}s (cumulative {spent}s)" - ordered := ordered.set! i ({ r with proveSec := some proveSec }, addr) - writeJson (ordered.map (·.1)) + | .ok (claim, proof, _ioBuf) => + spent := spent + proveSec + let peak ← TracingTexray.peakTreeRssBytes + let proofSize := (Aiur.Proof.toBytes proof).size + let (verifyRes, verifySec) ← timed fun _ => + aiurSystem.verify friParameters claim proof + let verifySec? ← match verifyRes with + | .ok () => pure (some verifySec) + | .error e => + IO.eprintln s!" verify {r.name} FAILED: {e}" + pure none + IO.println s!" {r.name}: prove={proveSec}s verify={verifySec}s \ + proof={proofSize} bytes (cumulative {spent}s)" + ordered := ordered.set! i + ({ r with proveSec := some proveSec, peakRss := some peak + , proofSize := some proofSize, verifySec := verifySec? }, addr) + writeJson (ordered.map (·.1)) catch e => IO.eprintln s!" prove {r.name} threw: {e}" match jsonOut with | some path => IO.println s!"wrote {ordered.size} benchmarks to {path} ({spent}s proving)" | none => IO.println s!"proved {ordered.size} constants ({spent}s); pass --json to emit results" - return 0 + return if ordered.any (·.1.failed) then Ix.Benchmark.Results.exitRejected else 0 def typecheckCmd : Cli.Cmd := `[Cli| typecheck VIA runTypecheckCmd; "Benchmark IxVM-kernel execution + proving of `Ix.Claim.check` over `.ixe` constants" FLAGS: - "ixe" : String; "Path to a serialized `Ixon.Env` (e.g. produced by `ix compile`). Required." - "manifest" : String; "Additionally read constant names from a file (one per line; `#` comments and blank lines ignored). Unions with the positional names." + "ixe" : String; "Path to a serialized `Ixon.Env` (e.g. produced by `ix compile`). Required." + "consts" : String; "Comma-separated fully-qualified constant names to benchmark (e.g. `Nat.add_comm,String.append`). Same flag/shape as `ix check --consts`, `zisk-host --consts`, and `sp1-host --consts`." + "consts-file" : String; "Additionally read constant names from a file (one per line; `#` comments and blank lines ignored). Unions with --consts." "json" : String; "Write per-constant results JSON to this path. Off by default; normal CLI usage prints only the human-readable summary." - "subject-only"; "Check only each target itself (verify_const, trusting its deps) instead of re-checking its whole transitive closure (verify_claim)." - texray; "Force the tracing-texray timeline + RAM breakdown on (per-prove spans on stderr)." - "no-texray"; "Force the breakdown off. Default: on iff --json was not given." + "skip-deps"; "Check only each target itself (verify_const, trusting its deps) instead of re-checking its whole transitive closure (verify_claim). Same flag as `zisk-host --skip-deps`." + "execute-only"; "Execute only (Phase 1: constants / fft-cost / execute-time) and skip proving. The fast per-PR `execute`-mode signal." + texray; "Enable the tracing-texray timeline + RAM breakdown (per-prove spans on stderr). Combined with --json, per-phase span timings are additionally written to `.spans` as JSON Lines for the CI drill-down. Off by default." - ARGS: - ...names : String; "Fully-qualified constant name(s) to benchmark (e.g. `Nat.add_comm String.append`). Optional if `--manifest` is given." ] def main (args : List String) : IO UInt32 := diff --git a/Benchmarks/Vectors.csv b/Benchmarks/Vectors.csv new file mode 100644 index 000000000..c2be8be65 --- /dev/null +++ b/Benchmarks/Vectors.csv @@ -0,0 +1,90 @@ +# Benchmark constant vectors -- single shared source of truth for which +# constants to run, selected per cell by `ix bench run` (see +# Ix/Cli/BenchCmd.lean, whose registry holds everything that isn't a +# per-constant row). Measurements (fft, cycles, prove-time, …) live in the +# benchmark results JSON each tool emits and in bencher.dev — never here. +# +# Columns (shard_target and primary default to 0 when trailing; most rows only +# carry the first three): +# name fully-qualified Lean name (resolves via NameResolve.resolveIxeAddr). +# env compile target / .ixe it resolves in: InitStd | Lean | Mathlib. +# tier cheap = prove-feasible on a CI runner; heavy = a single-shard +# prove exceeds the RAM watchdog ceiling (expect an OOM row). +# `ix bench run` proves every selected constant regardless of +# tier (--full prove runs default to cheap; --tier overrides); +# heavy zisk constants run as their closure-shard partition, +# pre-cut by `ix bench shard`. +# shard_target 1 = heavy constant designated as a multi-shard prove target +# (--shard-only restricts to these). +# primary 1 = part of the primary subset spanning shape + the +# cheap->heavy cost range. Default for the !benchmark PR +# comment and the bench-main cells (full set via BENCH_FULL=1 +# / --full). +name,env,tier,shard_target,primary +HEq,InitStd,cheap +Nat,InitStd,cheap +Eq.rec,InitStd,cheap +HEq.rec,InitStd,cheap +Trans.mk,InitStd,cheap +Array.toList,InitStd,cheap +Acc.rec,InitStd,cheap +Sum.elim,InitStd,cheap +Prod.map,InitStd,cheap +Option.bind,InitStd,cheap +Except.bind,InitStd,cheap +WellFounded.fix,InitStd,cheap +Nat.add,InitStd,cheap +List.filterMap,InitStd,cheap +Int.add,InitStd,cheap +BitVec.toFin,InitStd,cheap +Nat.add_comm,InitStd,cheap,0,1 +USize.toNat,InitStd,cheap +Nat.decEq,InitStd,cheap +ByteSlice.ofByteArray,InitStd,cheap +Nat.decLe,InitStd,cheap +Nat.strongRecOn,InitStd,cheap +Int.emod,InitStd,cheap +Array.foldlM,InitStd,cheap +Array.filter,InitStd,cheap +Nat.sub_le_of_le_add,InitStd,cheap,0,1 +BitVec.add,InitStd,cheap +Int.gcd,InitStd,cheap,0,1 +Nat.toDigits,InitStd,cheap +Array.map,InitStd,cheap +Lean.Name.hash,InitStd,cheap +BitVec.umod,InitStd,cheap +Nat.repr,InitStd,cheap +String.intercalate,InitStd,heavy +_private.Init.Prelude.0.Lean.extractMainModule._unsafe_rec,InitStd,heavy +Char.toLower,InitStd,heavy +Nat.gcd_comm,InitStd,heavy,0,1 +Int.emod_emod_of_dvd,InitStd,heavy +Array.append_assoc,InitStd,heavy +Vector.append,InitStd,heavy,0,1 +Fin.foldl,InitStd,heavy +List.mergeSort,InitStd,heavy,1,1 +Array.binSearch,InitStd,heavy,1 +Array.qsortOrd,InitStd,heavy +String.split,InitStd,heavy,0,1 +Std.Time.Week.Offset.ofMilliseconds,InitStd,heavy +Vector.extract_append._proof_2,InitStd,heavy,1,1 +ByteArray.utf8DecodeChar?_utf8EncodeChar_append,InitStd,heavy,0,1 +String.append,InitStd,cheap,0,1 +_private.Init.Data.Range.Polymorphic.SInt.0.Int8.instRxcHasSize_eq,InitStd,heavy,0,1 +_private.Init.Data.Range.Polymorphic.SInt.0.Int16.instRxcHasSize_eq,InitStd,heavy,0,1 +_private.Init.Data.Range.Polymorphic.SInt.0.Int32.instRxcHasSize_eq,InitStd,heavy,0,1 +_private.Init.Data.Range.Polymorphic.SInt.0.Int64.instRxcHasSize_eq,InitStd,heavy,0,1 +Char.ofOrdinal_le_of_le,InitStd,heavy,0,1 +Array.extract_append,InitStd,heavy,0,1 +Std.Tactic.BVDecide.BVExpr.bitblast.goCache_Inv_of_Inv._mutual,InitStd,heavy,0,1 +Lean.Expr.replace,Lean,cheap +List.Sorted,Mathlib,cheap +Nat.choose,Mathlib,cheap +Nat.factorial,Mathlib,cheap,0,1 +Nat.fib,Mathlib,cheap +GCDMonoid.gcd,Mathlib,heavy +Nat.Prime.two_le,Mathlib,heavy +Finset.prod,Mathlib,heavy +Finset.sum,Mathlib,heavy +Polynomial.eval,Mathlib,heavy +Multiset.sort,Mathlib,heavy,1,1 diff --git a/Cargo.lock b/Cargo.lock index 1efb6d500..7e44d6c16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,6 +1740,14 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ix-bench" +version = "0.1.0" +dependencies = [ + "serde_json", + "tracing-texray", +] + [[package]] name = "ix-common" version = "0.1.0" @@ -1787,6 +1795,7 @@ dependencies = [ "iroh", "iroh-base", "itertools 0.14.0", + "ix-bench", "ix-common", "ix-compile", "ix-kernel", @@ -1800,6 +1809,7 @@ dependencies = [ "rayon", "rustc-hash", "serde", + "serde_json", "sha2 0.10.9", "tiny-keccak", "tokio", @@ -3085,7 +3095,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -4022,7 +4032,7 @@ dependencies = [ [[package]] name = "tracing-texray" version = "0.2.0" -source = "git+https://github.com/argumentcomputer/tracing-texray?rev=31d194dd1bc50458d26f77c89bb68f67e5d1c149#31d194dd1bc50458d26f77c89bb68f67e5d1c149" +source = "git+https://github.com/argumentcomputer/tracing-texray?rev=465bbca0bea4721e58419c11cabd8cce21757822#465bbca0bea4721e58419c11cabd8cce21757822" dependencies = [ "loom", "parking_lot", @@ -4499,7 +4509,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4508,16 +4518,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4535,31 +4536,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4577,96 +4561,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 456f0b7a1..719fee16f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/aiur", + "crates/bench", "crates/common", "crates/compile", "crates/ffi", @@ -29,6 +30,7 @@ license = "MIT OR Apache-2.0" # Internal crates aiur = { path = "crates/aiur" } ixvm-codegen = { path = "crates/ixvm-codegen" } +ix-bench = { path = "crates/bench" } ix-common = { path = "crates/common" } ix-compile = { path = "crates/compile" } ixon = { path = "crates/ixon" } @@ -54,11 +56,12 @@ quickcheck = "1.0.3" quickcheck_macros = "1.0.0" rayon = "1" rustc-hash = "2" +serde_json = "1" sha2 = "0.10" tiny-keccak = { version = "2", features = ["keccak"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing-texray = { git = "https://github.com/argumentcomputer/tracing-texray", rev = "31d194dd1bc50458d26f77c89bb68f67e5d1c149" } +tracing-texray = { git = "https://github.com/argumentcomputer/tracing-texray", rev = "465bbca0bea4721e58419c11cabd8cce21757822" } [workspace.lints.rust] invalid_reference_casting = "warn" diff --git a/Ix/Benchmark/Results.lean b/Ix/Benchmark/Results.lean new file mode 100644 index 000000000..c84f87c84 --- /dev/null +++ b/Ix/Benchmark/Results.lean @@ -0,0 +1,78 @@ +/- + The benchmark-results row contract, Lean side. + + Every measured tool reports results as one JSON object per benchmark name + in a single file: + + { "": { "status": "ok" | "rejected" | "oom", + "": , ... } } + + Rows are flushed after every name, so a killed process still leaves a + complete file of the rows measured so far. Tools write `ok` or `rejected`; + `oom` is merged in by the orchestrator (`ix bench run`) after an abnormal + exit — a tool never observes its own OOM kill. + + Exit codes are the failure channel (no stdout markers, no log grepping): + 0 = every row ok; `exitUsage` = bad invocation; `exitRejected` = the + kernel rejected at least one constant (its row is on disk). Any other + exit is an infrastructure failure. The Rust mirror of this contract is + `crates/bench` (`ix-bench`); the constants must stay in lockstep. +-/ +module +public import Lean.Data.Json + +public section + +namespace Ix.Benchmark.Results + +/-- Bad invocation (unknown name, missing input, conflicting flags). -/ +def exitUsage : UInt32 := 2 + +/-- The kernel rejected at least one constant; its row is on disk. -/ +def exitRejected : UInt32 := 3 + +/-- A `Json` number with at most `d` decimal places, rendered decimally. + `Float`'s own `ToJson` prints the full binary representation + (`0.02602000000000000146…`), so build the `JsonNumber` (mantissa · + 10⁻ᵈ) directly from the rounded value instead. -/ +def jsonRound (d : Nat) (f : Float) : Lean.Json := + let scale := (10.0 : Float) ^ d.toFloat + let scaled := f * scale + let m : Int := + if scaled < 0 then -Int.ofNat (-scaled).round.toUInt64.toNat + else Int.ofNat scaled.round.toUInt64.toNat + Lean.Json.num ⟨m, d⟩ + +/-- Read the results object at `path`, tolerating a missing file or + unparseable content (both yield `{}` — a fresh accumulator). -/ +def readRows (path : String) : IO Lean.Json := do + if ← System.FilePath.pathExists path then + let s ← IO.FS.readFile path + match Lean.Json.parse s with + | .ok j@(.obj _) => return j + | _ => return Lean.Json.mkObj [] + else + return Lean.Json.mkObj [] + +/-- Write a prebuilt row object into the results file at `path`, merging + into any existing object (overwriting on collision) — the shape every + tool uses, so per-constant processes can safely share one file. + + The write is ATOMIC (temp file + rename): the accumulator is shared by + N sequential processes and their orchestrator, and a watchdog KILL + landing mid-write would otherwise truncate it — which the tolerant + `readRows` would then silently reset to `{}`, losing every prior + constant's row. -/ +def writeEntry (path : String) (name : String) (row : Lean.Json) : IO Unit := do + let existing ← readRows path + let tmp := path ++ ".tmp" + IO.FS.writeFile tmp ((existing.setObjVal! name row).compress) + IO.FS.rename tmp path + +/-- Write the row `{ "": { "status": , } }` into the + results file at `path`. -/ +def writeRow (path : String) (name : String) (status : String) + (fields : List (String × Lean.Json)) : IO Unit := + writeEntry path name (Lean.Json.mkObj (("status", Lean.Json.str status) :: fields)) + +end Ix.Benchmark.Results diff --git a/Ix/Cli/BenchCmd.lean b/Ix/Cli/BenchCmd.lean new file mode 100644 index 000000000..5ba0e1782 --- /dev/null +++ b/Ix/Cli/BenchCmd.lean @@ -0,0 +1,591 @@ +/- + `ix bench run`: the benchmark cell orchestrator — one command that + reproduces a CI benchmark cell locally, byte-for-byte on the same tools. + + A cell is (backend, env, mode). The orchestrator: + + 1. selects constant names from `Benchmarks/Vectors.csv` (filters ported + from the `!benchmark` manifest: env, tier, primary subset, shard flag); + 2. resolves the env's `.ixe` (an explicit `--ixe` path, else `ix + compile` — except for the `compile` backend, where the compile IS + the benchmark); + 3. spawns the cell's measured tool — `bench-typecheck` (aiur), + `zisk-host`/`sp1-host` (zkVM execute), `ix check-rs` (ooc), + `ix compile` (compile) — wrapped in the RAM watchdog (`watchdog.sh`: + cgroup `memory.max` via a systemd user scope; the kernel OOM-kills + at the ceiling). The per-constant backends (aiur, zkVM) spawn + ONE PROCESS PER CONSTANT: a kill costs exactly that constant (its row + is marked `status: oom`, keeping whatever the tool flushed), and each + spawn's texray window (`.spans`) belongs wholly to it, folded + into the row as flat `phase:` fields — independent bencher + measures with no attribution machinery; + 4. gates the cell on the row contract (`Ix.Benchmark.Results`): exit 3 + when any row is `rejected`, exit 1 when NO rows were produced, else 0. + + Every tool self-reports through the same `--json` results-rows contract, + so there is no output scraping anywhere: state flows through rows and + exit codes only. Registry data (envs, backends, testbeds) lives in this + module (`envSpecs`/`backendSpecs`) — one language, one owner. + + Note for the zisk backend: ZisK's ASM microservices need an unlimited + memlock hard limit (mmap with MAP_LOCKED). Raise it in the invoking shell + before running (`sudo prlimit --pid $$ --memlock=unlimited:unlimited`); + the tools inherit it. +-/ +module +public import Cli +public import Lean.Data.Json +public import Ix.Benchmark.Results +public import Ix.Cli.ConstsFile + +public section + +open System (FilePath) +open Ix.Benchmark.Results + +namespace Ix.Cli.BenchCmd + +/-- One `Vectors.csv` row (comments/header dropped). `shardTarget` and + `primary` default to false when the trailing columns are omitted. -/ +structure VectorRow where + name : String + env : String + tier : String + shardTarget : Bool + primary : Bool + +def parseVectorsCsv (contents : String) : Array VectorRow := + (contents.splitOn "\n").filterMap (fun line => + let s := ((line.splitOn "#").head?.getD "").trimAscii.toString + if s.isEmpty then none else + let cols := (s.splitOn ",").map (·.trimAscii.toString) + match cols with + | name :: env :: tier :: rest => + if name == "name" || name.isEmpty then none + else some { + name, env, tier + shardTarget := rest.head?.getD "0" == "1" + primary := (rest.drop 1).head?.getD "0" == "1" + } + | _ => none) |>.toArray + +/-- Manifest selection, mirroring the `!benchmark` config surface: the env's + rows, restricted to the primary subset unless `full`, to `tier` when + given (prove-mode full runs default to `cheap` — the prove-feasible + set), and to shard targets under `shardOnly`. -/ +def selectNames (rows : Array VectorRow) (env : String) (mode : String) + (full : Bool) (tier : String) (shardOnly : Bool) : + Array VectorRow := Id.run do + let effTier := + if tier != "" then tier + else if mode == "prove" && full then "cheap" else "all" + rows.filter fun r => + r.env == env + && (full || r.primary) + && (effTier == "all" || r.tier == effTier) + && (!shardOnly || r.shardTarget) + +/-! ## The registry — single source of truth for the benchmark pipeline + +`Benchmarks/Vectors.csv` holds the per-constant rows; everything else lives +here, in one language with one owner. The workflows never read it directly: +`ix bench ci matrix` serves the job matrices and `ix bench ci parse` the +`!benchmark` cells, both post-build. (`bencher-thresholds-reset.yml` keeps +a static workload list with a sync note — it runs on cheap runners with no +built `ix`.) -/ + +/-- One benchmark env. `name` (e.g. `InitStd`) is the single identifier + everywhere: the `!benchmark` token (matched case-insensitively), the + `ix bench run --env` value, the `.ixe` filename, the cache-key + suffix, and the env-keyed bencher benchmark name. -/ +structure EnvSpec where + name : String + /-- The Lean source `ix compile` builds the env from. -/ + module : String + /-- In the bench-main matrices (and `!benchmark`'s allowed env set). -/ + benched : Bool + +def envSpecs : List EnvSpec := [ + { name := "InitStd", module := "Benchmarks/Compile/CompileInitStd.lean", benched := true }, + { name := "Lean", module := "Benchmarks/Compile/CompileLean.lean", benched := false }, + { name := "Mathlib", module := "Benchmarks/Compile/CompileMathlib.lean", benched := true }, + { name := "FLT", module := "Benchmarks/Compile/CompileFLT.lean", benched := false } +] + +def findEnv (token : String) : Option EnvSpec := + envSpecs.find? fun e => e.name.toLower == token.toLower + +/-- One benchmark backend. Backends with several modes run one bench-main + cell per (mode, testbed) entry — shared measure names (`peak-rss`, + `throughput`) mean that mode's phase, and a `!benchmark` cell of either + mode finds a cached bencher baseline. -/ +structure BackendSpec where + name : String + defaultMode : String + /-- `some reason` ⇒ `parse` skips the backend with the note in the + config summary. -/ + disabled : Option String := none + /-- (mode, bencher testbed). -/ + testbeds : List (String × String) + /-- (mode, compare-table columns), rendered in list order; the head is + the table's row sort key. Column convention: the mode's headline + wall-clock time, throughput, peak-rss, then detail (secondary times, + sizes, deterministic counters). -/ + metrics : List (String × List String) + +def backendSpecs : List BackendSpec := [ + -- prove is aiur's default: the real-workload simulation (it measures + -- Phase 1 — execute-time, fft-cost — inside the same process en route). + -- execute is the fast Phase-1-only signal. + { name := "aiur", defaultMode := "prove", + testbeds := [("prove", "aiur-check-prove-x64-32x"), + ("execute", "aiur-check-execute-x64-32x")], + metrics := [("prove", ["prove-time", "peak-rss", "execute-time", + "verify-time", "proof-size", "fft-cost"]), + ("execute", ["execute-time", "throughput", "peak-rss", + "fft-cost"])] }, + { name := "zisk", defaultMode := "execute", + testbeds := [("execute", "zisk-check-execute-x64-32x")], + metrics := [("execute", ["execute-time", "throughput", "peak-rss", + "cycles"])] }, + { name := "sp1", defaultMode := "execute", + disabled := some "execute run too slow for per-push CI; re-enable here once trimmed", + testbeds := [("execute", "sp1-check-execute-x64-32x")], + metrics := [("execute", ["execute-time", "throughput", "peak-rss", + "cycles"])] }, + { name := "ooc", defaultMode := "execute", + testbeds := [("execute", "ooc-check-x64-32x")], + metrics := [("execute", ["check-time", "throughput", "peak-rss"])] }, + { name := "compile", defaultMode := "execute", + testbeds := [("execute", "ix-compile-x64-32x")], + metrics := [("execute", ["compile-time", "throughput", "peak-rss", + "file-size", "constants"])] } +] + +def findBackend (name : String) : Option BackendSpec := + backendSpecs.find? (·.name == name) + +def BackendSpec.testbedFor (b : BackendSpec) (mode : String) : Option String := + (b.testbeds.find? (·.1 == mode)).map (·.2) + +def BackendSpec.metricsFor (b : BackendSpec) (mode : String) : List String := + ((b.metrics.find? (·.1 == mode)).map (·.2)).getD [] + +/-- Default RAM watchdog ceiling, same rule for all backends: the + machine's total RAM minus 15 GB (the ~123 GiB CI runner lands at + ~108 — above Mathlib `ix compile`'s ~100 GB peak, the largest + legitimate workload). Enforced as cgroup `memory.max` on a systemd + user scope: the kernel OOM-kills the tool at the ceiling (exit 137); + the 15 GB stays outside the cap for the OS, runner agent, and page + cache. `--ceiling-gb` overrides. -/ +def defaultCeilingGb : IO Nat := do + let s ← try IO.FS.readFile "/proc/meminfo" catch _ => pure "" + let kb := (s.splitOn "\n").findSome? fun l => + if l.startsWith "MemTotal:" then + ((l.splitOn " ").filter (· ≠ "") |>.drop 1).head?.bind (·.toNat?) + else none + return match kb with + | some kb => max 8 (kb / (1024 * 1024) - 15) + | none => 16 + +/-- Resolve a tool binary: prefer the in-tree build under `repo` (so a base + checkout measures the base's code), else PATH. -/ +def resolveBin (repo : String) (name : String) : IO String := do + let inTree := s!"{repo}/.lake/build/bin/{name}" + if ← FilePath.pathExists inTree then + return inTree + return name + +/-- Spawn `cmd args` (inheriting stdio) under the RAM watchdog when one is + configured, and wait for its exit code. -/ +def runGuarded (watchdog : Option String) (ceilingGb : Nat) + (cmd : String) (args : Array String) (cwd : Option String := none) : + IO UInt32 := do + let (cmd, args) := match watchdog with + | some wd => (wd, #[toString ceilingGb, cmd] ++ args) + | none => (cmd, args) + IO.eprintln s!"[bench] run: {cmd} {" ".intercalate args.toList}" + let child ← IO.Process.spawn { + cmd, args + cwd := cwd.map FilePath.mk + } + child.wait + +/-- Merge `status: oom` into a constant's row, PRESERVING metrics the tool + flushed before the kill (e.g. bench-typecheck persists Phase-1 fields + before the prove starts). The compare surface renders OOM only for the + metrics that are absent. -/ +def markOom (out : String) (name : String) : IO Unit := do + let rows ← readRows out + let row := (rows.getObjVal? name).toOption.getD (Lean.Json.mkObj []) + writeEntry out name (row.setObjVal! "status" (Lean.Json.str "oom")) + +/-- Sum a texray spans JSONL window (`{"span": s, "seconds": n}` per line) + by span name. Missing or unparseable content contributes nothing. -/ +def readSpans (path : String) : IO (Array (String × Float)) := do + if !(← FilePath.pathExists path) then return #[] + let mut acc : Array (String × Float) := #[] + for line in (← IO.FS.readFile path).splitOn "\n" do + if let .ok j := Lean.Json.parse line then + let span := (j.getObjVal? "span").toOption.bind (·.getStr?.toOption) + let secs := match (j.getObjVal? "seconds").toOption with + | some (.num n) => some n.toFloat + | _ => none + if let (some s, some v) := (span, secs) then + match acc.findIdx? (·.1 == s) with + | some i => acc := acc.set! i (s, acc[i]!.2 + v) + | none => acc := acc.push (s, v) + return acc + +/-- Fold a spawn's texray window (`.spans`) into its constant's row as + flat `phase:` fields — the aiur prover's tracing spans and the + zkVM hosts' `record_manual` entries alike — then drop the window file. + The keys pass straight through `bmf` as independent bencher measures + and come back from `fetch-main` in the same shape. No row (the tool + died before writing one) → nothing to attach the spans to. -/ +def mergeSpans (out : String) (name : String) : IO Unit := do + let spansPath := out ++ ".spans" + let spans ← readSpans spansPath + if !spans.isEmpty then + let rows ← readRows out + if let some row := (rows.getObjVal? name).toOption then + let row := spans.foldl (init := row) fun r (s, v) => + r.setObjVal! s!"phase:{s}" (jsonRound 6 v) + writeEntry out name row + if ← FilePath.pathExists spansPath then + IO.FS.removeFile spansPath + +/-- Run a per-constant tool: ONE PROCESS PER CONSTANT, so a kill costs + exactly that constant with no resume inference, and each spawn's texray + window (`.spans`, truncated by the tool at startup) belongs wholly + to it. Per exit: ≥128 (watchdog TERM/KILL or the kernel OOM killer) → + mark the row `oom` (keeping whatever the tool flushed, spans included) + and continue; `exitRejected` → the rejected row is on disk, continue + (the final gate reddens the cell); any other nonzero exit is + deterministic (usage error, missing input, crash on startup) and would + repeat for every remaining name — abort loudly. + + `doneKey` is the mode's completion metric (e.g. `prove-time`): a kill + that lands AFTER the row carries it hit teardown, not measurement — + typically the prover releasing tens of GB right after the final row + write, while RSS is still at its peak — so the finished row stays ok. -/ +def runPerConstant (out : String) (names : Array String) + (doneKey : String) (spawn : String → IO UInt32) : IO Unit := do + for name in names do + let exit ← spawn name + -- 255 is never a signal death (our kills exit 134/137/143) — it's a + -- failed exec ("could not execute external process") or a tool bailing + -- with -1; labeling it oom would turn a broken spawn into a green cell + -- of fake-OOM rows. + if exit == 255 || (exit != 0 && exit != exitRejected && exit < 128) then + IO.eprintln s!"[bench] tool failed on '{name}' (exit {exit}, not a kill); aborting the remaining names" + return + if exit ≥ 128 then + let rows ← readRows out + let complete := ((rows.getObjVal? name).toOption.bind + fun r => (r.getObjVal? doneKey).toOption).isSome + if complete then + IO.eprintln s!"[bench] '{name}' killed in teardown (exit {exit}); row already complete" + else + IO.eprintln s!"[bench] '{name}' killed (exit {exit}); recording oom" + markOom out name + mergeSpans out name + +/-- Resolve the env's `.ixe`: an explicit `--ixe` path is used as-is (and + must exist — no silent recompile of a mistyped path); otherwise the + env is compiled fresh to `/.ixe`. -/ +def ensureIxe (repo : String) (info : EnvSpec) (explicit : Option String) : + IO String := do + if let some path := explicit then + if ← FilePath.pathExists path then + return path + throw <| IO.userError s!"--ixe {path} not found" + let ixe := s!"{repo}/{info.name}.ixe" + let ix ← resolveBin repo "ix" + let exit ← runGuarded none 0 ix + #["compile", s!"{repo}/{info.module}", "--out", ixe] + if exit != 0 then + throw <| IO.userError s!"ix compile {info.module} failed (exit {exit})" + return ixe + +/-- Cut the closure-shard artifacts for one heavy constant: `ix shard + extract` (standalone closure env) → `ix profile` → `ix shard` + (heartbeat-profiled min-cut manifest, capped by predicted RAM). Skips + work when the artifacts already exist. Returns `(ixe, ixes)` on + success, `none` when any step fails (the caller falls back to the + single-leaf run — the watchdog then records the honest OOM row). -/ +def cutClosureShards (ix : String) (envIxe : String) + (dir : String) (name : String) (maxRamGb : Nat) : + IO (Option (String × String)) := do + let slug := name.map fun c => + if c == '/' || c == ' ' || c == '.' || c == ':' then '_' else c + let subIxe := s!"{dir}/{slug}.ixe" + let manifest := s!"{dir}/{slug}.ixes" + if (← FilePath.pathExists subIxe) && (← FilePath.pathExists manifest) then + return some (subIxe, manifest) + IO.FS.createDirAll dir + let prof := s!"{dir}/{slug}.ixprof" + let steps : List (Array String) := + [ #["shard", "extract", envIxe, "--consts", name, "--out", subIxe] + , #["profile", subIxe, "--out", prof] + , #["shard", prof, "--max-ram", toString maxRamGb, "--out", manifest] ] + for args in steps do + let exit ← runGuarded none 0 ix args + if exit != 0 then + IO.eprintln s!"[bench] shard pipeline failed for '{name}' (exit {exit}); falling back to single leaf" + return none + return some (subIxe, manifest) + +/-- Final cell gate from the rows themselves: exit 1 when any EXPECTED name + lacks a row (an aborted loop, a killed batch, or a dropped whole-env + check must never look green — every selected name owes exactly one + row), exit 3 when any row is `rejected` (red cell, rows on disk say + why), else 0. -/ +def gate (out : String) (expected : Array String) : IO UInt32 := do + let rows ← readRows out + match rows with + | .obj kvs => + let entries := kvs.toArray + let missing := expected.filter fun n => + !(entries.any fun ⟨name, _⟩ => name == n) + for n in missing do + IO.eprintln s!"[bench] error: no row for '{n}'" + let rejected := entries.filter fun ⟨_, row⟩ => + (row.getObjVal? "status").toOption == some (Lean.Json.str "rejected") + for ⟨name, _⟩ in rejected do + IO.eprintln s!"[bench] ❌ '{name}' FAILED TO TYPECHECK — kernel rejected it" + IO.eprintln s!"[bench] {entries.size} row(s), {missing.size} missing, {rejected.size} rejected" + if !missing.isEmpty then return 1 + return if rejected.isEmpty then 0 else exitRejected + | _ => + IO.eprintln "[bench] error: results file is not an object" + return 1 + +/-- The benchmark output root shared with the `Ix.Benchmark` framework: + `BENCH_OUTPUT_DIR`, defaulting to `.lake/benches` (see + `Ix.Benchmark.Common.Config.outputDir`). -/ +def benchOutputDir : IO String := + return (← IO.getEnv "BENCH_OUTPUT_DIR").getD ".lake/benches" + +/-- Save the run as the local baseline (`/.json`), + rotating the previous baseline to `.prev.json` — `ix bench compare` + defaults to the pair, so a bare local rerun compares against the last + run automatically. -/ +def saveBaseline (out : String) (cell : String) : IO Unit := do + let dir ← benchOutputDir + IO.FS.createDirAll dir + let base := s!"{dir}/{cell}.json" + if ← FilePath.pathExists base then + IO.FS.writeFile s!"{dir}/{cell}.prev.json" (← IO.FS.readFile base) + IO.FS.writeFile base (← IO.FS.readFile out) + +def runBenchRunCmd (p : Cli.Parsed) : IO UInt32 := do + let backend := (p.flag? "backend").map (·.as! String) |>.getD "" + let some spec := findBackend backend + | p.printError s!"error: unknown backend '{backend}' (see backendSpecs)" + return exitUsage + let some info := findEnv ((p.flag? "env").map (·.as! String) |>.getD "InitStd") + | p.printError "error: unknown env (see envSpecs)" + return exitUsage + let env := info.name + let mode := (p.flag? "mode").map (·.as! String) |>.getD spec.defaultMode + let repo := (p.flag? "repo").map (·.as! String) |>.getD "." + let out := (p.flag? "out").map (·.as! String) |>.getD "bench.json" + let full := p.hasFlag "full" + let tier := (p.flag? "tier").map (·.as! String) |>.getD "" + let ceilingGb : Nat ← match p.flag? "ceiling-gb" with + | some f => pure (f.as! Nat) + | none => defaultCeilingGb + let watchdogPath := (p.flag? "watchdog").map (·.as! String) + |>.getD s!"{repo}/.github/scripts/watchdog.sh" + -- Absolute: the zkVM hosts spawn with their workspace as cwd, where a + -- repo-relative script path would fail to exec. No watchdog, no run — + -- an unenforced ceiling is not a benchmark run. + let watchdog : Option String ← + if ← FilePath.pathExists watchdogPath then + pure (some (← IO.FS.realPath watchdogPath).toString) + else do + p.printError s!"error: no watchdog at {watchdogPath} (--watchdog overrides)" + return exitUsage + let csv := (p.flag? "csv").map (·.as! String) + |>.getD s!"{repo}/Benchmarks/Vectors.csv" + let rows := parseVectorsCsv (← IO.FS.readFile csv) + -- `--consts` overrides the CSV selection — a one-off local run, or + -- bench-pr's targeted base run over just the constants bencher lacked; + -- tier metadata still comes from the CSV so heavy zisk names keep + -- their sharded pipeline. + let wanted := ((p.flag? "consts").map + (fun f => Ix.Cli.ConstsFile.parseCommaList (f.as! String))).getD #[] + let selected := + if wanted.isEmpty then + selectNames rows env mode full tier (p.hasFlag "shard-only") + else + wanted.map fun n => + (rows.find? (fun r => r.name == n && r.env == env)).getD + { name := n, env, tier := "cheap", shardTarget := false, primary := false } + let names := selected.map (·.name) + IO.eprintln s!"[bench] cell {backend}-{env}-{mode}: {names.size} constant(s)" + + -- Fresh accumulator per run. + if ← FilePath.pathExists out then IO.FS.removeFile out + let namesFile := out ++ ".names.txt" + + match backend with + | "compile" => + -- The compile IS the benchmark: always fresh, row keyed by the env name. + let ix ← resolveBin repo "ix" + let exit ← runGuarded watchdog ceilingGb ix + #["compile", s!"{repo}/{info.module}", "--out", s!"{repo}/{env}.ixe", + "--json", out, "--json-name", info.name] + if exit != 0 then + IO.eprintln s!"[bench] ix compile failed (exit {exit})" + return 1 + | "ooc" => + let ixe ← ensureIxe repo info ((p.flag? "ixe").map (·.as! String)) + let ix ← resolveBin repo "ix" + -- Whole-env row (keyed by the env name) … + let exit ← runGuarded watchdog ceilingGb ix + #["check-rs", ixe, "--anon", "--json", out, "--json-name", info.name] + if exit != 0 && exit != exitRejected then + IO.eprintln s!"[bench] whole-env check failed (exit {exit})" + -- … plus one full-closure row per primary. ONE process for all names + -- (unlike the per-constant backends below): the check-rs rows mode + -- attributes per name internally with the env loaded once — a + -- per-constant process would re-pay the multi-minute Mathlib env parse + -- per name — and out-of-circuit checks don't approach the RAM ceiling. + if !names.isEmpty then + IO.FS.writeFile namesFile ("\n".intercalate names.toList ++ "\n") + let exit ← runGuarded watchdog ceilingGb ix + #["check-rs", ixe, "--anon", "--consts-file", namesFile, "--json", out] + if exit != 0 && exit != exitRejected then + IO.eprintln s!"[bench] per-constant checks failed (exit {exit})" + | "aiur" => + let ixe ← ensureIxe repo info ((p.flag? "ixe").map (·.as! String)) + let bt ← resolveBin repo "bench-typecheck" + let modeArgs := if mode == "execute" then #["--execute-only"] else #[] + let doneKey := if mode == "prove" then "prove-time" else "execute-time" + runPerConstant out names doneKey fun name => + runGuarded watchdog ceilingGb bt + (#["--ixe", ixe, "--consts", name, "--json", out, "--texray"] + ++ modeArgs) + | "zisk" | "sp1" => + if mode != "execute" then + p.printError s!"error: {backend} supports only execute mode" + return exitUsage + let ixe ← ensureIxe repo info ((p.flag? "ixe").map (·.as! String)) + let ixeAbs := (← IO.FS.realPath ixe).toString + let outAbs ← do + IO.FS.writeFile out "" -- realPath needs an existing file + pure (← IO.FS.realPath out).toString + let host := s!"{backend}-host" + let work := s!"{repo}/{backend}" + let build ← runGuarded none 0 "cargo" + #["build", "--quiet", "--release", "--bin", host] (cwd := some work) + if build != 0 then + IO.eprintln s!"[bench] cargo build {host} failed (exit {build})" + return 1 + let bin := (← IO.FS.realPath s!"{work}/target/release/{host}").toString + -- Heavy-tier zisk constants run as their closure-shard partition (a + -- single full-closure leaf would blow the runner's RAM); everything + -- else runs as one host process per constant like the aiur cells. + let heavy := if backend == "zisk" + then (selected.filter (·.tier == "heavy")).map (·.name) else #[] + let light := names.filter (!heavy.contains ·) + runPerConstant outAbs light "execute-time" fun name => + runGuarded watchdog ceilingGb bin + #["--execute", "--ixe", ixeAbs, "--consts", name, + "--json", outAbs, "--texray"] (cwd := some work) + let ix ← resolveBin repo "ix" + runPerConstant outAbs heavy "execute-time" fun name => do + let plan ← cutClosureShards ix ixe s!"{repo}/zkshards-{env}" + name ceilingGb + match plan with + | some (subIxe, manifest) => + runGuarded watchdog ceilingGb bin + #["--execute", "--ixe", (← IO.FS.realPath subIxe).toString, + "--shard-plan", (← IO.FS.realPath manifest).toString, + "--json", outAbs, "--json-name", name, "--texray"] + (cwd := some work) + | none => + runGuarded watchdog ceilingGb bin + #["--execute", "--ixe", ixeAbs, "--consts", name, + "--json", outAbs, "--texray"] (cwd := some work) + | other => + p.printError s!"error: backend '{other}' has no runner" + return exitUsage + + -- Every selected name owes a row; the env-keyed backends owe the env + -- row too. + let expected := match backend with + | "compile" => #[info.name] + | "ooc" => #[info.name] ++ names + | _ => names + let code ← gate out expected + if code == 0 || code == exitRejected then + saveBaseline out s!"{backend}-{env}-{mode}" + return code + +/-- `ix bench shard`: pre-cut the closure-shard artifacts for the env's + heavy-tier constants into `zkshards-/` — `ix shard extract` → + `ix profile` → `ix shard` per name, skipping names whose artifacts + already exist. Not a benchmark cell (no rows, no watchdog): bench-main's + compile job runs it next to the fresh `.ixe` so the artifacts ride the + same cache; the zisk cells cut lazily as a fallback when they're + absent. -/ +def runBenchShardCmd (p : Cli.Parsed) : IO UInt32 := do + let some info := findEnv ((p.flag? "env").map (·.as! String) |>.getD "InitStd") + | p.printError "error: unknown env (see envSpecs)" + return exitUsage + let env := info.name + let repo := (p.flag? "repo").map (·.as! String) |>.getD "." + let ceilingGb : Nat ← match p.flag? "ceiling-gb" with + | some f => pure (f.as! Nat) + | none => defaultCeilingGb + let csv := (p.flag? "csv").map (·.as! String) + |>.getD s!"{repo}/Benchmarks/Vectors.csv" + let rows := parseVectorsCsv (← IO.FS.readFile csv) + let heavy := (rows.filter fun r => + r.env == env && r.primary && r.tier == "heavy").map (·.name) + IO.eprintln s!"[bench] shard {env}: {heavy.size} heavy constant(s)" + let ixe ← ensureIxe repo info ((p.flag? "ixe").map (·.as! String)) + let ix ← resolveBin repo "ix" + for name in heavy do + let _ ← cutClosureShards ix ixe s!"{repo}/zkshards-{env}" name ceilingGb + return 0 + +end Ix.Cli.BenchCmd + +open Ix.Cli.BenchCmd in +def benchRunCmd : Cli.Cmd := `[Cli| + "run" VIA runBenchRunCmd; + "Run one benchmark cell (backend × env × mode), writing benchmark results JSON. Exits 0 on success (rows saved as the local baseline), 3 when the kernel rejected any constant, 1 when no rows were produced." + + FLAGS: + backend : String; "aiur | zisk | sp1 | ooc | compile" + env : String; "Benchmark env from the registry (default: InitStd)" + mode : String; "prove | execute (default: the backend's defaultMode)" + out : String; "Benchmark results JSON output path (default: bench.json)" + repo : String; "Checkout to benchmark: tools resolve from /.lake/build/bin first, then PATH (default: .)" + csv : String; "Vectors path (default: /Benchmarks/Vectors.csv)" + full; "Run the env's full curated set instead of the primary subset" + consts : String; "Run exactly these comma-separated names instead of the Vectors.csv selection (same grammar as the tools' --consts)" + tier : String; "cheap | heavy | all — tier filter (default: all; prove-mode --full defaults to cheap)" + "shard-only"; "Restrict to shard_target rows" + ixe : String; "Path to an existing .ixe env to use (default: compile fresh; ignored by the compile backend)" + "ceiling-gb" : Nat; "RAM watchdog ceiling in GB (default: machine RAM minus 15 GB)" + watchdog : String; "Watchdog wrapper path (default: /.github/scripts/watchdog.sh; missing = run unguarded)" +] + +open Ix.Cli.BenchCmd in +def benchShardCmd : Cli.Cmd := `[Cli| + "shard" VIA runBenchShardCmd; + "Pre-cut closure-shard artifacts (ix shard extract → profile → shard) for the env's heavy-tier constants into zkshards-/; skips names already cut. The zisk cells cut lazily when these are absent — this front-loads the work so the artifacts can be cached once per commit." + + FLAGS: + env : String; "Benchmark env from the registry (default: InitStd)" + repo : String; "Checkout to shard: tools resolve from /.lake/build/bin first, then PATH (default: .)" + csv : String; "Vectors path (default: /Benchmarks/Vectors.csv)" + ixe : String; "Path to an existing .ixe env to use (default: compile fresh)" + "ceiling-gb" : Nat; "Predicted-RAM cap per shard, passed to `ix shard --max-ram` (default: machine RAM minus 15 GB)" +] + diff --git a/Ix/Cli/BenchReport.lean b/Ix/Cli/BenchReport.lean new file mode 100644 index 000000000..ea0df3019 --- /dev/null +++ b/Ix/Cli/BenchReport.lean @@ -0,0 +1,776 @@ +/- + `ix bench` reporting subcommands — everything between the results rows + JSON (`Ix.Benchmark.Results`) and a human or bencher.dev: + + compare two rows files → a Markdown main-vs-PR table. With no + explicit inputs, compares the cell's local baseline against + its previous run (`.lake/benches/.json` vs `.prev.json`), so + a bare local rerun reports its own delta. + report per-cell table files → one Markdown report (the PR comment) + bmf rows JSON → Bencher Metric Format (status ≠ ok rows dropped) + fetch-main base SHA + cell → rows JSON pulled from bencher.dev (curl) + ci … CI adapters: parse (!benchmark comment → job matrix) and + matrix (registry → workflow job matrices) + + CI and local runs share this surface: the PR comment table and the local + terminal table are the same renderer. +-/ +module +public import Cli +public import Lean.Data.Json +public import Ix.Benchmark.Results +public import Ix.Cli.BenchCmd + +public section + +open Lean (Json) +open Ix.Benchmark.Results + +namespace Ix.Cli.BenchReport + +/-! ## Metric formatting -/ + +/-- Per-metric formatting kind. Metric names are the results-JSON keys the + tools emit (see the registry in Ix.Cli.BenchCmd). Unknown metrics fall through to a + generic decimal rendering. -/ +def metricKind (metric : String) : String := + if ["peak-rss", "file-size", "proof-size"].contains metric + then "bytes" + else if metric.startsWith "phase:" then "seconds" + else if ["execute-time", "prove-time", "verify-time", "check-time", + "compile-time"].contains metric then "seconds" + else if ["fft-cost", "cycles", "steps", "max-shard-cycles", + "throughput"].contains metric then "count" + else if ["constants", "shards"].contains metric then "int" + else "auto" + +/-- Display label for a metric column. Keys stay the bencher measure slugs + (renaming one would orphan its threshold/history); only the table + rendering differs. `file-size` is the serialized `.ixe` env — bencher + plots it as "Environment Size"; `peak-rss` reads better as plain RAM. -/ +def metricLabel (metric : String) : String := + if metric == "file-size" then "env-size" + else if metric == "peak-rss" then "peak-ram" + else metric + +/-- Group a digit string by thousands: `"105492"` → `"105,492"`. -/ +private def commafy (s : String) : String := Id.run do + let (sign, digits) := + if s.startsWith "-" then ("-", (s.drop 1).toString) else ("", s) + let n := digits.length + let mut out := sign + let mut i := 0 + for c in digits.toList do + if i != 0 && (n - i) % 3 == 0 then + out := out.push ',' + out := out.push c + i := i + 1 + return out + +private def fmtF (f : Float) (decimals : Nat) : String := + let scale := (10.0 : Float) ^ decimals.toFloat + let n := (f * scale).round / scale + let s := toString n + -- `toString 3.0 = "3.000000"`; trim to the requested precision. + match s.splitOn "." with + | [i, frac] => if decimals == 0 then i else i ++ "." ++ frac.take decimals + | _ => s + +def humanBytes (v : Float) : String := Id.run do + let mut v := v + for unit in ["B", "KiB", "MiB", "GiB"] do + if v.abs < 1024.0 then + return if unit == "B" then s!"{fmtF v 0} B" else s!"{fmtF v 2} {unit}" + v := v / 1024.0 + return s!"{fmtF v 2} TiB" + +def humanSeconds (v : Float) : String := + if v.abs < 0.001 then s!"{fmtF (v * 1e6) 1} µs" + else if v.abs < 1.0 then s!"{fmtF (v * 1e3) 1} ms" + else if v.abs < 60.0 then s!"{fmtF v 3} s" + else s!"{fmtF (v / 60.0).floor 0}m {fmtF (v - 60.0 * (v / 60.0).floor) 1}s" + +def humanCount (v : Float) : String := Id.run do + if v.abs < 1000.0 then + return if v == v.round then fmtF v 0 else fmtF v 3 + let mut v := v + for unit in ["K", "M", "B"] do + v := v / 1000.0 + if v.abs < 1000.0 then return s!"{fmtF v 2}{unit}" + return s!"{fmtF (v / 1000.0) 2}T" + +def human (v : Option Float) (metric : String) : String := + match v with + | none => "n/a" + | some v => + match metricKind metric with + | "bytes" => humanBytes v + | "seconds" => humanSeconds v + | "count" => humanCount v + | "int" => commafy (fmtF v 0) + | _ => if v == v.round then commafy (fmtF v 0) else fmtF v 3 + +/-- Metrics where a LARGER value is the improvement; everything else is + lower-is-better (times, RAM, cycles, sizes). `throughput` is reused + across backends with different units (consts/s on most; cycles/s on the + zkVM hosts) — testbeds keep the series separate, and within any one + table the unit is uniform. -/ +def higherIsBetter (metric : String) : Bool := metric == "throughput" + +/-! ## Row access -/ + +def rowNum (rows : Json) (name metric : String) : Option Float := do + let row ← (rows.getObjVal? name).toOption + match (row.getObjVal? metric).toOption with + | some (.num n) => some n.toFloat + | _ => none + +def rowStatus (rows : Json) (name : String) : String := + ((rows.getObjVal? name).toOption.bind + fun r => (r.getObjVal? "status").toOption.bind (·.getStr?.toOption)) + |>.getD "ok" + +def rowNames (rows : Json) : Array String := + match rows with + | .obj kvs => kvs.toArray.map (·.1) + | _ => #[] + +/-- The `phase:` field names of one row (flat keys — same shape on + the PR side, in local baselines, and coming back from bencher). -/ +def rowPhaseKeys (rows : Json) (name : String) : Array String := + match (rows.getObjVal? name).toOption with + | some (.obj kvs) => kvs.toArray.map (·.1) |>.filter (·.startsWith "phase:") + | _ => #[] + +/-! ## compare -/ + +/-- Signed regression magnitude: positive ⇒ the PR side got worse. -/ +def badness (deltaPct : Float) (metric : String) : Float := + if higherIsBetter metric then -deltaPct else deltaPct + +/-- `(factor ≥ 1, direction word)` for the ratio annotation; wording follows + the metric kind ("1.15× slower" is meaningless for a byte metric). -/ +def ratio (mainV prV : Float) (metric : String) : Option (Float × String) := + if mainV <= 0.0 || prV <= 0.0 then none else + let grew := prV >= mainV + let factor := if grew then prV / mainV else mainV / prV + if higherIsBetter metric then + some (factor, if grew then "faster" else "slower") + else + let words := match metricKind metric with + | "seconds" => ("slower", "faster") + | "bytes" => ("larger", "smaller") + | _ => ("more", "fewer") + some (factor, if grew then words.1 else words.2) + +structure CompareArgs where + mainRows : Json + prRows : Json + metrics : Array String + threshold : Float + title : String + +def renderCompare (a : CompareArgs) : String := Id.run do + let names := (rowNames a.mainRows ++ rowNames a.prRows).foldl + (fun acc n => if acc.contains n then acc else acc.push n) #[] + if names.isEmpty then + return a.title ++ "\n\n_No results were produced (every constant failed, \ + timed out, or was dropped). See the workflow logs._" + -- Sort by the primary metric, largest first (n/a rows last). + let primary := a.metrics[0]?.getD "" + let key := fun n => (rowNum a.prRows n primary).orElse + fun _ => rowNum a.mainRows n primary + let names := names.qsort fun x y => + match key x, key y with + | some vx, some vy => vx > vy + | some _, none => true + | none, some _ => false + | none, none => x < y + + let mut head := #["constant"] + for m in a.metrics do + head := head ++ #[s!"{metricLabel m} (main)", s!"{metricLabel m} (PR)", "Δ%"] + let mut lines := #[ + "| " ++ " | ".intercalate head.toList ++ " |", + "|" ++ "|".intercalate (head.toList.map fun _ => "---") ++ "|"] + + let mut failures : Array (String × String) := #[] + let mut regressed := 0 + let mut improved := 0 + for n in names do + let mainStatus := rowStatus a.mainRows n + let prStatus := rowStatus a.prRows n + if mainStatus == "rejected" then failures := failures.push (n, "main") + if prStatus == "rejected" then failures := failures.push (n, "PR") + let mut rowRegressed := false + let mut rowImproved := false + let mut cells := #[s!"`{n}`"] + for m in a.metrics do + let mv := rowNum a.mainRows n m + let pv := rowNum a.prRows n m + -- An OOM row may still carry real partial measurements; render those, + -- and OOM only for the metrics the kill prevented. A REJECTED row is + -- spelled out — the constant was rejected, not benchmarked. + let renderSide := fun (status : String) (v : Option Float) => + if status == "rejected" then "❌ failed typecheck" + else if status == "oom" && v.isNone then "OOM" + else human v m + let mut delta := "n/a" + if let (some mvv, some pvv) := (mv, pv) then + if mvv != 0.0 then + let dp := (pvv - mvv) / mvv * 100.0 + delta := (if dp >= 0.0 then "+" else "") ++ fmtF dp 1 ++ "%" + -- Ratio only when it adds signal beyond the percentage. + if let some (f, word) := ratio mvv pvv m then + if f >= 1.05 then delta := delta ++ s!" ({fmtF f 2}× {word})" + let bad := badness dp m + if bad > a.threshold then + delta := delta ++ " ⚠️"; rowRegressed := true + else if bad < -a.threshold then + delta := delta ++ " 🟢"; rowImproved := true + cells := cells ++ #[renderSide mainStatus mv, renderSide prStatus pv, delta] + if rowRegressed then regressed := regressed + 1 + else if rowImproved then improved := improved + 1 + lines := lines.push ("| " ++ " | ".intercalate cells.toList ++ " |") + + let mut out := #[a.title, ""] ++ lines ++ #[""] + -- Typecheck failures first and loud — a constant the kernel REJECTS is a + -- correctness signal, not a benchmark blip. + for (n, side) in failures do + out := out.push s!"❌ **`{n}` FAILED TO TYPECHECK on the {side} side** — \ + the kernel rejected it; see the logs." + if !failures.isEmpty then out := out.push "" + out := out.push s!"_{names.size} constants · {regressed} regressed · \ + {improved} improved (|Δ| > {fmtF a.threshold 1}% on any metric)._" + -- One side empty while the other measured is almost always a broken side + -- (e.g. the base-run fallback hit a base binary that predates a flag), + -- not a real all-regressed/all-new comparison — flag it briefly instead + -- of leaving a silent all-n/a column; the workflow logs carry the why. + if (rowNames a.mainRows).isEmpty then + out := out.push "" |>.push + "_⚠️ no main-side results (base run failed — see the workflow logs)._" + else if (rowNames a.prRows).isEmpty then + out := out.push "" |>.push + "_⚠️ no PR-side results (see the workflow logs)._" + -- Per-phase drill-down: the main table above carries every constant's + -- high-level row; below it, each constant with `phase:` fields + -- (aiur witness/commit/quotient breakdowns, zkVM coarse phases) gets its + -- own collapsed mini-table (`phase | main | PR | Δ%`), opened as + -- desired. + let mut detail : Array String := #[] + for n in names do + let keys := (rowPhaseKeys a.mainRows n ++ rowPhaseKeys a.prRows n).foldl + (fun acc k => if acc.contains k then acc else acc.push k) #[] + if keys.isEmpty then continue + detail := detail.push "" |>.push "

" + |>.push s!"{n}" |>.push "" + |>.push "| phase | main | PR | Δ% |" |>.push "|---|---|---|---|" + for k in keys.qsort (· < ·) do + let mv := rowNum a.mainRows n k + let pv := rowNum a.prRows n k + let delta := match mv, pv with + | some m, some p => + if m != 0.0 then + let dp := (p - m) / m * 100.0 + (if dp >= 0.0 then "+" else "") ++ fmtF dp 1 ++ "%" + else "n/a" + | _, _ => "n/a" + detail := detail.push + s!"| `{k.drop 6}` | {human mv k} | {human pv k} | {delta} |" + detail := detail.push "" |>.push "
" + if !detail.isEmpty then + out := (out.push "" |>.push "**per-phase drill-down**") ++ detail + return "\n".intercalate out.toList + +def runCompareCmd (p : Cli.Parsed) : IO UInt32 := do + let backend := (p.flag? "backend").map (·.as! String) |>.getD "" + let env := (p.flag? "env").map (·.as! String) |>.getD "InitStd" + let mode := (p.flag? "mode").map (·.as! String) + |>.getD (((Ix.Cli.BenchCmd.findBackend backend).map (·.defaultMode)).getD "execute") + let cell := s!"{backend}-{env}-{mode}" + -- Explicit paths win; otherwise the local baseline pair, so a bare + -- `ix bench compare --backend …` after two runs reports the local delta. + let benchDir ← Ix.Cli.BenchCmd.benchOutputDir + let mainPath := (p.flag? "main").map (·.as! String) + |>.getD s!"{benchDir}/{cell}.prev.json" + let prPath := (p.flag? "pr").map (·.as! String) + |>.getD s!"{benchDir}/{cell}.json" + let metrics := + let flagged := (p.flag? "metric").map (·.as! (Array String)) |>.getD #[] + if flagged.isEmpty then + (((Ix.Cli.BenchCmd.findBackend backend).map + (·.metricsFor mode)).getD []).toArray + else flagged + if metrics.isEmpty then + p.printError s!"error: no metrics for {backend}/{mode}; pass --metric or fix backendSpecs" + return exitUsage + let threshold := (p.flag? "threshold").map (fun f => (f.as! Nat).toFloat) + |>.getD 3.0 + let mainSrc := (p.flag? "main-source").map (·.as! String) |>.getD mainPath + -- Name the mode only where a mode choice exists (aiur); a single-mode + -- backend's `· execute` is registry plumbing, not information. + let cellName := + if ((Ix.Cli.BenchCmd.findBackend backend).map (·.testbeds.length)).getD 0 > 1 + then s!"`{backend}` · `{env}` · `{mode}`" + else s!"`{backend}` · `{env}`" + let title := (p.flag? "title").map (·.as! String) + |>.getD s!"### {cellName} — main from: {mainSrc}" + let table := renderCompare { + mainRows := ← readRows mainPath + prRows := ← readRows prPath + metrics, threshold, title + } + match p.flag? "out" with + | some f => IO.FS.writeFile (f.as! String) (table ++ "\n") + | none => IO.println table + return 0 + +/-! ## report -/ + +/-- Assemble per-cell compare tables (`/table-*.md`) into one Markdown + report — run several cells locally, `--out table-.md` each, and + read them as a single document. The link flags are optional garnish: + with `--head`/`--repo-url`/`--run-id` (the PR workflow passes them) the + header links the commit and a logs footer is appended. -/ +def runReportCmd (p : Cli.Parsed) : IO UInt32 := do + let tablesDir := (p.flag? "tables").map (·.as! String) |>.getD "tables" + let head := (p.flag? "head").map (·.as! String) |>.getD "" + let repoUrl := (p.flag? "repo-url").map (·.as! String) |>.getD "" + let runId := (p.flag? "run-id").map (·.as! String) |>.getD "" + let summary := (p.flag? "summary").map (·.as! String) |>.getD "" + let commit := if head.isEmpty then "" + else if repoUrl.isEmpty then s!" — main vs `{head.take 7}`" + else s!" — main vs [`{head.take 7}`]({repoUrl}/commit/{head})" + let mut parts := #[s!"## `!benchmark`{commit}", "", summary, ""] + let entries := (← System.FilePath.readDir tablesDir).map (·.path.toString) + let tables := (entries.filter fun p => + (p.splitOn "/").getLast!.startsWith "table-" && p.endsWith ".md").qsort (· < ·) + if tables.isEmpty then + parts := parts ++ #["_No result tables were produced — see the run logs._", ""] + else + for t in tables do + parts := parts ++ #[(← IO.FS.readFile t).trimAscii.toString, ""] + if !repoUrl.isEmpty && !runId.isEmpty then + parts := parts.push s!"[Workflow logs]({repoUrl}/actions/runs/{runId})" + let body := "\n".intercalate parts.toList ++ "\n" + match p.flag? "out" with + | some f => IO.FS.writeFile (f.as! String) body; IO.println body + | none => IO.println body + return 0 + +/-! ## bmf -/ + +/-- Measure slug for a per-shard breakdown key: the shard rows share the + parent row's measure vocabulary (`shard-cycles` → `cycles`, + `shard-time` → `execute-time`, `shard-peak-rss` → `peak-rss`). -/ +def shardMeasure (k : String) : String := + if k == "shard-time" then "execute-time" + else if k.startsWith "shard-" then (k.drop 6).toString + else k + +/-- Rows JSON → Bencher Metric Format. Rows with `status ≠ ok` are dropped + whole — a rejected or OOM'd constant must never become a bencher data + point. Numeric fields (`phase:` included) pass through as + measures. Nested per-shard objects (`shard-cycles: {"0": …}`) become + per-shard BENCHMARKS (`/shard-0`) sharing the parent's measure + slugs — multiplicity belongs in bencher's benchmark dimension, not as + one measure per shard index. -/ +def toBmf (rows : Json) : Json := Id.run do + let mut out : Array (String × Json) := #[] + for name in rowNames rows do + if rowStatus rows name != "ok" then continue + let some row := (rows.getObjVal? name).toOption | continue + let mut measures : Array (String × Json) := #[] + let mut shardRows : Array (String × Array (String × Json)) := #[] + match row with + | .obj kvs => + for (k, v) in kvs.toArray do + if k == "status" then continue + match v with + | .num _ => measures := measures.push (k, Json.mkObj [("value", v)]) + | .obj sub => + for (subK, subV) in sub.toArray do + if let .num _ := subV then + let bench := s!"{name}/shard-{subK}" + let m := (shardMeasure k, Json.mkObj [("value", subV)]) + shardRows := match shardRows.findIdx? (·.1 == bench) with + | some i => shardRows.modify i fun (b, ms) => (b, ms.push m) + | none => shardRows.push (bench, #[m]) + | _ => pure () + | _ => pure () + if !measures.isEmpty then + out := out.push (name, Json.mkObj measures.toList) + for (bench, ms) in shardRows do + out := out.push (bench, Json.mkObj ms.toList) + return Json.mkObj out.toList + +def runBmfCmd (p : Cli.Parsed) : IO UInt32 := do + let inPath := (p.flag? "in").map (·.as! String) |>.getD "bench.json" + let outPath := (p.flag? "out").map (·.as! String) |>.getD "bmf.json" + let bmf := toBmf (← readRows inPath) + IO.FS.writeFile outPath bmf.pretty + IO.println s!"bmf: {(rowNames bmf).size} benchmark(s) → {outPath}" + -- Zero survivors (missing input, or every row OOM'd/rejected) is not an + -- uploadable result — exit nonzero so the caller skips the upload + -- instead of sending bencher an empty report it will reject. + if (rowNames bmf).isEmpty then + IO.eprintln "bmf: no ok rows — nothing to upload" + return 1 + return 0 + +/-! ## fetch-main -/ + +def curlJson (url : String) : IO (Except String Json) := do + let r ← IO.Process.output { + cmd := "curl" + args := #["-sf", "--retry", "3", "--retry-delay", "2", + "--max-time", "30", url] + } + if r.exitCode != 0 then + return .error s!"curl exit {r.exitCode}: {r.stderr.take 300}" + return Lean.Json.parse r.stdout + +/-- Pull the base SHA's rows JSON from bencher.dev. Exit codes are + load-bearing for the workflow: 3 = transient (no report at that hash + yet, or the API failed) — the caller falls back to running main + locally; 2 = config error (unknown backend/mode) — the caller fails + the cell loudly instead of paying the fallback forever. A PARTIAL miss + still exits 0: `--missing-out` lists the uncovered names so the caller + measures just those against the base checkout and merges. -/ +def runFetchMainCmd (p : Cli.Parsed) : IO UInt32 := do + let sha := (p.flag? "sha").map (·.as! String) |>.getD "" + let backend := (p.flag? "backend").map (·.as! String) |>.getD "" + let mode := (p.flag? "mode").map (·.as! String) |>.getD "" + let out := (p.flag? "out").map (·.as! String) |>.getD "main.json" + let some testbed := (Ix.Cli.BenchCmd.findBackend backend).bind + (·.testbedFor mode) + | IO.println s!"fetch-main: no testbed for {backend}/{mode}" + return exitUsage + let wanted : Option (Array String) ← do + let names ← Ix.Cli.ConstsFile.gather p "consts" "names" + if (p.flag? "consts").isNone && (p.flag? "names").isNone then pure none + else + -- The env-keyed row (ooc whole-env, compile) isn't a Vectors.csv + -- constant; admit it past the names filter explicitly. + match (p.flag? "env").map (·.as! String) with + | some env => pure (some (names.push env)) + | none => pure (some names) + + -- Page newest-first until the SHA's reports are found; aggregate across + -- reports (matrix envs upload separately to one testbed), NEWEST report + -- winning on a name collision (a re-run at the same commit supersedes). + let reportHash := fun (r : Json) => + (r.getObjVal? "branch").toOption.bind fun b => + (b.getObjVal? "head").toOption.bind fun h => + (h.getObjVal? "version").toOption.bind fun v => + (v.getObjVal? "hash").toOption.bind (·.getStr?.toOption) + let mut atSha : Array Json := #[] + let mut page := 1 + while page <= 8 do + let url := s!"https://api.bencher.dev/v0/projects/ix/reports?branch=main&testbed={testbed}&per_page=255&page={page}" + match ← curlJson url with + | .error e => + IO.println s!"fetch-main: bencher API error: {e}" + return exitRejected + | .ok j => + let reports := j.getArr?.toOption.getD #[] + let hits := reports.filter fun r => reportHash r == some sha + if !atSha.isEmpty && hits.isEmpty then break + atSha := atSha ++ hits + if reports.size < 255 then break + page := page + 1 + if atSha.isEmpty then + IO.println s!"fetch-main: no reports for {backend}/{mode} @ {sha.take 8}" + return exitRejected + + let mut rows : Array (String × Json) := #[] + let mut seen : Array String := #[] + for r in atSha do + let iterations := (r.getObjVal? "results").toOption.bind (·.getArr?.toOption) + |>.getD #[] + for iteration in iterations do + for bench in iteration.getArr?.toOption.getD #[] do + let some name := (bench.getObjVal? "benchmark").toOption.bind fun b => + (b.getObjVal? "name").toOption.bind (·.getStr?.toOption) | continue + if seen.contains name then continue + if let some w := wanted then + if !w.contains name then continue + let mut metrics : Array (String × Json) := #[("status", Json.str "ok")] + let ms := (bench.getObjVal? "measures").toOption.bind (·.getArr?.toOption) + |>.getD #[] + for m in ms do + let mName := (m.getObjVal? "measure").toOption.bind fun x => + (x.getObjVal? "name").toOption.bind (·.getStr?.toOption) + let mVal := (m.getObjVal? "metric").toOption.bind fun x => + (x.getObjVal? "value").toOption + if let (some mn, some mv) := (mName, mVal) then + metrics := metrics.push (mn, mv) + if metrics.size > 1 then + seen := seen.push name + rows := rows.push (name, Json.mkObj metrics.toList) + if rows.isEmpty then + IO.println "fetch-main: reports found but no matching benchmarks in --names" + return exitRejected + if let some missingOut := (p.flag? "missing-out").map (·.as! String) then + -- Computed against the listed names verbatim: the env-keyed row is an + -- admit-filter, not a per-constant expectation. + let nameSet ← Ix.Cli.ConstsFile.gather p "consts" "names" + let missing := nameSet.filter fun n => !seen.contains n + IO.FS.writeFile missingOut + ("\n".intercalate missing.toList ++ (if missing.isEmpty then "" else "\n")) + if !missing.isEmpty then + IO.println s!"fetch-main: {missing.size} name(s) not on bencher @ \ + {sha.take 8} (base run will measure): {", ".intercalate missing.toList}" + IO.FS.writeFile out (Json.mkObj rows.toList).compress + IO.println s!"fetch-main: {rows.size} constant(s) from bencher for {backend}/{mode}" + return 0 + +/-! ## matrix -/ + +/-- Emit GitHub Actions matrix JSON from the registry, so workflow matrices + are generated instead of hand-copied. `--kind envs` lists the benched + env names; `--kind cells` fans enabled backends × benched envs. -/ +def runMatrixCmd (p : Cli.Parsed) : IO UInt32 := do + let kind := (p.flag? "kind").map (·.as! String) |>.getD "envs" + let benched := (Ix.Cli.BenchCmd.envSpecs.filter (·.benched)).map (·.name) + match kind with + | "envs" => + IO.println (Json.arr (benched.map Json.str).toArray).compress + | "cells" => + let mut cells : Array Json := #[] + for b in Ix.Cli.BenchCmd.backendSpecs do + if b.disabled.isSome then continue + for env in benched do + cells := cells.push <| Json.mkObj + [("backend", Json.str b.name), ("env", Json.str env), + ("mode", Json.str b.defaultMode)] + IO.println (Json.arr cells).compress + | other => + p.printError s!"error: unknown --kind '{other}' (envs | cells)" + return exitUsage + return 0 + +/-! ## parse -/ + +/-- The runner every CI benchmark cell measures on — a `runs-on` field for + the workflows' job matrices, meaningless locally. -/ +def ciRunner : String := "warp-ubuntu-latest-x64-32x" + +/-- Parse a `!benchmark` command into the cells it schedules — locally a + dry-run preview (`ix bench ci parse --comment "!benchmark aiur"` prints + the summary and cell list), in CI the matrix generator. The text comes + from `--comment`, else the `COMMENT_BODY` env var (how the PR workflow + passes the comment without inline shell interpolation). When + `$GITHUB_OUTPUT` is set, the machine outputs (matrix, flags, summary, + passthrough env) are appended there in Actions format. + + Grammar (unknown tokens fall off the allowlist): + + !benchmark ([aiur] [zisk] [sp1] [ooc] [compile] | all) [execute] + BENCH_ENVS=InitStd,Mathlib (case-insensitive; default InitStd) + BENCH_FULL=1 (full curated set, not just primary) + BENCH_SHARD=1 (only the multi-shard target constants) + RUST_LOG=… / WITHOUT_VK_VERIFICATION=… / RUSTFLAGS=… (passthrough) + + The bare `execute` token flips a backend with an execute metrics entry + to execute-only — a real switch only for aiur, whose two modes store on + separate testbeds, so either kind of cell finds a cached baseline. -/ +def runParseCmd (p : Cli.Parsed) : IO UInt32 := do + let body ← match p.flag? "comment" with + | some f => pure (f.as! String) + | none => pure ((← IO.getEnv "COMMENT_BODY").getD "") + let lines := (body.splitOn "\n").map (·.replace "\r" "") + let isCmd := fun (l : String) => (l.splitOn "!benchmark").length > 1 + let cmd := (lines.find? isCmd).getD "" + let toks := if isCmd cmd + then (((cmd.splitOn "!benchmark")[1]!).splitOn " ").filter (· ≠ "") + else [] + + let mut backends : Array Ix.Cli.BenchCmd.BackendSpec := #[] + let mut skipped : Array Ix.Cli.BenchCmd.BackendSpec := #[] + let mut executeFlag := false + for t in toks.map (·.toLower) do + let requested := if t == "all" + then Ix.Cli.BenchCmd.backendSpecs + else (Ix.Cli.BenchCmd.findBackend t).toList + for b in requested do + if b.disabled.isSome then + if skipped.all (·.name != b.name) then skipped := skipped.push b + else if backends.all (·.name != b.name) then + backends := backends.push b + if t == "execute" then executeFlag := true + if backends.isEmpty then + backends := (Ix.Cli.BenchCmd.findBackend "aiur").toList.toArray + + -- KEY=VALUE config lines below the command line. + let mut envs : Array String := #[] + let mut shard := "0" + let mut full := "0" + let mut passthrough : Array String := #[] + let mut seenCmd := false + for ln in lines do + if !seenCmd then + seenCmd := isCmd ln + continue + let s := ln.trimAscii.toString + match s.splitOn "=" with + | key :: rest => + if rest.isEmpty then continue + let val := "=".intercalate rest |>.trimAscii.toString + match key.trimAscii.toString with + | "BENCH_ENVS" => + for tok in val.splitOn "," do + if let some e := Ix.Cli.BenchCmd.findEnv tok.trimAscii.toString then + if e.benched && !envs.contains e.name then + envs := envs.push e.name + | "BENCH_SHARD" => if val == "1" then shard := "1" + | "BENCH_FULL" => if val == "1" then full := "1" + | k => + if ["RUST_LOG", "WITHOUT_VK_VERIFICATION", "RUSTFLAGS"].contains k then + passthrough := passthrough.push s!"{k}={val}" + | [] => continue + if envs.isEmpty then envs := #["InitStd"] + + let modeFor := fun (b : Ix.Cli.BenchCmd.BackendSpec) => + if executeFlag && !(b.metricsFor "execute").isEmpty then "execute" + else b.defaultMode + let mut cells : Array Json := #[] + for b in backends do + for e in envs do + cells := cells.push <| Json.mkObj + [("backend", Json.str b.name), ("env", Json.str e), + ("mode", Json.str (modeFor b)), + ("runner", Json.str ciRunner), + ("label", Json.str s!"{b.name}-{e}-{modeFor b}")] + + -- Annotate the mode only where a mode CHOICE exists (aiur's + -- execute/prove); `ooc=execute` for a single-mode backend is noise. + let modes := " ".intercalate + (backends.map (fun b => + if b.testbeds.length > 1 then s!"{b.name}={modeFor b}" else b.name)).toList + let mut summary := s!"backends: `{modes}` · envs: `{",".intercalate envs.toList}` · \ + set: `{if full == "1" then "full" else "primary"}` · shard: `{shard}`" + for b in skipped do + summary := summary ++ + s!" · skipped `{b.name}` ({b.disabled.getD "disabled in the registry"})" + if !passthrough.isEmpty then + summary := summary ++ " · env: `" ++ " ".intercalate passthrough.toList ++ "`" + + -- `envs` drives the workflow's compile stage: every requested env is + -- compiled exactly once — the `.ixe` artifact the prover cells restore, + -- AND the measured row the compile cell reuses as its PR side instead + -- of compiling a second time. + if let some outPath := ← IO.getEnv "GITHUB_OUTPUT" then + let h ← IO.FS.Handle.mk outPath IO.FS.Mode.append + h.putStr <| s!"matrix={(Json.arr cells).compress}\n" + ++ s!"envs={(Json.arr (envs.map Json.str)).compress}\n" + ++ s!"shard={shard}\nfull={full}\n" + ++ s!"config-summary={summary}\n" + ++ "passthrough-env</{.prev,}.json), so a bare rerun compares against the previous local run." + + FLAGS: + backend : String; "Cell backend (metrics come from the registry)" + env : String; "Cell env (default: InitStd)" + mode : String; "Cell mode (default: the backend's default_mode)" + main : String; "Main-side rows JSON (default: the cell baseline .prev.json)" + pr : String; "PR-side rows JSON (default: the cell baseline .json)" + metric : Array String; "Metric column(s), overriding the registry" + threshold : Nat; "Flag |Δ| above this percentage (default: 3)" + title : String; "Table title (default: derived from the cell)" + "main-source" : String; "Where the main side came from, for the title" + out : String; "Write the table here instead of stdout" +] + +open Ix.Cli.BenchReport in +def benchReportCmd : Cli.Cmd := `[Cli| + "report" VIA runReportCmd; + "Assemble per-cell compare tables (/table-*.md) into one Markdown report. The link flags are optional: the PR workflow passes them to make the report a linkable comment body." + + FLAGS: + tables : String; "Directory of table-*.md files (default: tables)" + summary : String; "Summary line for the header" + head : String; "Commit SHA to title the report with" + "repo-url" : String; "Repository URL, enabling commit/logs links" + "run-id" : String; "Workflow run id for the logs link" + out : String; "Also write the report here (always printed)" +] + +open Ix.Cli.BenchReport in +def benchBmfCmd : Cli.Cmd := `[Cli| + "bmf" VIA runBmfCmd; + "Convert benchmark results JSON to Bencher Metric Format (rows with status ≠ ok are dropped whole)" + + FLAGS: + "in" : String; "Benchmark results JSON (default: bench.json)" + out : String; "BMF output path (default: bmf.json)" +] + +open Ix.Cli.BenchReport in +def benchFetchMainCmd : Cli.Cmd := `[Cli| + "fetch-main" VIA runFetchMainCmd; + "Pull a base SHA's rows from bencher.dev. Exits 3 when bencher has no usable data yet (caller falls back to a local base run), 2 on registry drift." + + FLAGS: + sha : String; "Base commit SHA to fetch reports for" + backend : String; "Cell backend (testbed from the registry)" + env : String; "Cell env — admits the env-keyed row past --names" + mode : String; "Cell mode" + consts : String; "Only fetch these comma-separated benchmark names" + names : String; "Additionally read names from a file (one per line); unions with --consts" + "missing-out" : String; "Write the --names entries bencher lacked (the caller measures just these on the base checkout)" + out : String; "Rows JSON output (default: main.json)" +] + +open Ix.Cli.BenchReport in +def benchCiMatrixCmd : Cli.Cmd := `[Cli| + "matrix" VIA runMatrixCmd; + "Emit GitHub Actions matrix JSON from the registry (--kind envs | cells)" + + FLAGS: + kind : String; "envs = benched env names; cells = enabled backends × benched envs" +] + +open Ix.Cli.BenchReport in +def benchCiParseCmd : Cli.Cmd := `[Cli| + "parse" VIA runParseCmd; + "Parse a !benchmark command into the cells it schedules and write the job matrix to $GITHUB_OUTPUT (when set). Unknown tokens fall off the allowlist; --comment doubles as a local pre-flight of a comment before posting it." + + FLAGS: + comment : String; "The command text (default: the COMMENT_BODY env var)" +] + +def benchCiCmd : Cli.Cmd := `[Cli| + ci NOOP; + "CI adapters: the workflows' matrix and !benchmark-comment plumbing (safe to run by hand, rarely needed)" + + SUBCOMMANDS: + benchCiParseCmd; + benchCiMatrixCmd +] + +def benchCmd : Cli.Cmd := `[Cli| + bench NOOP; + "Benchmark cells: run, compare, and publish locally exactly what CI runs" + + SUBCOMMANDS: + benchRunCmd; + benchShardCmd; + benchCompareCmd; + benchReportCmd; + benchBmfCmd; + benchFetchMainCmd; + benchCiCmd +] diff --git a/Ix/Cli/CheckRsCmd.lean b/Ix/Cli/CheckRsCmd.lean index 1cb0fbf7d..9d22f0299 100644 --- a/Ix/Cli/CheckRsCmd.lean +++ b/Ix/Cli/CheckRsCmd.lean @@ -6,18 +6,22 @@ - Default (Meta): kernel runs with metadata fields populated (Lean.Name, binder info, mdata). Supports `--ns` / `--consts` / `--consts-file` - for seed filtering and `--fail-out` for bisect-loop workflows. + for seed filtering and `--fail-out` for bisect-loop workflows. Seeded + meta checks are SUBJECT-ONLY: each seed is checked with its deps + lazily ingressed but trusted, not re-checked. - `--anon` (metadata-free): the env is loaded via `Env::get_anon` — `named`/`names`/`comms` sections are discarded at load time, never - reaching the kernel. Every kernel-checkable address (every constant - except Muts blocks and projections — projections are covered by - their parent block) is checked. The kernel's typechecking logic - structurally cannot read metadata (`M::MField` is `()` in Anon - mode); progress labels are `@` addresses, not names. - - `--anon` is incompatible with `--ns` / `--consts` / `--consts-file`: - the anon path checks everything in the env. Add `--addrs ` - in the future if address-based filtering is needed. + reaching the kernel. The kernel's typechecking logic structurally + cannot read metadata (`M::MField` is `()` in Anon mode); progress + labels are `@` addresses, not names. + + Without a filter, every kernel-checkable address is checked (whole + env). With `--consts` / `--consts-file`, the named constants are + checked together with their FULL dependency closures — the same mode + and scope as the zkVM hosts' `--consts` execute path, so an + out-of-circuit run is directly comparable to the in-circuit one. Add + `--skip-deps` for a subject-only check (deps trusted), mirroring + `zisk-host --skip-deps`. `--ns` prefix filtering stays meta-only. Direct Lean → kernel typechecking (compile-and-check from source) is available via the `rsCheckConstsFFI` API for tests @@ -29,6 +33,9 @@ public import Cli public import Ix.Common public import Ix.KernelCheck public import Ix.Meta +public import Ix.TracingTexray +public import Ix.Benchmark.Results +public import Ix.Cli.ConstsFile public import Ix.Cli.ValidateCmd public import Std.Internal.UV.System @@ -48,18 +55,6 @@ private structure SeedSpec where private def SeedSpec.isEmpty (s : SeedSpec) : Bool := s.prefixes.isEmpty && s.exacts.isEmpty -/-- Read one constant name per line from `path`. Blank lines and lines - starting with `#` (after trimming) are ignored. -/ -private def readNamesFile (path : String) : IO (List Lean.Name) := do - let content ← IO.FS.readFile path - let lines := content.splitOn "\n" - let names : List Lean.Name := lines.filterMap fun raw => - let cs := raw.toList.dropWhile Char.isWhitespace - let trimmed := String.ofList (cs.reverse.dropWhile Char.isWhitespace).reverse - if trimmed.isEmpty || trimmed.startsWith "#" then none - else some trimmed.toName - pure names - /-- Build a `SeedSpec` from `--ns`, `--consts`, and `--consts-file`. -/ private def resolveSeedSpec (p : Cli.Parsed) : IO (Option SeedSpec) := do let nsFlag := p.flag? "ns" @@ -82,7 +77,8 @@ private def resolveSeedSpec (p : Cli.Parsed) : IO (Option SeedSpec) := do exacts := exacts ++ parsed if let some flag := fileFlag then let path := flag.as! String - let parsed ← readNamesFile path + -- Shared grammar (Ix.Cli.ConstsFile); meta seeds resolve via `toName`. + let parsed := (← ConstsFile.read path).toList.map (·.toName) if parsed.isEmpty then IO.println s!"[check] warning: --consts-file '{path}' yielded zero names" else @@ -137,7 +133,8 @@ private def reportFailures (failures : Array (String × String)) /-- Anon-mode runner: dispatch to `rsCheckAnonFFI`. The Rust side checks every kernel-checkable address in the env and streams failures to - `failOutPath` (when nonempty). -/ + `failOutPath` (when nonempty). With `--json`, the run is additionally + recorded as one env-keyed results row (key from `--json-name`). -/ private def runCheckAnon (envPath : String) (p : Cli.Parsed) : IO UInt32 := do let verbose := p.flag? "verbose" |>.isSome let failOutPath : String := @@ -168,8 +165,72 @@ private def runCheckAnon (envPath : String) (p : Cli.Parsed) : IO UInt32 := do if !failOutPath.isEmpty then IO.println s!"[check] streamed {failures.size} failure(s) to {failOutPath}" - IO.println s!"##check## {elapsed} {passed} {failures.size} {results.size}" - return if failures.isEmpty then 0 else 1 + if let some flag := p.flag? "json" then + let key := (p.flag? "json-name").map (·.as! String) |>.getD "env" + let secs := elapsed.toFloat / 1000.0 + let tput := if elapsed > 0 + then results.size.toFloat * 1000.0 / elapsed.toFloat else 0.0 + let peakRss ← TracingTexray.peakTreeRssBytes + let status := if failures.isEmpty then "ok" else "rejected" + Ix.Benchmark.Results.writeRow (flag.as! String) key status + [ ("constants", Lean.toJson results.size) + , ("check-time", Ix.Benchmark.Results.jsonRound 3 secs) + , ("throughput", Ix.Benchmark.Results.jsonRound 2 tput) + , ("peak-rss", Lean.toJson peakRss) ] + + return if failures.isEmpty then 0 else Ix.Benchmark.Results.exitRejected + +/-- Anon-mode per-constant runner: dispatch to `rsCheckAnonConstsFFI`. Checks + the named constants and (by default) their full dependency closures — the + zkVM hosts' semantics — or subject-only under `--skip-deps`. With + `--json`, each name runs as its own closure check and records its own + per-name row (see `Ix.Benchmark.Results`); without it, the names union + into one check set. -/ +private def runCheckAnonConsts (envPath : String) (p : Cli.Parsed) : IO UInt32 := do + let verbose := p.flag? "verbose" |>.isSome + let skipDeps := p.hasFlag "skip-deps" + let failOutPath : String := + match p.flag? "fail-out" with + | some flag => flag.as! String + | none => "" + let jsonPath : String := + match p.flag? "json" with + | some flag => flag.as! String + | none => "" + if !jsonPath.isEmpty && !failOutPath.isEmpty then + p.printError "error: --json per-name rows and --fail-out cannot be combined" + return Ix.Benchmark.Results.exitUsage + -- Raw strings (no toName round-trip): the FFI resolves displayed forms + -- against the env's `named` map, matching the zkVM hosts' resolution. + let names ← ConstsFile.gather p + if names.isEmpty then + IO.println "[check] error: --consts/--consts-file resolved to zero names" + return Ix.Benchmark.Results.exitUsage + + let scope := if skipDeps then "subject-only" else "full-closure" + let shape := if jsonPath.isEmpty then "union" else "per-name rows" + IO.println s!"Running Ix kernel check (anon mode, {scope}, {shape}) on {envPath}" + IO.println s!"[check] {names.size} seed constant(s): {", ".intercalate names.toList}" + let start ← IO.monoMsNow + let results ← rsCheckAnonConstsFFI envPath names skipDeps (!verbose) + failOutPath jsonPath + let elapsed := (← IO.monoMsNow) - start + + let mut passed := 0 + let mut failures : Array (String × String) := #[] + for (hex, res) in results do + match res with + | none => passed := passed + 1 + | some err => failures := failures.push (s!"#{hex}", err.message) + + IO.println s!"[check] checked {results.size} constants in {elapsed.formatMs}" + IO.println s!"[check] {passed}/{results.size} passed" + reportFailures failures + + if !failOutPath.isEmpty then + IO.println s!"[check] streamed {failures.size} failure(s) to {failOutPath}" + + return if failures.isEmpty then 0 else Ix.Benchmark.Results.exitRejected /-- Meta-mode runner: dispatch to `rsCheckIxonFFI` with seed filtering. -/ private def runCheckMeta (envPath : String) (p : Cli.Parsed) : IO UInt32 := do @@ -219,8 +280,7 @@ private def runCheckMeta (envPath : String) (p : Cli.Parsed) : IO UInt32 := do if !failOutPath.isEmpty then IO.println s!"[check] streamed {failures.size} failure(s) to {failOutPath}" - IO.println s!"##check## {elapsed} {passed} {failures.size} {seedNames.size}" - return if failures.isEmpty then 0 else 1 + return if failures.isEmpty then 0 else Ix.Benchmark.Results.exitRejected def runCheckRsCmd (p : Cli.Parsed) : IO UInt32 := do let some pathArg := p.positionalArg? "path" @@ -228,6 +288,10 @@ def runCheckRsCmd (p : Cli.Parsed) : IO UInt32 := do return 1 let envPath := pathArg.as! String + -- Start the process-tree RSS sampler so `--json` rows can report an + -- accurate peak-rss (the parallel kernel check's high-water mark). + TracingTexray.startSampler + -- `--workers N` is plumbed through the existing -- `IX_KERNEL_CHECK_WORKERS` env var that `resolve_kernel_check_workers` -- (`src/ffi/kernel.rs`) reads. Setting `1` forces a single-threaded @@ -240,14 +304,19 @@ def runCheckRsCmd (p : Cli.Parsed) : IO UInt32 := do Std.Internal.UV.System.osSetenv "IX_KERNEL_CHECK_WORKERS" (toString n) let anon := p.flag? "anon" |>.isSome + let hasConsts := (p.flag? "consts").isSome || (p.flag? "consts-file").isSome + if p.hasFlag "skip-deps" && !(anon && hasConsts) then + p.printError "error: --skip-deps only applies to `--anon --consts/--consts-file` \ + (meta-mode seeded checks are always subject-only)" + return 1 if anon then - let hasConsts := p.flag? "consts" |>.isSome - let hasNs := p.flag? "ns" |>.isSome - let hasConstsFile := p.flag? "consts-file" |>.isSome - if hasConsts || hasNs || hasConstsFile then - p.printError "error: --anon checks the entire env; --consts/--ns/--consts-file are unsupported" + if p.flag? "ns" |>.isSome then + p.printError "error: --ns prefix filtering is meta-only; --anon supports --consts/--consts-file" return 1 - runCheckAnon envPath p + if hasConsts then + runCheckAnonConsts envPath p + else + runCheckAnon envPath p else runCheckMeta envPath p @@ -256,14 +325,17 @@ end Ix.Cli.CheckRsCmd open Ix.Cli.CheckRsCmd in def checkRsCmd : Cli.Cmd := `[Cli| "check-rs" VIA runCheckRsCmd; - "Typecheck a `.ixe` through the Rust kernel" + "Typecheck a `.ixe` through the Rust kernel. Exits 0 when everything passes, 3 when the kernel rejects any constant (with --json, the rejected rows are on disk), nonzero otherwise on infrastructure failures." FLAGS: anon; "Run the kernel in anon mode (no metadata read from .ixe)" ns : String; "Comma-separated Lean.Name prefixes to filter on (meta mode only)" - consts : String; "Comma-separated EXACT constant names to seed (meta mode only)" - "consts-file" : String; "Path to a file with one constant name per line (meta mode only)" + consts : String; "Comma-separated EXACT constant names. Meta mode: subject-only seed check. Anon mode: full-closure check of each name (the zkVM hosts' semantics; --skip-deps for subject-only)." + "consts-file" : String; "Path to a file with one constant name per line (`#` comments); unions with --consts." + "skip-deps"; "With --anon --consts: check each named constant subject-only, trusting its deps (same flag as zisk-host/sp1-host/bench-typecheck)." "fail-out" : String; "Write failing constants to this path (consumable by --consts-file)" + json : String; "Write benchmark results rows to this path (anon mode). Whole-env: one row keyed by --json-name. With --consts: each name runs as its OWN closure check (the zkVM hosts' per-constant scope) and records its own timed row, env loaded once." + "json-name" : String; "Row key for the whole-env --json row (default: `env`)" workers : Nat; "Number of parallel kernel-check workers; 1 disables parallelism (default: available_parallelism). Plumbs via IX_KERNEL_CHECK_WORKERS env var." verbose; "Log every constant on its own line (default: quiet)" diff --git a/Ix/Cli/CompileCmd.lean b/Ix/Cli/CompileCmd.lean index a00f95e59..edf323db4 100644 --- a/Ix/Cli/CompileCmd.lean +++ b/Ix/Cli/CompileCmd.lean @@ -3,6 +3,9 @@ public import Cli public import Ix.Common public import Ix.CompileM public import Ix.Meta +public import Ix.TracingTexray +public import Ix.Benchmark.Results +public import Ix.Cli.ConstsFile public import Ix.Cli.ValidateCmd public section @@ -14,19 +17,6 @@ private def defaultOutPathFor (pathStr : String) : String := let stem := path.fileStem.getD (path.fileName.getD pathStr) stem.toLower ++ ".ixe" -/-- Read one constant name per line from `path`. Blank lines and lines - starting with `#` (after trimming) are ignored. Mirrors - `Ix.Cli.CheckCmd.readNamesFile`. -/ -private def readNamesFile (path : String) : IO (List Lean.Name) := do - let content ← IO.FS.readFile path - let lines := content.splitOn "\n" - let names : List Lean.Name := lines.filterMap fun raw => - let cs := raw.toList.dropWhile Char.isWhitespace - let trimmed := String.ofList (cs.reverse.dropWhile Char.isWhitespace).reverse - if trimmed.isEmpty || trimmed.startsWith "#" then none - else some trimmed.toName - pure names - def runCompileCmd (p : Cli.Parsed) : IO UInt32 := do let some path := p.positionalArg? "path" | p.printError "error: must specify to a Lean source file" @@ -51,7 +41,10 @@ def runCompileCmd (p : Cli.Parsed) : IO UInt32 := do if let some flag := p.flag? "exclude" then for n in parsePrefixes (flag.as! String) do s := s.insert n if let some flag := p.flag? "exclude-file" then - for n in ← readNamesFile (flag.as! String) do s := s.insert n + -- Shared names-file grammar (Ix.Cli.ConstsFile); names resolve here + -- via `toName` like the `--exclude` comma-list. + for n in ← Ix.Cli.ConstsFile.read (flag.as! String) do + s := s.insert n.toName pure s if !excludeSet.isEmpty then IO.println s!"[compile] exclude: {excludeSet.size} name(s) will be dropped from seed set" @@ -67,7 +60,43 @@ def runCompileCmd (p : Cli.Parsed) : IO UInt32 := do -- Seeds pass through `collectDeps` for the transitive-dep closure. -- Flag name is `--module` (not `--ns`) because the match is against -- the source module name, not the decl's own namespace. - let constList ← match p.flag? "module" with + -- `--consts` / `--consts-file`: seed by EXACT constant name, transitive + -- deps via `collectDeps` — a closure-only env (e.g. one benchmark constant + -- + deps) instead of the whole import env. Resolution tries `String.toName` + -- first, then a displayed-form scan so `_private`/numeric components + -- round-trip. Mutually exclusive with `--module`; `--exclude` doesn't + -- apply (the seed list is already explicit). + let constsSeeds ← Ix.Cli.ConstsFile.gather p + if !constsSeeds.isEmpty && (p.flag? "module").isSome then + p.printError "error: --consts/--consts-file and --module are mutually exclusive" + return 1 + let constList ← + if !constsSeeds.isEmpty then do + let mut seeds : List Lean.Name := [] + let mut missing : List String := [] + -- Displayed-form fallback, built at most once: a fresh + -- `constants.toList.find?` scan per missing name is O(names × env), + -- seconds of allocation per name at mathlib scale. + let mut byDisplayed : Option (Std.HashMap String Lean.Name) := none + for n in constsSeeds do + let name := n.toName + if leanEnv.constants.contains name then + seeds := name :: seeds + else + let map := byDisplayed.getD <| .ofList <| + leanEnv.constants.toList.map fun (m, _) => (toString m, m) + byDisplayed := some map + match map.get? n with + | some m => seeds := m :: seeds + | none => missing := n :: missing + if !missing.isEmpty then + p.printError s!"error: no constant(s) named {missing} in the environment" + return 1 + IO.println s!"[compile] consts: {seeds.length} seed constant(s)" + let closed := collectDeps leanEnv seeds + IO.println s!"[compile] consts: {closed.length} constants after transitive-dep closure" + pure closed + else match p.flag? "module" with | none => if excludeSet.isEmpty then pure leanEnv.constants.toList else @@ -98,13 +127,31 @@ def runCompileCmd (p : Cli.Parsed) : IO UInt32 := do let totalConsts := constList.length println! "Total constants: {totalConsts}" + -- Window the tree-RSS sampler around the compile so the row's peak-rss + -- is the compile step's own high-water (the loaded Lean env is still in + -- the baseline — RSS is absolute). + let benched := (p.flag? "json").isSome + if benched then + TracingTexray.startSampler + TracingTexray.resetPeakTreeRss let start ← IO.monoMsNow let bytes ← Ix.CompileM.rsCompileEnvBytesFFI constList let elapsed := (← IO.monoMsNow) - start println! "Compiled {fmtBytes bytes.size} env in {elapsed.formatMs}" - -- Machine-readable line for CI benchmark tracking - IO.println s!"##benchmark## {elapsed} {bytes.size} {totalConsts}" + if let some flag := p.flag? "json" then + let key := (p.flag? "json-name").map (·.as! String) + |>.getD ((FilePath.mk pathStr).fileStem.getD "env") + let secs := elapsed.toFloat / 1000.0 + let tput := if elapsed > 0 + then totalConsts.toFloat * 1000.0 / elapsed.toFloat else 0.0 + let peakRss ← TracingTexray.peakTreeRssBytes + Ix.Benchmark.Results.writeRow (flag.as! String) key "ok" + [ ("compile-time", Ix.Benchmark.Results.jsonRound 3 secs) + , ("file-size", Lean.toJson bytes.size) + , ("constants", Lean.toJson totalConsts) + , ("throughput", Ix.Benchmark.Results.jsonRound 2 tput) + , ("peak-rss", Lean.toJson peakRss) ] -- Persist the serialized IxonEnv (`Env::put` bytes) to disk so subsequent -- runs (e.g. `ix check-ixon`) can skip the Lean → IxOn compile step. The @@ -124,9 +171,13 @@ def compileCmd : Cli.Cmd := `[Cli| FLAGS: out : String; "Output path for serialized Ixon.Env bytes; defaults to the lowercased input file stem with `.ixe` (e.g. CompileMathlib.lean -> compilemathlib.ixe)" + consts : String; "Comma-separated EXACT constant names to compile (transitive deps pulled in automatically) instead of the whole import env — e.g. `Nat.add_comm`. Same flag/shape as `ix check --consts`. Mutually exclusive with --module; --exclude does not apply." + "consts-file" : String; "Additionally read seed constant names from a file (one per line; `#` comments and blank lines ignored). Unions with --consts." module : String; "Comma-separated module-name prefixes to filter on (e.g. 'Tests.Ix.Kernel.TutorialDefs,Tests.Ix.Kernel.NatReduction'). Match is against the SOURCE MODULE a constant came from (via `Lean.Environment.getModuleIdxFor?`), not the constant's own name — so macro-emitted decls that register under unqualified names still get caught when their host module's name matches. Transitive deps are pulled in automatically." exclude : String; "Comma-separated exact Lean.Name(s) to strip from the seed set. Excluded names that are still referenced by another seed will reappear via the transitive-dep closure." "exclude-file" : String; "Path to a file with one Lean.Name per line to strip from the seed set. Same semantics as --exclude; same line format as `ix check --consts-file`." + json : String; "Write the compile's benchmark results row (compile-time, file-size, constants, throughput) to this path, merging into any existing rows object." + "json-name" : String; "Row key for the --json row (default: the input file's stem, e.g. `CompileInitStd`)" ARGS: path : String; "Path to the Lean source file to compile." diff --git a/Ix/Cli/ConstsFile.lean b/Ix/Cli/ConstsFile.lean new file mode 100644 index 000000000..5083348b4 --- /dev/null +++ b/Ix/Cli/ConstsFile.lean @@ -0,0 +1,61 @@ +/- + Shared parsing for constant-name inputs (`--consts` comma-lists and + `--consts-file` files) across every CLI that takes them: `ix check-rs`, + `ix compile --exclude-file`, and `bench-typecheck`. + + One grammar everywhere: one name per line, everything from a `#` to end of + line is a comment (whole-line or inline), blank lines dropped. `#` never + appears in a Lean name, so splitting on it is safe. The zkVM hosts' + `--consts-file` (Rust `collect_consts`) parses the same grammar, so a single + names file drives all backends identically. + + Names stay RAW strings here — resolution differs per caller (`toName` for + meta-mode seeds, string-match against the env's `named` map for the anon / + zkVM-style paths, where a `toName` round-trip could mangle numeric or + private components). +-/ +module +public import Cli + +public section + +namespace Ix.Cli.ConstsFile + +/-- Parse names-file contents: one name per line, `#`-to-EOL comments, + blank lines dropped. -/ +def parseLines (contents : String) : Array String := + (contents.splitOn "\n").filterMap (fun line => + let s := ((line.splitOn "#").head?.getD "").trimAscii + if s.isEmpty then none else some s.toString) |>.toArray + +/-- Read and parse a names file. -/ +def read (path : String) : IO (Array String) := + parseLines <$> IO.FS.readFile path + +/-- Split a `--consts`-style comma-list into trimmed, non-empty names. -/ +def parseCommaList (arg : String) : Array String := + (arg.splitOn ",").filterMap (fun s => + let t := s.trimAscii + if t.isEmpty then none else some t.toString) |>.toArray + +/-- Union of a parsed `--consts` comma-list flag and a `--consts-file` file + (both optional), deduped in first-seen order. -/ +def gather (p : Cli.Parsed) + (constsFlag : String := "consts") (fileFlag : String := "consts-file") : + IO (Array String) := do + let fromFlag : Array String := + match p.flag? constsFlag with + | some f => parseCommaList (f.as! String) + | none => #[] + let fromFile : Array String ← + match p.flag? fileFlag with + | some f => read (f.as! String) + | none => pure #[] + -- Linear-scan dedupe: name lists are tens of entries, not thousands. + let mut acc : Array String := #[] + for n in fromFlag ++ fromFile do + if !acc.contains n then + acc := acc.push n + return acc + +end Ix.Cli.ConstsFile diff --git a/Ix/Cli/IngressCmd.lean b/Ix/Cli/IngressCmd.lean index b66c4f219..948e4dcd4 100644 --- a/Ix/Cli/IngressCmd.lean +++ b/Ix/Cli/IngressCmd.lean @@ -64,8 +64,7 @@ def runIngressCmd (p : Cli.Parsed) : IO UInt32 := do let elapsed := (← IO.monoMsNow) - start IO.println s!"[ingress] ingressed {kenvLen} kernel consts in {elapsed.formatMs}" - -- Machine-readable line for CI benchmark tracking, mirrors - -- `ix compile`'s `##benchmark##` shape. + -- Machine-readable line: ` `. IO.println s!"##ingress## {elapsed} {kenvLen} {totalConsts}" return 0 diff --git a/Ix/Cli/ShardCmd.lean b/Ix/Cli/ShardCmd.lean index 002e9cd74..8bdee9638 100644 --- a/Ix/Cli/ShardCmd.lean +++ b/Ix/Cli/ShardCmd.lean @@ -14,10 +14,19 @@ manifest and prints a what-if report (per-shard cost + total cross-shard ingress). The partitioner is self-contained — no external graph-library dependency. + + `ix shard extract --consts `: the pipeline's scoping + step — extract the named constants' dependency closure from a serialized + env into a standalone `.ixe`, without recompiling from source. The output + carries the closure's genuine constant bytes, blobs, and reducibility + hints, plus each closure constant's name→address entry, so it composes + with everything that consumes a `.ixe` (`ix profile` → `ix shard`, + `ix check-rs --consts`, the zkVM hosts, `bench-typecheck`). -/ module public import Cli public import Ix.KernelCheck +public import Ix.Cli.ConstsFile public section @@ -25,6 +34,43 @@ open Ix.KernelCheck namespace Ix.Cli.ShardCmd +def runShardExtractCmd (p : Cli.Parsed) : IO UInt32 := do + let some pathArg := p.positionalArg? "path" + | p.printError "error: must specify to a .ixe file" + return 1 + let envPath := pathArg.as! String + let names ← Ix.Cli.ConstsFile.gather p + if names.isEmpty then + p.printError "error: pass at least one name via --consts or --consts-file" + return 1 + let outPath : String := + match p.flag? "out" with + | some flag => flag.as! String + -- Default output mirrors the first constant's slug next to the source + -- env: `init.ixe --consts Nat.add_comm` → `nat_add_comm.ixe`. + | none => + let slug := names[0]!.map fun c => + if c.isAlphanum then c.toLower else '_' + s!"{slug}.ixe" + let quiet := !(p.flag? "verbose" |>.isSome) + rsEnvExtractFFI envPath names outPath quiet + IO.println s!"[extract] wrote {outPath} ({names.size} root name(s))" + return 0 + +def shardExtractCmd : Cli.Cmd := `[Cli| + "extract" VIA runShardExtractCmd; + "Extract named constants + their dependency closure from a `.ixe` into a standalone `.ixe`" + + FLAGS: + consts : String; "Comma-separated EXACT constant names (displayed form) to extract, e.g. `Nat.add_comm,String.append`. Same flag/shape as `ix check-rs --consts`. A mutual-block member extracts its whole block." + "consts-file" : String; "Additionally read names from a file (one per line; `#` comments and blank lines ignored). Unions with --consts." + out : String; "Output `.ixe` path. Defaults to a slug of the first name (e.g. `nat_add_comm.ixe`)." + verbose; "Print extraction details to stderr." + + ARGS: + path : String; "Path to the source `.ixe` (e.g. from `ix compile`)." +] + def runShardCmd (p : Cli.Parsed) : IO UInt32 := do let some pathArg := p.positionalArg? "path" | p.printError "error: must specify to a .ixprof file" @@ -87,6 +133,9 @@ def shardCmd : Cli.Cmd := `[Cli| ARGS: path : String; "Path to a .ixprof produced by `ix profile`" + + SUBCOMMANDS: + shardExtractCmd ] end diff --git a/Ix/KernelCheck.lean b/Ix/KernelCheck.lean index f32a6fab7..5449c0a5c 100644 --- a/Ix/KernelCheck.lean +++ b/Ix/KernelCheck.lean @@ -142,6 +142,58 @@ opaque rsCheckAnonFFI : @& String → -- fail-out path ("" = none) IO (Array (String × Option CheckError)) +/-- FFI: anon-mode type-check of named constants with (by default) their full + dependency closures — the same mode and scope as the zkVM hosts' `--consts` + execute path, so an out-of-circuit run is directly comparable to the + in-circuit one. The `Bool` after the names is `skip-deps`: `true` checks + only each name's own work item (subject-only; deps trusted), mirroring + `zisk-host --skip-deps`. + + Names are the constants' displayed forms (e.g. `"Nat.add_comm"`, + `"_private.Init.….instRxcHasSize_eq"`), resolved through the env's `named` + metadata by string match — the same resolution the zkVM hosts use — after + which the check runs on the anon view (the kernel never sees names). A + member of a mutual block selects the whole block's work item. Multiple + names union their closures into one check set. + + The trailing `String` is the results-rows JSON path (`""` = off). When + set, each name is instead checked as its own independent closure run — + the zkVM hosts' per-constant scope — with the env still loaded once, and + the row `{ name: { status, constants, check-time, throughput, peak-rss } }` + flushed per name (see `Ix.Benchmark.Results` for the contract). The + per-name `check-time`/`peak-rss` window wraps only that name's closure + selection + check, so rows aren't distorted by the env load; meaningful + `peak-rss` needs `TracingTexray.startSampler`. Not combinable with a + fail-out path. + + Returns `(hex_address, Option CheckError)` pairs, one per checked target, + exactly like `rsCheckAnonFFI` (concatenated across the per-name runs when + rows are on). Errors (rather than returning) when a name doesn't resolve, + so a typo can't silently produce an empty check. -/ +@[extern "rs_kernel_check_anon_consts"] +opaque rsCheckAnonConstsFFI : + @& String → -- .ixe path + @& Array String → -- constant names (displayed form) + @& Bool → -- skip-deps (subject-only) + @& Bool → -- quiet + @& String → -- fail-out path ("" = none) + @& String → -- benchmark results JSON path ("" = off) + IO (Array (String × Option CheckError)) + +/-- FFI: extract the named constants' dependency closure from a serialized + env into a standalone `.ixe` — genuine constant bytes, blobs, and + reducibility hints, plus the closure constants' Named entries so names + still resolve — without recompiling from source. Names resolve like + `rsCheckAnonConstsFFI` (displayed form); a mutual-block member pulls its + whole block. Errors on an unresolvable name. -/ +@[extern "rs_env_extract"] +opaque rsEnvExtractFFI : + @& String → -- source .ixe path + @& Array String → -- constant names (displayed form) + @& String → -- output .ixe path + @& Bool → -- quiet + IO Unit + /-- FFI: profile a `.ixe` out of circuit, writing a `.ixprof` sidecar with per-block heartbeats + the delta-unfold graph (the sharding cost model, see `plans/sharding.md`). Runs the anon kernel over every checkable target. diff --git a/Ix/TracingTexray.lean b/Ix/TracingTexray.lean index c0c4c064d..8e37be078 100644 --- a/Ix/TracingTexray.lean +++ b/Ix/TracingTexray.lean @@ -41,6 +41,39 @@ private opaque initWith def init (s : Settings := {}) : IO Unit := initWith s.namePrefixes s.trackRam s.streaming +@[extern "rs_texray_start_sampler"] +private opaque startSamplerFFI (intervalMs : UInt64) : IO Unit + +/-- Start the process-tree RSS sampler (idempotent). Unlike the per-span + `/proc/self/status` reads, this sums RSS across this process and all its + children, so [`peakTreeRssBytes`] captures memory resident in helper + processes (e.g. a zkVM host's services). `intervalMs` is the sample period. -/ +def startSampler (intervalMs : UInt64 := 50) : IO Unit := + startSamplerFFI intervalMs + +@[extern "rs_texray_peak_tree_rss_bytes"] +private opaque peakTreeRssBytesFFI : IO UInt64 + +/-- Peak resident-set size in bytes across this process and its children. `0` + until [`startSampler`] has run or on non-Linux platforms. -/ +def peakTreeRssBytes : IO Nat := do + return (← peakTreeRssBytesFFI).toNat + +@[extern "rs_texray_reset_peak_tree_rss"] +private opaque resetPeakTreeRssFFI : IO Unit + +/-- Reset the tree sampler's high-water mark, opening a new measurement + window — use between benchmark items so each reports its own peak. -/ +def resetPeakTreeRss : IO Unit := resetPeakTreeRssFFI + +@[extern "rs_texray_json_sink"] +private opaque jsonSinkFFI (path : @& String) : IO Unit + +/-- Direct the per-span timing sink to `path` (JSON Lines). Pair with `init` + (streaming) so the examined `aiur/*` / `stark/*` spans are recorded. -/ +def jsonSink (path : String) : IO Unit := + jsonSinkFFI path + end TracingTexray end diff --git a/Main.lean b/Main.lean index b1ee08f42..7bdfd5b88 100644 --- a/Main.lean +++ b/Main.lean @@ -1,5 +1,6 @@ --import Ix.Cli.StoreCmd import Ix.Cli.AddrOfCmd +import Ix.Cli.BenchReport import Ix.Cli.CheckCmd import Ix.Cli.CodegenCmd import Ix.Cli.CheckRsCmd @@ -25,6 +26,7 @@ def ixCmd : Cli.Cmd := `[Cli| SUBCOMMANDS: --storeCmd; + benchCmd; compileCmd; checkCmd; checkRsCmd; diff --git a/README.md b/README.md index d17190b0f..d9410d4cd 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,10 @@ Ix consists of the following core components: ## Benchmarks -Compiler performance benchmarks are tracked at https://bencher.dev/console/projects/ix/plots +Benchmarks (compiler, kernel, and zk-prover backends) are tracked at +https://bencher.dev/console/projects/ix/plots. `ix bench` runs the same +cells locally, and `!benchmark` runs them on a PR — see +[docs/benchmarking.md](docs/benchmarking.md). ## Usage @@ -231,7 +234,7 @@ Non-Nix users: install the SP1 toolchain manually per the ``` For a larger, realistic env compile one of the `Benchmarks/Compile` - targets, then scope proving to a single constant with `--constant` + targets, then scope proving to one or more constants with `--consts` (step 2): ``` @@ -250,7 +253,7 @@ Non-Nix users: install the SP1 toolchain manually per the # Prove a single constant out of a larger env (Anon-only): the host resolves # the name and ships only that constant's closure sub-env. Full-closure by # default; add --skip-deps for a subject-only check (deps trusted). - WITHOUT_VK_VERIFICATION=1 RUST_LOG=info cargo run --release -- --ixe ../../init.ixe --constant Nat.add_comm + WITHOUT_VK_VERIFICATION=1 RUST_LOG=info cargo run --release -- --ixe ../../init.ixe --consts Nat.add_comm ``` With no `--ixe`, the host runs against an empty `Ixon.Env`. @@ -354,19 +357,20 @@ Non-Nix users: install Zisk manually per the RUST_LOG=info cargo run --release -- --verify-constraints --ixe ../minimal.ixe # Generate and verify a VadcopFinal proof of the same typecheck (CPU) RUST_LOG=info cargo run --release -- --ixe ../minimal.ixe - # Check a single named constant out of a larger env. The host resolves the + # Check one or more named constants out of a larger env. The host resolves each # name and ships only its closure sub-env (lazy fault-in, no whole-env load). # By default this is the FULL-CLOSURE typecheck — the constant and its whole - # dependency closure (matching the Aiur `bench-typecheck --constant`). + # dependency closure (matching the Aiur `bench-typecheck --consts `). # Composes with --execute (cycles only) and plain prove. - RUST_LOG=info cargo run --release -- --execute --ixe ../init.ixe --constant Nat.add_comm - RUST_LOG=info cargo run --release -- --ixe ../init.ixe --constant Nat.add_comm + RUST_LOG=info cargo run --release -- --execute --ixe ../init.ixe --consts Nat.add_comm + RUST_LOG=info cargo run --release -- --ixe ../init.ixe --consts Nat.add_comm,Nat.succ # Add --skip-deps for a subject-only check (deps trusted, not re-checked): - RUST_LOG=info cargo run --release -- --execute --ixe ../init.ixe --constant Vector.extract_append --skip-deps + RUST_LOG=info cargo run --release -- --execute --ixe ../init.ixe --consts Vector.extract_append --skip-deps ``` - `--constant` / `--skip-deps` are the same flags the Aiur `bench-typecheck` - uses, so the two backends share one vocabulary. `--skip-deps` trusts + `--consts` / `--skip-deps` are the same flags `ix check`, `sp1-host`, and the + Aiur `bench-typecheck` use, so all four share one vocabulary. `--skip-deps` + trusts dependencies rather than re-checking them, so it is far cheaper than the full-closure default — reserve it for constants too expensive to full-closure-check that also can't be sharded (e.g. `Vector.extract_append` @@ -477,41 +481,24 @@ Non-Nix users: install Zisk manually per the [`DEFAULT_MEMORY_LIMIT`](https://github.com/succinctlabs/sp1/blob/v6.2.0/crates/core/executor/src/opts.rs#L25), configurable via `MEMORY_LIMIT` env var up to a ~1 TB JIT ceiling [`MAX_JIT_LOG_ADDR`](https://github.com/succinctlabs/sp1/blob/v6.2.0/crates/primitives/src/consts.rs#L11)), - or scope to a single constant with `--constant ` (all backends), - which resolves the name and ships only that constant's closure sub-env to the - guest. By default it re-checks the full dependency closure; add `--skip-deps` - to check it **subject-only** (dependencies trusted and lazily faulted in, not + or scope to one or more constants with `--consts ` (all backends), + which resolves each name and ships only that constant's closure sub-env to the + guest. By default each re-checks its full dependency closure; add `--skip-deps` + to check them **subject-only** (dependencies trusted and lazily faulted in, not re-typechecked) — so individual constants of a large env still fit the cap, even ones whose full-closure typecheck would not. To prove a large env in full under Zisk, shard it (see *Sharding large environments* below): each shard ships only its own closure sub-env, so the pieces fit the cap even when the whole env does not. - **Host RAM cap (`--max-witness-stored`).** Distinct from the in-guest - heap cap above, the prover side (Zisk's `proofman`) holds in-flight - witness traces in host RAM during `CALCULATING_CONTRIBUTIONS`. Peak - host RAM per shard ≈ `fixed-overhead + N × avg-witness-size`, where - `N` is the `max_witness_stored` setting. With the Blake3f precompile the - Ix kernel typecheck workload measures roughly `40 GB + N × 16 GB` on - typical 200–300 kB anon-byte shards — e.g. `N = 10` peaks near 200 GB - (a `--shard-bytes 250000 --max-witness-stored 10` mergesort run completes - under a 200 GiB guard without tripping it). An earlier pre-Blake3f figure - of ~25 GB per witness is stale; the precompile shrank the witness. - - The `zisk-host` CLI defaults to `--max-witness-stored 5` (Zisk's - built-in default is 10, tuned for larger-memory boxes). Override per - machine: - - | Host RAM | `--max-witness-stored` | Notes | - | -------- | ---------------------- | ------------------------------------------------------ | - | ≤ 128 GB | `3` | Override down; consider smaller shards too | - | 256 GB | `5` (project default) | Comfortable margin on the typical setup | - | 512 GB | `10` (Zisk default) | Override up for maximum prover parallelism | - | ≥ 1 TB | `10` (Zisk default) | Override up; default is conservative for this workload | - - Lowering the cap roughly linearly bounds peak RAM but throttles - prover parallelism (~10–30 % slower in practice). Raise it if your - machine has more RAM headroom; lower it if you OOM during + **Host RAM during proving.** Distinct from the in-guest heap cap above, + the prover side (Zisk's `proofman`) holds in-flight witness traces in host + RAM during `CALCULATING_CONTRIBUTIONS`. The number of resident witnesses + (Zisk's `max_witness_stored`) is left at Zisk's built-in default of 10: + we measured that lowering it does not materially reduce peak host RAM or + prove time for the Ix kernel typecheck workload, so it is not exposed as a + knob. Peak host RAM per shard is instead governed by shard size — prove + smaller shards (`--shard-bytes`) if you OOM during `CALCULATING_CONTRIBUTIONS`. Not relevant for `--execute` or `--verify-constraints` modes. diff --git a/crates/aiur/src/execute.rs b/crates/aiur/src/execute.rs index 21f24f234..561f2ee33 100644 --- a/crates/aiur/src/execute.rs +++ b/crates/aiur/src/execute.rs @@ -846,6 +846,61 @@ pub fn unconstrained_big_uint_div_mod_helper( Ok((q_ptr, r_ptr)) } +/// Read-only twin of `unconstrained_big_uint_div_mod_helper` for trace +/// population: recompute `(q, r)` and resolve the list-head pointers the +/// execution already recorded in `memory[10]` — every node was built there +/// during execution, so each key must be present. +pub fn find_unconstrained_big_uint_div_mod( + a_ptr: G, + b_ptr: G, + memory: &FxIndexMap, +) -> Result<(G, G), String> { + let a_limbs = read_klimbs_u64(memory, a_ptr)?; + let b_limbs = read_klimbs_u64(memory, b_ptr)?; + let a_big = klimbs_u64_to_biguint(&a_limbs); + let b_big = klimbs_u64_to_biguint(&b_limbs); + let (q_big, r_big) = if b_big == num_bigint::BigUint::ZERO { + (num_bigint::BigUint::ZERO, a_big.clone()) + } else { + (&a_big / &b_big, &a_big % &b_big) + }; + let q_ptr = find_klimbs_u64(memory, &biguint_to_klimbs_u64(&q_big))?; + let r_ptr = find_klimbs_u64(memory, &biguint_to_klimbs_u64(&r_big))?; + Ok((q_ptr, r_ptr)) +} + +/// Read-only twin of `build_klimbs_u64`: resolve the pointer of each +/// (already-recorded) list node without inserting. +fn find_klimbs_u64( + memory: &FxIndexMap, + limbs: &[u64], +) -> Result { + let queries = memory.get(&10).ok_or_else(|| { + "memory[10] channel not registered (no List in program?)".to_string() + })?; + let nil_key: Vec = + std::iter::once(G::ONE).chain((0..9).map(|_| G::ZERO)).collect(); + let mut tail_ptr = queries + .get(&nil_key) + .ok_or_else(|| "List Nil node not recorded".to_string())? + .output[0]; + for limb in limbs.iter().rev() { + let mut key: Vec = Vec::with_capacity(10); + key.push(G::ZERO); // Cons tag (first variant of ListNode‹U64›) + for b in &limb.to_le_bytes() { + key.push(G::from_u8(*b)); + } + key.push(tail_ptr); + tail_ptr = queries + .get(&key) + .ok_or_else(|| { + format!("List Cons node for limb {limb} not recorded") + })? + .output[0]; + } + Ok(tail_ptr) +} + /// Walk a `List` chain from `head_ptr` in `memory[10]`, returning the /// u64 limbs in head-first order. Each memory[10] entry is the standard Aiur /// tagged-enum layout: `[tag, byte0..byte7, next_ptr]`. `tag == 0` = Nil diff --git a/crates/aiur/src/trace.rs b/crates/aiur/src/trace.rs index e696e688b..5b45d06c3 100644 --- a/crates/aiur/src/trace.rs +++ b/crates/aiur/src/trace.rs @@ -13,7 +13,9 @@ use rayon::{ use crate::{ FxIndexMap, G, bytecode::{Block, Ctrl, Function, Op, Toplevel}, - execute::{IOBuffer, IOKeyInfo, QueryRecord}, + execute::{ + IOBuffer, IOKeyInfo, QueryRecord, find_unconstrained_big_uint_div_mod, + }, function_channel, gadgets::{bytes1::Bytes1, bytes2::Bytes2}, memory::Memory, @@ -558,10 +560,26 @@ impl Op { ), ); }, + Op::UnconstrainedBigUintDivMod(a, b) => { + // Mirrors the execute arm and the two auxiliary columns the + // constraints allocate: recompute `(q, r)` and resolve the head + // pointers execution recorded in memory[10]. Skipping the two map + // pushes would shift every later `ValIdx` (and witness column) in + // the block. + let (q_ptr, r_ptr) = find_unconstrained_big_uint_div_mod( + map[*a].0, + map[*b].0, + &context.query_record.memory_queries, + ) + .expect("BigUint div-mod result not recorded"); + for f in [q_ptr, r_ptr] { + map.push((f, 1)); + slice.push_auxiliary(index, f); + } + }, Op::AssertEq(..) | Op::IOSetInfo(..) | Op::IOWrite(..) - | Op::UnconstrainedBigUintDivMod(..) | Op::Debug(..) => {}, } } diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml new file mode 100644 index 000000000..527e17f7f --- /dev/null +++ b/crates/bench/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ix-bench" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +serde_json = { workspace = true } +tracing-texray = { workspace = true } + +[lints] +workspace = true diff --git a/crates/bench/src/lib.rs b/crates/bench/src/lib.rs new file mode 100644 index 000000000..8d1dbd5a5 --- /dev/null +++ b/crates/bench/src/lib.rs @@ -0,0 +1,209 @@ +//! The benchmark row contract shared by every measured tool. +//! +//! Each tool (`bench-typecheck`, `zisk-host`, `sp1-host`, `ix check-rs`, +//! `ix compile`) reports its results as one JSON object per benchmark name in +//! a single file: +//! +//! ```json +//! { "": { "status": "ok", "": , ..., +//! "phase:": , ... } } +//! ``` +//! +//! Rows are flushed to disk after every name, so a killed process still +//! leaves a complete file of the rows measured so far. The orchestrator +//! (`ix bench run`) identifies a killed run's in-flight name as the first +//! requested name without a row, and merges `"status": "oom"` into whatever +//! partial metrics that row holds — tools themselves only ever write `ok` or +//! `rejected`. +//! +//! Exit codes are the failure channel (no stdout markers, no log grepping): +//! `0` — every row `ok`; [`EXIT_USAGE`] — bad invocation; [`EXIT_REJECTED`] — +//! the kernel rejected a constant (its `rejected` row is on disk; the tool +//! fails fast, so later names have no rows). Any other exit is an +//! infrastructure failure. + +use std::collections::HashSet; +use std::io; +use std::path::Path; + +use serde_json::{Map, Value}; + +/// Bad invocation (unknown name, missing input, conflicting flags). +pub const EXIT_USAGE: u8 = 2; +/// The kernel rejected at least one constant; its row is on disk. +pub const EXIT_REJECTED: u8 = 3; + +/// Row outcome as written by a tool (`Oom` is only ever set by the +/// orchestrator, after the fact — a tool never observes its own OOM kill). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + Ok, + Rejected, + Oom, +} + +impl Status { + pub fn as_str(self) -> &'static str { + match self { + Status::Ok => "ok", + Status::Rejected => "rejected", + Status::Oom => "oom", + } + } +} + +/// Typed kernel-rejection error. Binaries write the `rejected` row, propagate +/// this through their error chain, and map it to [`EXIT_REJECTED`] in `main` — +/// keeping the failure channel an exit code rather than an error-message +/// format some consumer has to grep. +#[derive(Debug)] +pub struct Rejection { + pub failures: u64, + pub ctx: String, +} + +impl std::fmt::Display for Rejection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "kernel typecheck rejected {} item(s) in {}", + self.failures, self.ctx + ) + } +} + +impl std::error::Error for Rejection {} + +/// Peak resident set size (bytes) across this process *and its children*, +/// from tracing-texray's tree sampler. `None` until the sampler has started +/// or off Linux. Unlike a bare `/proc/self/status` read this includes child +/// processes (e.g. Zisk's ASM microservices, which mmap large ROMs in +/// separate PIDs). +pub fn peak_rss_bytes() -> Option { + match tracing_texray::rss_sampler::peak_tree_rss_bytes() { + 0 => None, + n => Some(n), + } +} + +/// Write the row `{ "": { "status": …, …metrics } }` into the results +/// file at `path`, merging into any existing object (overwriting on +/// collision) so a multi-name run accumulates one map with a row per name. +pub fn write_row( + path: &Path, + name: &str, + status: Status, + metrics: Value, +) -> io::Result<()> { + let mut row = match metrics { + Value::Object(m) => m, + Value::Null => Map::new(), + other => { + let mut m = Map::new(); + m.insert("value".to_string(), other); + m + }, + }; + row.insert("status".to_string(), Value::String(status.as_str().into())); + write_json_entry(path, name, Value::Object(row)) +} + +/// Append the entry `{ "": }` to the JSON object at `path`, +/// creating the file if absent. The write is ATOMIC (temp file + rename): +/// the accumulator is shared by sequential per-constant processes and +/// their orchestrator, and a watchdog KILL landing mid-write would +/// otherwise truncate it — which the tolerant readers on both sides would +/// silently reset to `{}`, losing every prior row. serde_json handles key +/// escaping, so arbitrary Lean names are safe. +pub fn write_json_entry( + path: &Path, + name: &str, + value: Value, +) -> io::Result<()> { + let mut map: Map = match std::fs::read(path) { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_default(), + Err(e) if e.kind() == io::ErrorKind::NotFound => Map::new(), + Err(e) => return Err(e), + }; + map.insert(name.to_string(), value); + let mut tmp = path.as_os_str().to_owned(); + tmp.push(".tmp"); + std::fs::write(&tmp, serde_json::to_string(&Value::Object(map))?)?; + std::fs::rename(&tmp, path) +} + +/// Union comma-separated `--consts` values with names read from a +/// `--consts-file` (one per line, `#` comments and blank lines dropped), +/// preserving first-seen order so the same name is never measured twice. +pub fn collect_consts( + consts: &[String], + consts_file: Option<&Path>, +) -> io::Result> { + let mut seen: HashSet = HashSet::new(); + let mut out: Vec = Vec::new(); + for name in consts { + let trimmed = name.trim(); + if !trimmed.is_empty() && seen.insert(trimmed.to_string()) { + out.push(trimmed.to_string()); + } + } + if let Some(path) = consts_file { + let contents = std::fs::read_to_string(path)?; + for line in contents.lines() { + let name = line.split('#').next().unwrap_or("").trim(); + if !name.is_empty() && seen.insert(name.to_string()) { + out.push(name.to_string()); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn collect_unions_and_dedups() { + let path = std::env::temp_dir().join("ix_bench_test_consts.txt"); + std::fs::write(&path, "a\nb\n# comment\n c \n\na\n").expect("write"); + let consts = vec!["a".to_string(), "d".to_string()]; + let got = collect_consts(&consts, Some(&path)).expect("collect"); + assert_eq!(got, vec!["a", "d", "b", "c"]); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn collect_trims_and_skips_empty() { + let consts = vec![" a ".to_string(), "".to_string(), "a".to_string()]; + let got = collect_consts(&consts, None).expect("collect"); + assert_eq!(got, vec!["a"]); + } + + #[test] + fn rows_accumulate_and_overwrite() { + let path = std::env::temp_dir().join("ix_bench_test_rows.json"); + let _ = std::fs::remove_file(&path); + write_row(&path, "a", Status::Ok, json!({"cycles": 1})).expect("write a"); + write_row(&path, "b", Status::Rejected, json!({})).expect("write b"); + write_row(&path, "a", Status::Ok, json!({"cycles": 2})).expect("rewrite"); + let v: Value = serde_json::from_slice(&std::fs::read(&path).expect("read")) + .expect("parse"); + assert_eq!(v["a"], json!({"status": "ok", "cycles": 2})); + assert_eq!(v["b"], json!({"status": "rejected"})); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn row_survives_preexisting_garbage() { + let path = std::env::temp_dir().join("ix_bench_test_garbage.json"); + std::fs::write(&path, "not json").expect("write"); + write_row(&path, "a", Status::Ok, json!({"t": 1.5})).expect("write"); + let v: Value = serde_json::from_slice(&std::fs::read(&path).expect("read")) + .expect("parse"); + assert_eq!(v["a"]["status"], "ok"); + let _ = std::fs::remove_file(&path); + } +} diff --git a/crates/ffi/Cargo.toml b/crates/ffi/Cargo.toml index 8ddc31c02..477add5af 100644 --- a/crates/ffi/Cargo.toml +++ b/crates/ffi/Cargo.toml @@ -16,6 +16,7 @@ dashmap = { workspace = true, features = ["rayon"] } indexmap = { workspace = true, features = ["rayon"] } itertools = { workspace = true } ixvm-codegen = { workspace = true } +ix-bench = { workspace = true } ix-common = { workspace = true } ix-compile = { workspace = true } ixon = { workspace = true } @@ -26,6 +27,7 @@ mimalloc = { workspace = true } num-bigint = { workspace = true } rayon = { workspace = true } rustc-hash = { workspace = true } +serde_json = { workspace = true } sha2 = { workspace = true } tiny-keccak = { workspace = true } tracing = { workspace = true } diff --git a/crates/ffi/src/aiur/protocol.rs b/crates/ffi/src/aiur/protocol.rs index 3bd74c322..5a9b5e147 100644 --- a/crates/ffi/src/aiur/protocol.rs +++ b/crates/ffi/src/aiur/protocol.rs @@ -190,6 +190,8 @@ extern "C" fn rs_aiur_toplevel_execute_ixvm( let fun_idx = lean_unbox_nat_as_usize(fun_idx.inner()); let mut io_buffer = decode_io_buffer(&io_data_arr, &io_map_arr); + // Same execution-phase span as `dispatch_execute`/the prove pipeline. + let _g = tracing::info_span!("aiur/execute_ixvm").entered(); let (query_record, output) = match ixvm_codegen::aiur_ixvm_runner::execute_ixvm( &toplevel, @@ -403,6 +405,11 @@ fn dispatch_execute( io_buffer: &mut IOBuffer, use_bytecode: bool, ) -> Result<(QueryRecord, Vec), String> { + // Same span name as the prove pipeline's execution phase + // (`synthesis.rs`), so a standalone execute renders/records through the + // one texray channel — timing and RAM come from the subscriber, not + // per-benchmark arithmetic. + let _g = tracing::info_span!("aiur/execute_ixvm").entered(); if use_bytecode { toplevel .execute(fun_idx, input, io_buffer) diff --git a/crates/ffi/src/kernel.rs b/crates/ffi/src/kernel.rs index c645bf18e..d7d1c2577 100644 --- a/crates/ffi/src/kernel.rs +++ b/crates/ffi/src/kernel.rs @@ -1318,12 +1318,14 @@ enum AnonWorkItem { /// [`ix_kernel::anon_work::build_anon_work`] (shared with the /// SP1/Zisk guests) and layers the FFI's per-target result-slot /// bookkeeping on top. -fn build_anon_work( - env: &IxonEnv, -) -> Result<(Vec, Vec
), String> { +/// Assign result slots to a set of kernel work items — the indexing step +/// shared by the whole-env check (every item) and the per-constant closure +/// check (a filtered subset). +fn index_anon_work( + kernel_work: Vec, +) -> (Vec, Vec
) { use ix_kernel::anon_work::AnonWorkItem as KItem; - let kernel_work = ix_kernel::anon_work::build_anon_work(env)?; let mut work: Vec = Vec::with_capacity(kernel_work.len()); let mut addrs: Vec
= Vec::new(); for item in kernel_work { @@ -1341,7 +1343,13 @@ fn build_anon_work( }, } } - Ok((work, addrs)) + (work, addrs) +} + +fn build_anon_work( + env: &IxonEnv, +) -> Result<(Vec, Vec
), String> { + Ok(index_anon_work(ix_kernel::anon_work::build_anon_work(env)?)) } #[allow(clippy::needless_pass_by_value)] @@ -1623,6 +1631,473 @@ pub extern "C" fn rs_kernel_check_anon( build_anon_result_array(&addrs_for_return, &results) } +/// Shared anon-mode seed context: the anon-view env, its whole-env work +/// items, and each requested name resolved to its covering work item +/// (standalone → itself; a mutual-block member → the whole block, checked +/// atomically). +struct AnonSeedContext { + env: Arc, + kernel_work: Vec, + /// One `(displayed name, work-item index)` per requested name, in request + /// order. Two names may share a work item (same mutual block). + seeds: Vec<(String, usize)>, +} + +/// Load `path` once and resolve `names` for anon-mode seeded checking: +/// displayed names → addresses via the full view's `named` metadata (dropped +/// right after), then the anon view + whole-env work items. `label` prefixes +/// diagnostics and error messages. +fn load_anon_seed_context( + label: &str, + path: &str, + names: &[String], +) -> Result { + use ix_kernel::anon_work::{block_of_addr, work_block_addr}; + + let t0 = Instant::now(); + let bytes = std::fs::read(path) + .map_err(|e| format!("{label}: failed to read {path}: {e}"))?; + + let resolved: Vec
= { + let mut slice: &[u8] = &bytes; + let full = IxonEnv::get(&mut slice) + .map_err(|e| format!("{label}: failed to deserialize {path}: {e}"))?; + let by_name: FxHashMap = full + .named + .iter() + .map(|e| (e.key().to_string(), e.value().addr.clone())) + .collect(); + let mut addrs = Vec::with_capacity(names.len()); + let mut missing: Vec<&str> = Vec::new(); + for n in names { + match by_name.get(n) { + Some(a) => addrs.push(a.clone()), + None => missing.push(n), + } + } + if !missing.is_empty() { + return Err(format!( + "{label}: no constant(s) named [{}] in {path}", + missing.join(", ") + )); + } + addrs + }; + eprintln!( + "[{label}] resolve: {:>8.1?} ({} name(s))", + t0.elapsed(), + resolved.len() + ); + + let t1 = Instant::now(); + let mut slice: &[u8] = &bytes; + let ixon_env = IxonEnv::get_anon(&mut slice).map_err(|e| { + format!("{label}: failed to deserialize (anon) {path}: {e}") + })?; + drop(bytes); + eprintln!( + "[{label}] anon parse: {:>8.1?} ({} consts)", + t1.elapsed(), + ixon_env.const_count(), + ); + + let t2 = Instant::now(); + let kernel_work = ix_kernel::anon_work::build_anon_work(&ixon_env) + .map_err(|e| format!("{label}: build_anon_work: {e}"))?; + let by_block: FxHashMap = kernel_work + .iter() + .enumerate() + .map(|(i, w)| (work_block_addr(&ixon_env, w), i)) + .collect(); + let mut seeds: Vec<(String, usize)> = Vec::with_capacity(names.len()); + for (n, addr) in names.iter().zip(&resolved) { + let block = block_of_addr(&ixon_env, addr); + match by_block.get(&block) { + Some(i) => seeds.push((n.clone(), *i)), + None => { + return Err(format!( + "{label}: no work item covers {} (block {})", + addr.hex(), + block.hex() + )); + }, + } + } + eprintln!( + "[{label}] build work: {:>8.1?} ({} item(s))", + t2.elapsed(), + kernel_work.len(), + ); + Ok(AnonSeedContext { env: Arc::new(ixon_env), kernel_work, seeds }) +} + +/// Work items inside the seeds' full dependency closure — the same set a +/// zkVM guest enumerates from its closure sub-env (`build_sub_env` + +/// `build_anon_work`), computed directly from `closure_addrs` without +/// serializing a sub-env. +fn closure_work_items( + ctx: &AnonSeedContext, + seed_items: &[usize], +) -> Vec { + use ix_kernel::anon_work::{AnonWorkItem as KItem, closure_addrs}; + + let roots: Vec
= seed_items + .iter() + .flat_map(|&i| ctx.kernel_work[i].proven_targets()) + .collect(); + let closure = closure_addrs(&ctx.env, &roots); + ctx + .kernel_work + .iter() + .filter(|w| match w { + KItem::Standalone { addr } => closure.contains(addr), + KItem::Block { block_addr, .. } => closure.contains(block_addr), + }) + .cloned() + .collect() +} + +/// FFI: anon-mode type-check of named constants with (by default) their full +/// dependency closures — the same mode and scope as the zkVM hosts' `--consts` +/// execute path, so an out-of-circuit run is directly comparable to the +/// in-circuit one. `skip_deps` restricts the check to each name's own work +/// item (subject-only; deps trusted), mirroring `zisk-host --skip-deps`. +/// +/// Names resolve through the env's `named` metadata by displayed form (the +/// same string match the zkVM hosts use), then the metadata is dropped and +/// the check runs on the anon view — the kernel never sees names. A member +/// of a mutual block selects the whole block's work item (blocks check +/// atomically). Multiple names union their closures into one check set. +/// +/// `json_path` (empty = off) additionally emits per-name benchmark results +/// rows. With it set, each name is checked as its own closure run — the +/// per-constant scope the zkVM hosts measure — instead of one union set: +/// the env still loads once, and each row's `check-time`/`peak-rss` window +/// wraps only that name's closure selection + check. Rows are flushed per +/// name, so a killed run keeps completed rows; a rejected name gets +/// `status: rejected` and the loop continues. +/// +/// Returns `(hex_address, Option CheckError)` pairs, one per checked target, +/// exactly like `rs_kernel_check_anon` (concatenated across names when +/// per-name rows are on — shared deps then appear once per name that +/// re-checked them). +#[unsafe(no_mangle)] +pub extern "C" fn rs_kernel_check_anon_consts( + env_path: LeanString>, + names: LeanArray>, + skip_deps: LeanBool>, + quiet: LeanBool>, + fail_out: LeanString>, + json_path: LeanString>, +) -> LeanIOResult { + let total_start = Instant::now(); + let quiet = quiet.to_bool(); + let skip_deps = skip_deps.to_bool(); + let path = env_path.to_string(); + let fail_out_path = fail_out.to_string(); + let fail_out_path = + if fail_out_path.is_empty() { None } else { Some(fail_out_path) }; + let json_path = json_path.to_string(); + let names_vec: Vec = names.map(|obj| obj.as_string().to_string()); + if names_vec.is_empty() { + return LeanIOResult::error_string( + "rs_kernel_check_anon_consts: no constant names given", + ); + } + if !json_path.is_empty() && fail_out_path.is_some() { + return LeanIOResult::error_string( + "rs_kernel_check_anon_consts: --fail-out is not supported with per-name rows (--json)", + ); + } + + let ctx = match load_anon_seed_context( + "rs_kernel_check_anon_consts", + &path, + &names_vec, + ) { + Ok(ctx) => ctx, + Err(e) => return LeanIOResult::error_string(&e), + }; + + // ---- Per-name rows mode: one independent closure run per name (the + // zkVM hosts' per-constant scope), timing only each name's closure + // selection + check. Rows flush per name; a rejected name records + // `status: rejected` and the loop continues. + if !json_path.is_empty() { + let json_file = std::path::PathBuf::from(&json_path); + let mut all_addrs: Vec
= Vec::new(); + let mut all_results = Vec::new(); + for (name, item_idx) in &ctx.seeds { + tracing_texray::rss_sampler::reset_peak_tree_rss(); + let t = Instant::now(); + let selected = if skip_deps { + vec![ctx.kernel_work[*item_idx].clone()] + } else { + closure_work_items(&ctx, &[*item_idx]) + }; + let (work, addrs) = index_anon_work(selected); + let total = addrs.len(); + let results = match run_anon_checks_parallel( + Arc::clone(&ctx.env), + work, + addrs.clone(), + quiet, + None, + ) { + Ok(r) => r, + Err(msg) => { + return build_uniform_error(total, &format!("[thread] {msg}")); + }, + }; + let failed = results.iter().filter(|r| r.is_err()).count(); + let secs = t.elapsed().as_secs_f64(); + let tput = if secs > 0.0 { + f64::from(u32::try_from(total).unwrap_or(u32::MAX)) / secs + } else { + 0.0 + }; + let status = if failed > 0 { + ix_bench::Status::Rejected + } else { + ix_bench::Status::Ok + }; + eprintln!( + "[rs_kernel_check_anon_consts] {name}: {}/{total} passed ({secs:.3}s)", + total - failed, + ); + if let Err(e) = ix_bench::write_row( + &json_file, + name, + status, + serde_json::json!({ + "constants": total, + "check-time": (secs * 1e6).round() / 1e6, + "throughput": (tput * 100.0).round() / 100.0, + "peak-rss": ix_bench::peak_rss_bytes(), + }), + ) { + return LeanIOResult::error_string(&format!( + "rs_kernel_check_anon_consts: write {}: {e}", + json_file.display() + )); + } + all_addrs.extend(addrs); + all_results.extend(results); + } + eprintln!( + "[rs_kernel_check_anon_consts] rows total: {:>8.1?} ({} name(s))", + total_start.elapsed(), + ctx.seeds.len(), + ); + return build_anon_result_array(&all_addrs, &all_results); + } + + let mut seed_items: Vec = Vec::new(); + for &(_, i) in &ctx.seeds { + if !seed_items.contains(&i) { + seed_items.push(i); + } + } + + // The check set. Subject-only: the seeds' own items. Full-closure: every + // work item inside the seeds' dependency closure (union across seeds). + let selected = if skip_deps { + seed_items.iter().map(|&i| ctx.kernel_work[i].clone()).collect() + } else { + closure_work_items(&ctx, &seed_items) + }; + let (work, addrs) = index_anon_work(selected); + eprintln!( + "[rs_kernel_check_anon_consts] check set: {} items, {} targets, {}", + work.len(), + addrs.len(), + if skip_deps { "subject-only" } else { "full-closure" }, + ); + + let failure_log: Option> = match fail_out_path.as_deref() { + None => None, + Some(out_path) => match FailureLog::open(out_path, &path, addrs.len()) { + Ok(log) => { + eprintln!( + "[rs_kernel_check_anon_consts] streaming failures to {out_path}" + ); + Some(Arc::new(log)) + }, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_kernel_check_anon_consts: failed to open fail-out file {out_path}: {e}" + )); + }, + }, + }; + + let total = addrs.len(); + let addrs_for_return = addrs.clone(); + let t3 = Instant::now(); + let results = match run_anon_checks_parallel( + ctx.env, + work, + addrs, + quiet, + failure_log.clone(), + ) { + Ok(r) => r, + Err(msg) => { + if let Some(log) = failure_log.as_ref() { + log.finalize(); + } + return build_uniform_error(total, &format!("[thread] {msg}")); + }, + }; + + let passed = results.iter().filter(|r| r.is_ok()).count(); + let failed = results.iter().filter(|r| r.is_err()).count(); + eprintln!( + "[rs_kernel_check_anon_consts] {passed}/{total} passed, {failed} failed ({:.1?})", + t3.elapsed() + ); + eprintln!( + "[rs_kernel_check_anon_consts] total: {:>8.1?}", + total_start.elapsed() + ); + if let Some(log) = failure_log.as_ref() { + log.finalize(); + eprintln!( + "[rs_kernel_check_anon_consts] streamed {} failure(s) to fail-out", + log.count() + ); + } + + build_anon_result_array(&addrs_for_return, &results) +} + +/// FFI: extract the named constants' dependency closure from a serialized +/// env into a standalone `.ixe` — genuine constant bytes, blobs, and +/// reducibility hints (via the anon view), plus every closure constant's +/// Named entry (via the full view) so names still resolve downstream — all +/// without recompiling from source. Each name extracts its covering work +/// item (a mutual-block member pulls the whole block). +#[unsafe(no_mangle)] +pub extern "C" fn rs_env_extract( + env_path: LeanString>, + names: LeanArray>, + out_path: LeanString>, + quiet: LeanBool>, +) -> LeanIOResult { + use ix_kernel::anon_work::{ + block_of_addr, build_anon_work, build_sub_env_named, work_block_addr, + }; + + let quiet = quiet.to_bool(); + let path = env_path.to_string(); + let out = out_path.to_string(); + let names_vec: Vec = names.map(|obj| obj.as_string().to_string()); + if names_vec.is_empty() { + return LeanIOResult::error_string( + "rs_env_extract: no constant names given", + ); + } + + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_env_extract: failed to read {path}: {e}" + )); + }, + }; + let mut slice: &[u8] = &bytes; + let full = match IxonEnv::get(&mut slice) { + Ok(env) => env, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_env_extract: failed to deserialize {path}: {e}" + )); + }, + }; + // Resolve displayed names → addresses through the full env's `named` + // metadata (the anon view discards it). + let by_name: FxHashMap = full + .named + .iter() + .map(|e| (e.key().to_string(), e.value().addr.clone())) + .collect(); + let mut resolved: Vec
= Vec::with_capacity(names_vec.len()); + let mut missing: Vec<&str> = Vec::new(); + for n in &names_vec { + match by_name.get(n.as_str()) { + Some(a) => resolved.push(a.clone()), + None => missing.push(n), + } + } + if !missing.is_empty() { + return LeanIOResult::error_string(&format!( + "rs_env_extract: no constant(s) named [{}] in {path}", + missing.join(", ") + )); + } + + let mut slice: &[u8] = &bytes; + let anon = match IxonEnv::get_anon(&mut slice) { + Ok(env) => env, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_env_extract: failed to deserialize (anon) {path}: {e}" + )); + }, + }; + + // Roots: each name's covering work item's proven targets (standalone → + // itself; a mutual-block member → every sibling, checked atomically). + let work = match build_anon_work(&anon) { + Ok(work) => work, + Err(e) => { + return LeanIOResult::error_string(&format!( + "rs_env_extract: build_anon_work: {e}" + )); + }, + }; + let by_block: FxHashMap = work + .iter() + .enumerate() + .map(|(i, w)| (work_block_addr(&anon, w), i)) + .collect(); + let mut roots: Vec
= Vec::new(); + for addr in &resolved { + let block = block_of_addr(&anon, addr); + match by_block.get(&block) { + Some(&i) => roots.extend(work[i].proven_targets()), + None => { + return LeanIOResult::error_string(&format!( + "rs_env_extract: no work item covers block {}…", + &block.hex()[..16] + )); + }, + } + } + + let sub_bytes = match build_sub_env_named(&anon, &full, &roots) { + Ok(b) => b, + Err(e) => { + return LeanIOResult::error_string(&format!("rs_env_extract: {e}")); + }, + }; + if let Err(e) = std::fs::write(&out, &sub_bytes) { + return LeanIOResult::error_string(&format!( + "rs_env_extract: failed to write {out}: {e}" + )); + } + if !quiet { + eprintln!( + "[rs_env_extract] {} name(s) → {} ({} bytes) from {path}", + names_vec.len(), + out, + sub_bytes.len(), + ); + } + LeanIOResult::ok(LeanOwned::box_usize(0)) +} + // =========================================================================== // Sharding profiler: run the anon kernel out of circuit over a `.ixe`, // recording per-block heartbeats + the delta-unfold graph into a `.ixprof`. diff --git a/crates/ffi/src/texray.rs b/crates/ffi/src/texray.rs index 77d91b5bb..ae8add6d2 100644 --- a/crates/ffi/src/texray.rs +++ b/crates/ffi/src/texray.rs @@ -59,3 +59,47 @@ extern "C" fn rs_texray_init( let _ = Registry::default().with(layer.with_filter(filter)).try_init(); LeanIOResult::ok(LeanOwned::box_usize(0)) } + +/// Start tracing-texray's process-tree RSS sampler (idempotent). `interval_ms` +/// is the sampling period in milliseconds. Runs on a background daemon thread; +/// [`rs_texray_peak_tree_rss_bytes`] reads back the high-water mark. Captures +/// child-process memory (e.g. a zkVM host's helper processes) that a bare +/// `/proc/self/status` read misses. +#[unsafe(no_mangle)] +extern "C" fn rs_texray_start_sampler( + interval_ms: u64, +) -> LeanIOResult { + tracing_texray::rss_sampler::start(std::time::Duration::from_millis( + interval_ms, + )); + LeanIOResult::ok(LeanOwned::box_usize(0)) +} + +/// Peak resident-set size (bytes) across this process and its children per the +/// tree sampler. `0` until [`rs_texray_start_sampler`] has run or off Linux. +#[unsafe(no_mangle)] +extern "C" fn rs_texray_peak_tree_rss_bytes() -> LeanIOResult { + let bytes = tracing_texray::rss_sampler::peak_tree_rss_bytes(); + LeanIOResult::ok(LeanOwned::box_u64(bytes)) +} + +/// Reset the tree sampler's high-water mark, opening a new measurement +/// window — the per-item peaks a benchmark loop needs (the zkVM hosts do +/// the same between shards). +#[unsafe(no_mangle)] +extern "C" fn rs_texray_reset_peak_tree_rss() -> LeanIOResult { + tracing_texray::rss_sampler::reset_peak_tree_rss(); + LeanIOResult::ok(LeanOwned::box_usize(0)) +} + +/// Direct tracing-texray's per-span timing sink to `path` (one +/// `{"span","seconds"}` JSON line per closed examined span). Combine with a +/// `streaming`/examined subscriber so the prover's `aiur/*` + `stark/*` spans +/// are recorded for the CI drill-down. +#[unsafe(no_mangle)] +extern "C" fn rs_texray_json_sink( + path: LeanString>, +) -> LeanIOResult { + let _ = tracing_texray::json_sink::to_file(&path.to_string()); + LeanIOResult::ok(LeanOwned::box_usize(0)) +} diff --git a/crates/kernel/src/anon_work.rs b/crates/kernel/src/anon_work.rs index be0aae24a..1f5be74dd 100644 --- a/crates/kernel/src/anon_work.rs +++ b/crates/kernel/src/anon_work.rs @@ -279,6 +279,17 @@ pub fn build_sub_env( source: &IxonEnv, roots: &[Address], ) -> Result, String> { + let sub = sub_env_of(source, roots); + let mut buf = Vec::new(); + sub.put(&mut buf).map_err(|e| format!("sub-env serialize: {e}"))?; + Ok(buf) +} + +/// The in-memory closure sub-env behind [`build_sub_env`]: copy the BFS +/// dependency closure of `roots` (genuine bytes + blobs + reducibility +/// hints), no Named section. +#[cfg(not(target_arch = "riscv64"))] +fn sub_env_of(source: &IxonEnv, roots: &[Address]) -> IxonEnv { let closure = closure_addrs(source, roots); let mut sub = IxonEnv::new(); for addr in &closure { @@ -296,6 +307,55 @@ pub fn build_sub_env( sub.anon_hints.insert(addr.clone(), *h); } } + sub +} + +/// [`build_sub_env`] plus a name→address entry for every closure constant +/// named in the FULL view of the same env — a standalone `.ixe` whose names +/// still resolve (for `--consts`-style tools), extracted without recompiling +/// from source. +/// +/// METADATA IS DROPPED: each copied entry is `Named::with_addr` (empty +/// `ConstantMeta`), because real metadata references name addresses +/// throughout its tree and carrying it would require the full env's +/// hash-consed name index. The extract serves the ANON pipeline +/// (`check-rs --anon`, the zkVM hosts, `ix profile`/`ix shard`, +/// `bench-typecheck`), where metadata is never read and reducibility hints +/// travel in the `anon_hints` section instead. Meta-mode tools need the +/// source env. +#[cfg(not(target_arch = "riscv64"))] +pub fn build_sub_env_named( + source: &IxonEnv, + full: &IxonEnv, + roots: &[Address], +) -> Result, String> { + use ix_common::env::NameData; + + let sub = sub_env_of(source, roots); + // The Named section serializes keys as name HASHES resolved through the + // names section, so each key's component chain must be interned too. + fn intern_chain(sub: &IxonEnv, name: &ix_common::env::Name) { + let addr = Address::from_blake3_hash(*name.get_hash()); + if sub.get_name(&addr).is_some() { + return; + } + match name.as_data() { + NameData::Anonymous(_) => {}, + NameData::Str(parent, _, _) | NameData::Num(parent, _, _) => { + intern_chain(sub, parent); + }, + } + sub.store_name(addr, name.clone()); + } + for e in full.named.iter() { + if sub.get_const(&e.value().addr).is_some() { + intern_chain(&sub, e.key()); + sub.register_name( + e.key().clone(), + ixon::env::Named::with_addr(e.value().addr.clone()), + ); + } + } let mut buf = Vec::new(); sub.put(&mut buf).map_err(|e| format!("sub-env serialize: {e}"))?; Ok(buf) diff --git a/crates/kernel/src/shard.rs b/crates/kernel/src/shard.rs index 8e312f6c9..e594ff09f 100644 --- a/crates/kernel/src/shard.rs +++ b/crates/kernel/src/shard.rs @@ -1653,8 +1653,8 @@ pub fn block_step_cost(b: &BlockEntry) -> u64 { /// size shards straight from `MemTotal` without ever picking a budget. Inverts /// the measured single-leaf prover model on this setup /// (`peak_RAM_GiB ≈ 50 + 33 × steps_billions`, measured by a guarded 7-shard GPU -/// prove sweep over 0.27–3.79e9-step Init shards, R²=0.99, `--max-witness-stored -/// 5`) at [`RAM_USABLE_FRAC`] of RAM (reserving the rest for the OS, cross-shard +/// prove sweep over 0.27–3.79e9-step Init shards, R²=0.99) at +/// [`RAM_USABLE_FRAC`] of RAM (reserving the rest for the OS, cross-shard /// re-ingress, and run-to-run variance). Returns 0 when the box can't even hold the ~50 GiB /// prover base (nothing will prove). Approximate by design — pair with /// [`partition_for_cycle_cap`] to get N. The earlier `45 + 32` model was @@ -1662,8 +1662,7 @@ pub fn block_step_cost(b: &BlockEntry) -> u64 { /// target actually used ~225 GB). /// Measured prover-RAM model (the single source of truth, used by both /// [`cycle_cap_for_ram`] and [`ram_gib_for_steps`]): peak host RAM ≈ -/// `RAM_BASE_GIB + RAM_GIB_PER_BCYCLE × steps_billions` (at -/// `--max-witness-stored 5`). +/// `RAM_BASE_GIB + RAM_GIB_PER_BCYCLE × steps_billions`. pub const RAM_BASE_GIB: f64 = 50.0; pub const RAM_GIB_PER_BCYCLE: f64 = 33.0; /// Usable fraction of a host-RAM budget (headroom for OS + variance) — applied @@ -1685,8 +1684,8 @@ pub fn cycle_cap_for_ram(ram_gb: f64) -> u64 { } /// Measured single-GPU **leaf prove time**: `≈ PROVE_SETUP_SECS + -/// PROVE_SECS_PER_BCYCLE × steps_billions` per shard (RTX PRO 6000, -/// `--max-witness-stored 5`). Aggregation adds a smaller per-fold term this model +/// PROVE_SECS_PER_BCYCLE × steps_billions` per shard (RTX PRO 6000). +/// Aggregation adds a smaller per-fold term this model /// omits — minutes next to hours of leaf proving at large shard counts. pub const PROVE_SETUP_SECS: f64 = 54.0; pub const PROVE_SECS_PER_BCYCLE: f64 = 158.0; diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 000000000..7902b9315 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,194 @@ +# Benchmarking + +One orchestrator — `ix bench` — runs every benchmark cell, locally and in CI. +A **cell** is `(backend, env, mode)`, e.g. `zisk-InitStd-execute`. CI is a +thin wrapper: the same `ix bench run` you type in a terminal is what both +workflows execute, so every CI number is reproducible on your machine. + +- **`!benchmark` PR comment** (`.github/workflows/bench-pr.yml`) — on demand, + posts a **main-vs-PR** comparison table on the pull request. main's numbers + come from bencher.dev (`ix bench fetch-main`); the PR side is measured + fresh. When bencher can't supply the base SHA (not ingested yet, or the PR + adds new constants), the workflow measures the gap on a base checkout. +- **Bencher.dev** (`.github/workflows/bench-main.yml`) — on every push to + `main`, tracks each measure over time at (project + `ix`), the canonical store the PR path reads from. + +## The row contract + +Every measured tool reports through one shape — the **benchmark results JSON** — +and one exit-code convention (Rust: `crates/bench`; Lean: +`Ix/Benchmark/Results.lean`): + +```json +{ "": { "status": "ok", "": 123, "phase:": 1.5 } } +``` + +- Rows are flushed after every name, so a killed run keeps its completed rows. +- `status` is `ok` or `rejected` (written by the tool) or `oom` (merged in by + `ix bench run` after an abnormal exit — a process never observes its own + OOM kill). An `oom` row keeps whatever metrics landed before the kill. +- Exit codes: `0` all ok · `2` usage error · `3` the kernel **rejected** a + constant (its row is on disk) · anything else is an infrastructure failure. + +There is no output scraping anywhere: no marker lines, no log grepping, no +sentinel-key jq. State flows through rows and exit codes only. A rejected or +OOM'd constant never reaches bencher (`ix bench bmf` drops non-`ok` rows), +and `ix bench run` exits nonzero unless **every selected name produced a +row** — an empty or quietly-partial cell can't be green. + +## `ix bench` subcommands + +| subcommand | job | +|---|---| +| `run` | run one cell: select names, ensure the `.ixe`, spawn the tool under the RAM watchdog (one process per constant on aiur/zkVM), fold each spawn's span window into its row, gate on the rows | +| `shard` | pre-cut the closure-shard artifacts for the env's heavy-tier constants (`ix shard extract` → `ix profile` → `ix shard`) | +| `compare` | two rows files → Markdown main-vs-PR table (thresholds, ratios, OOM/❌ rows, per-constant phase drop-downs) | +| `bmf` | rows → Bencher Metric Format (non-`ok` rows dropped) | +| `fetch-main` | pull a base SHA's rows from bencher.dev (exit 3 = transient, fall back to a local base run; exit 2 = config error, fail loudly) | +| `report` | assemble per-cell tables into one Markdown report (CI posts it as the PR comment) | +| `ci matrix` | emit the workflows' job matrices from the registry (CI adapter) | +| `ci parse` | `!benchmark` comment → job matrix (CI adapter; `--comment` pre-flights a comment locally) | + +### Local usage + +```shell +# Run the ooc cell over InitStd's primary constants: +ix bench run --backend ooc --env InitStd + +# Change something, run again, and diff against your previous run +# (runs save baselines under .lake/benches/{,.prev}.json — the same +# BENCH_OUTPUT_DIR root the Ix.Benchmark framework writes to): +ix bench run --backend ooc --env InitStd --ixe InitStd.ixe +ix bench compare --backend ooc --env InitStd + +# One constant through aiur — the fast Phase-1 signal, then the full prove +# (cap the watchdog to what your machine can spare): +ix bench run --backend aiur --env InitStd --mode execute \ + --consts Nat.add_comm --ixe InitStd.ixe --ceiling-gb 50 +ix bench run --backend aiur --env InitStd --mode prove \ + --consts Nat.add_comm --ixe InitStd.ixe --ceiling-gb 50 + +# Compare a local run against main's numbers straight from bencher.dev +# (no token needed; --consts filters to your constants — the testbed +# holds every benched env's): +ix bench fetch-main --sha $(git merge-base origin/main HEAD) \ + --backend aiur --mode prove --consts Nat.add_comm --out main.json +ix bench compare --backend aiur --env InitStd --mode prove \ + --main main.json --pr .lake/benches/aiur-InitStd-prove.json +``` + +`--repo ` points the run at another checkout: the *measured* tools +resolve from `/.lake/build/bin` first, so one `ix` can drive a base and +a PR tree and compare them — exactly what the PR workflow does. + +## Backends + +| backend | what it measures | tool | +|---|---|---| +| `aiur` | Aiur STARK check, one bench-main cell per mode on its own testbed: prove — the real-workload simulation (prove-time, proof-size, verify-time, peak-rss, plus fft-cost / execute-time from its own Phase 1) — and execute, the fast Phase-1-only signal (fft-cost, execute-time, throughput, peak-rss). `!benchmark aiur [execute]` picks the mode | `bench-typecheck` | +| `zisk` | ZisK VM execute: cycles, execute-time, throughput, peak-rss | `zisk-host` | +| `sp1` | SP1 VM execute (currently disabled in the registry) | `sp1-host` | +| `ooc` | out-of-circuit Rust kernel: whole-env row + one full-closure row per primary (`check-time` wraps only the check — the env loads once, outside every row's timed window) | `ix check-rs --json` | +| `compile` | `ix compile .lean → .ixe`: compile-time, file-size, constants, throughput | `ix compile --json` | + +All tools take the same `--consts`/`--consts-file` grammar and emit the same +rows. The ooc and zkVM cells share per-constant **full-closure** scope, so +their delta isolates in-circuit vs out-of-circuit overhead. + +With `--texray`, tools write per-phase span timings (`aiur/prove_ixvm`, +`aiur/witness`, `stark/*`, `zisk/execute`, …) to `.spans`. The +per-constant backends run **one process per constant**, so each spawn's +window belongs wholly to its constant: `ix bench run` folds it into the +row as flat `phase:` fields, which flow to bencher as independent +measures (witness gen, stage commits, quotient, … each get a trend line) +and render in the PR comment as a collapsible per-constant drill-down. + +## RAM: watchdog, OOM rows, sharding + +`ix bench run` wraps each tool in `.github/scripts/watchdog.sh +`: a thin wrapper over `systemd-run --user --scope` that runs the +tool under a cgroup-v2 `memory.max` cap with swap disabled. The kernel +OOM-kills at the ceiling — SIGKILL, exit 137 — with no sampler to race +and nothing to sum: the cgroup charges the whole tree's resident memory, +locked shared segments included, while an allocator's cached virtual +reservations don't count. A killed per-constant process gets +its row marked `status: oom` +(keeping whatever was flushed, spans included) and the loop continues — one +constant's death costs one constant. A kill that lands *after* the row +carries the mode's completion metric hit teardown (the prover releasing +tens of GB right after the final write), so the finished row stays `ok`. +ooc and compile run as single processes instead — their checks never +approach the ceiling, and a kill there means missing rows and a red cell. +There are **no per-constant timeouts**; the job-level `timeout-minutes` is +the only clock. + +Heavy-tier zisk constants (whose single-leaf closure would blow the runner's +RAM) run as their closure-shard partition instead: `ix shard extract` → +`ix profile` → `ix shard` cut a manifest, and one `--shard-plan` host run +executes the shards sequentially, emitting the constant's row with per-shard +breakdowns. bench-main's compile job pre-cuts these artifacts +(`ix bench shard`) and ships them via cache. + +## Registry and constant set + +- **`Benchmarks/Vectors.csv`** — the curated constants: one row per + `(name, env, tier[, shard_target[, primary]])`. `tier: heavy` marks + constants whose full prove is expected to OOM (they still run; the row + records it). `primary: 1` is the default `!benchmark` subset. +- **The registry** (`envSpecs`/`backendSpecs` in `Ix/Cli/BenchCmd.lean`) — + everything else: env modules, backends (disabled reason, default mode, + bencher testbeds, compare columns). Typed Lean data with one owner: the + workflows never read it directly — `ix bench ci matrix` serves the job + matrices and `ix bench ci parse` the `!benchmark` cells, both post-build. + (`bencher-thresholds-reset.yml` keeps a static workload list with a sync + note.) CI-only data stays out of it: the runner name lives with the `ci` + adapters. The watchdog ceiling defaults to the machine's RAM minus + 15 GB — above the largest legitimate workload (Mathlib `ix compile` + peaks ~100 GB) wherever the run happens (`--ceiling-gb` overrides). + +## `!benchmark` grammar + +``` +!benchmark ([aiur] [zisk] [sp1] [ooc] [compile] | all) [execute] +BENCH_ENVS=InitStd,Mathlib # default InitStd (case-insensitive) +BENCH_FULL=1 # full curated set, not just primary +BENCH_SHARD=1 # only the multi-shard target constants +RUST_LOG=info # passthrough env (allowlist: RUST_LOG, + # WITHOUT_VK_VERIFICATION, RUSTFLAGS) +``` + +Parsed by `ix bench ci parse` in the PR build job, right after the `ix` +binary exists — the registry lives in Lean, so nothing pre-build reads it +(and no Python remains). Mode defaults per backend from the registry; the +bare `execute` token flips `aiur` to Phase-1 only. + +## CI shape + +**bench-main.yml**: `build` (compile `ix` + `bench-typecheck` once, cache by +SHA) → `plan` (`ix bench ci matrix` → job matrices) + `compile` (per env: +`ix bench run --backend compile`, cache the `.ixe` + pre-cut zisk shards) → +`aiur` (execute + prove cells) / `zkvm-execute` / `ooc-check` (each: restore caches, one +`ix bench run … --ixe`, `ix bench bmf`, upload via +`.github/actions/bencher-track`). A kernel rejection exits 3 and reddens the +run step while the clean rows still upload. + +**bench-pr.yml**: `setup` (authorize the comment, resolve base/head SHAs) → +`build` (PR binaries, cached by head SHA; ends with `ix bench ci parse` — +the matrix can only exist once `ix` does) → `compile` (one measured +`ix compile` per env: publishes the `.ixe` the prover cells restore AND +the row the compile cell reuses as its PR side) → `benchmark` matrix (per cell: +PR-side `ix bench run`; `ix bench fetch-main` for main's numbers, with a +targeted base-checkout run covering only what bencher lacked; +`ix bench compare` → table artifact) → `assemble` (`ix bench report` builds +the comment body, unprivileged) → `comment` (posts it — the only job with a +write token, running no PR code). + +## Not yet covered + +- **zkVM prove** — the hosts prove, but CI has no GPU runner; cells are + execute-only. +- **sp1** — disabled in the registry (execute too slow per push); + re-enable it there and it returns to the matrices and the parser. +- **Non-`main` base branches** — `fetch-main` queries `branch=main`; a PR + against another base always pays the local base run. diff --git a/docs/zisk-cycle-cost-model.md b/docs/zisk-cycle-cost-model.md index 04e7a1927..3c961ccf1 100644 --- a/docs/zisk-cycle-cost-model.md +++ b/docs/zisk-cycle-cost-model.md @@ -68,7 +68,7 @@ objective for a cost spanning 50M–9B cycles. **Full-closure typecheck** — a self-contained closure: a small program, or a constant checked with all its dependencies. Calibrated **cross-library** on n=76: 12 small programs + 64 diverse constants checked full-closure via -`--constant`, spanning **Init (51), Std (3), and Mathlib (10)**: +`--consts`, spanning **Init (51), Std (3), and Mathlib (10)**: ``` cycles ≈ 0.68M + 96,989·hb + 4,151·subst R² 0.987, MAPE 6% ``` @@ -99,23 +99,23 @@ cycles ≈ 0.39M + 6,740·block_bytes + 14·subenv_bytes + 4,070·subst R² 0. Reserved for constants too expensive to full-closure-check that can't be sharded (Finding 4). -**Measuring one constant.** `zisk-host --execute --ixe initstd.ixe --constant +**Measuring one constant.** `zisk-host --execute --ixe initstd.ixe --consts ` resolves the name, builds just its closure sub-env (deps lazily faulted in — no separate `.ixe`, no whole-env ingress), and checks it. Full-closure by default (`Ix.Claim.check addr none`); add `--skip-deps` for subject-only. Same -flags as the Aiur `bench-typecheck`. +`--consts` / `--skip-deps` vocabulary as `ix check`, `sp1-host`, and the Aiur +`bench-typecheck`. --- ## Prover RAM and prove time -Measured proving 7 Init shards on GPU at `--max-witness-stored 5`, STEPS -0.27–3.79e9: +Measured proving 7 Init shards on GPU, STEPS 0.27–3.79e9: ``` peak host RAM (GiB) ≈ 50 + 33·STEPS_billions R² 0.99 GPU prove time (s) ≈ 54 + 158·STEPS_billions R² 0.98 (~6.3M steps/s) ``` -RAM scales with `--max-witness-stored N` (this is N=5). Inverting the RAM model +Inverting the RAM model at `RAM_USABLE_FRAC` = 0.85 gives a safe per-shard cap of **~3.6e9 steps** for a 200 GiB target (`shard.rs:cycle_cap_for_ram`). @@ -148,12 +148,18 @@ side. the cap still executes under it. Each leaf pays the ~180M fixed floor and adds an aggregation node, so cheap constants are batched rather than proven one at a time. (`--shards N` still does balanced bisection, for manual control.) -4. **A few constants can't be full-closure-proven on a 250 GiB box and aren't - shardable** (single atomic constants): `Vector/Array.extract_append._proof_1`, - the `instRxcHasSize_eq` family. The escape hatch is `--skip-deps`: - `Vector.extract_append` is the 143 GiB OOM case under full-closure but checks - subject-only in 74M cycles. The planner flags these via - `infeasible_atomic_floor`. +4. **A few constants can't be proven as a single full-closure leaf on a 250 GiB + box**: `Vector/Array.extract_append._proof_1`, the `instRxcHasSize_eq` family. + This is a per-leaf ingress/RAM ceiling on the `--consts ` (full-closure, + `Ix.Claim.check addr none`) mode — not a global unshardability. In + env-sharding mode (`--shard-plan`) these same constants are fine: their + subject checks fit in one work item and their deps are proved in other + shards, folded in through the assumptions root. The only work unit the + env-sharding planner truly can't split is a **mutual block** (`build_anon_work` + emits one item per Muts block, checked atomically). The escape hatch in + single-constant mode is `--skip-deps`: `Vector.extract_append` is the 143 GiB + OOM case under full-closure but checks subject-only in 74M cycles. The + planner flags these via `infeasible_atomic_floor`. 5. **The packing order comes from min-cut.** Whole-env profiling of Init/Std/**Mathlib** (mathlib = 631k blocks) shows Lean typecheck is uniformly reduction-dominated: own-bytes is only **2.6–7% of member cost** (mathlib @@ -171,7 +177,7 @@ side. Applying the cost model to every Init constant's native profile (51,003 blocks / 51,678 constants) vs a single-leaf cap: **~99.98% of Init typechecks on Zisk.** -Exactly **12 constants** exceed a single 250 GiB leaf (`--max-witness-stored 5`) +Exactly **12 constants** exceed a single 250 GiB leaf — all single atomic constants (un-shardable), `hb`-dominated. Listed by name (all are `_private` proof terms; private prefix elided): @@ -190,7 +196,7 @@ Exactly **12 constants** exceed a single 250 GiB leaf (`--max-witness-stored 5`) | `Array.extract_append_extract._proof_1_1` | 13,226 | 4.7e9 | | `Char.ofOrdinal_ordinal` | 14,136 | 4.5e9 | -Each has a workaround — lower `--max-witness-stored`, a bigger box, `--skip-deps` +Each has a workaround — a bigger box, `--skip-deps` (subject-only), or upstream proof restructuring — so 12 is an upper bound on truly-stuck constants. At a 200 GiB cap the list grows to 16. Estimate uses the planner's `block_step_cost` model (`162,339·hb + 4,276·subst + 652·bytes`, the @@ -222,7 +228,7 @@ Shards 630/634 are one atomic constant each (`Int*.instRxcHasSize_eq`): tiny `bytes`/`subst` but huge cycles, driven by `hb` (deep nat-range def-eq) — the "expensive atomic" case (Finding 4). -### Full-closure single constants — `--constant`, diverse shapes +### Full-closure single constants — `--consts`, diverse shapes One library constant each, checked full-closure (the constant and its whole dependency closure). The 35 Init constants below (over `initstd.ixe`) are shown @@ -289,7 +295,7 @@ lake exe ix shard env.ixprof --max-ram 256 # or --max-cycles C / --shards cargo run --release --bin zisk-host -- --execute --ixe \ [--shard-plan --only-shard K] cargo run --release --bin zisk-host -- --execute --ixe initstd.ixe \ - --constant "Nat.add_comm" [--skip-deps] # full-closure / subject-only + --consts "Nat.add_comm" [--skip-deps] # full-closure / subject-only # fits python3 ~/benchdata/prof/fit_xlib.py # full-closure model (76 pts, Init+Std+Mathlib) @@ -311,8 +317,7 @@ native profiling run and compile out on the zkvm target. 5–8%); the shard model is still n=12 and Init-derived, so the ~190M shard intercept is the term most likely to shift on Std/Mathlib — worth re-checking there. -- RAM/prove-time models are for one GPU at `--max-witness-stored 5`; both scale - with that setting. +- RAM/prove-time models are for one GPU. - Cycle counts are deterministic for a fixed guest ELF; the profiler counters compile out on the zkvm target, so the proven ELF is unaffected. - **Regimes have distinct coefficients and don't transfer.** diff --git a/sp1/Cargo.lock b/sp1/Cargo.lock index a5f9903d6..ff86c074b 100644 --- a/sp1/Cargo.lock +++ b/sp1/Cargo.lock @@ -1029,7 +1029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1247,6 +1247,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "304de19db7028420975a296ab0fcbbc8e69438c4ed254a1e41e2a7f37d5f0e0a" +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -1751,6 +1766,14 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ix-bench" +version = "0.1.0" +dependencies = [ + "serde_json", + "tracing-texray", +] + [[package]] name = "ix-common" version = "0.1.0" @@ -1894,6 +1917,19 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.12.5" @@ -2041,7 +2077,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2831,7 +2867,7 @@ dependencies = [ "once_cell", "socket2 0.6.3", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -3123,7 +3159,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3225,6 +3261,12 @@ dependencies = [ "sdd", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4069,12 +4111,15 @@ dependencies = [ "bincode", "clap", "human-repr", + "ix-bench", "ix-common", "ix-kernel", "ixon", + "serde_json", "sp1-build", "sp1-sdk", "tokio", + "tracing-texray", ] [[package]] @@ -4633,7 +4678,17 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.59.0", ] [[package]] @@ -5062,6 +5117,18 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-texray" +version = "0.2.0" +source = "git+https://github.com/argumentcomputer/tracing-texray?rev=bd4faa08a4fa4edb46bde393b4d20c6bd49591d0#bd4faa08a4fa4edb46bde393b4d20c6bd49591d0" +dependencies = [ + "loom", + "parking_lot", + "terminal_size", + "tracing", + "tracing-subscriber", +] + [[package]] name = "transpose" version = "0.2.3" @@ -5479,15 +5546,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -5521,30 +5579,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5557,12 +5598,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5575,12 +5610,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5593,24 +5622,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5623,12 +5640,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5641,12 +5652,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5659,12 +5664,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -5677,12 +5676,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.5.40" diff --git a/sp1/host/Cargo.toml b/sp1/host/Cargo.toml index f45c1d3c5..4212bc48b 100644 --- a/sp1/host/Cargo.toml +++ b/sp1/host/Cargo.toml @@ -10,6 +10,9 @@ path = "src/main.rs" [dependencies] ixon = { path = "../../crates/ixon" } +# Benchmark results-row contract (status enum, row writer, exit codes) shared +# with zisk-host and `ix bench`. +ix-bench = { path = "../../crates/bench" } ix-common = { path = "../../crates/common" } # Used only to count work items for human-readable output via # `build_anon_work` — same enumeration the guest runs in-circuit. @@ -30,6 +33,11 @@ sp1-sdk = { git = "https://github.com/argumentcomputer/sp1", branch = "blake3-pr tokio = { version = "1", features = ["rt-multi-thread", "macros"] } clap = { version = "4.0", features = ["derive"] } anyhow = "1" +# Per-constant benchmark results JSON (`--json`), merged by the CI bench driver. +serde_json = "1" +# Process-tree RSS sampler (accurate peak RAM) + per-phase timing sink for the +# CI drill-down. +tracing-texray = { git = "https://github.com/argumentcomputer/tracing-texray", rev = "bd4faa08a4fa4edb46bde393b4d20c6bd49591d0" } # Throughput formatting (e.g. `42.0 consts/s`). human-repr = "1" # Proof-size measurement: SP1's `SP1ProofWithPublicValues::bytes()` returns diff --git a/sp1/host/src/main.rs b/sp1/host/src/main.rs index 5c69741e0..f826864a9 100644 --- a/sp1/host/src/main.rs +++ b/sp1/host/src/main.rs @@ -6,7 +6,7 @@ //! RUST_LOG=info cargo run --release -- --execute --ixe ../../minimal.ixe //! WITHOUT_VK_VERIFICATION=1 RUST_LOG=info cargo run --release # prove (compressed) //! # prove a single constant out of a large env (Anon-only): -//! WITHOUT_VK_VERIFICATION=1 RUST_LOG=info cargo run --release -- --ixe ../../init.ixe --constant Nat.add_comm +//! WITHOUT_VK_VERIFICATION=1 RUST_LOG=info cargo run --release -- --ixe ../../init.ixe --consts Nat.add_comm //! ``` //! //! Proving (any non-`--execute` run) requires `WITHOUT_VK_VERIFICATION=1` in @@ -24,6 +24,9 @@ use std::time::Instant; use anyhow::{Result, bail}; use clap::Parser; use human_repr::{HumanCount, HumanThroughput}; +use ix_bench::{ + EXIT_REJECTED, Rejection, Status, collect_consts, peak_rss_bytes, write_row, +}; use ix_kernel::anon_work::{ block_of_addr, build_anon_work, build_sub_env, work_block_addr, }; @@ -37,40 +40,39 @@ pub const GUEST_ELF: Elf = include_elf!("sp1-guest"); #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { - /// Run the program in the VM only - no proof. + /// Execute in the VM only — no proof. #[arg(long)] execute: bool, - /// Run the kernel in Meta mode (preserves names + dup-level-param-name - /// check). Default is Anon mode, which matches Aiur's `kernel_check_test` - /// semantics. Both modes prove the same structural typecheck; Meta is - /// strictly more constrained but slightly more expensive. + /// Run the kernel in Meta mode (default: Anon). Meta preserves names. #[arg(long)] meta: bool, - /// Path to a `.ixe` file produced by `lake exe ix compile`. If omitted, an - /// empty `IxonEnv` is used. + /// Path to a `.ixe` (default: empty env). #[arg(long)] ixe: Option, - /// Check a single constant selected by its Lean NAME (e.g. "Nat.add_comm"). - /// The name resolves through the env's `named` metadata to its ingress - /// block; the guest receives only that block's closure sub-env, so one - /// constant can be proved out of a large env without shipping (or - /// typechecking) the whole thing. By default this is the **full-closure** - /// typecheck (the constant and its whole dependency closure, matching - /// `Ix.Claim.check addr none` / the Aiur `bench-typecheck --constant`); pass - /// `--skip-deps` for a subject-only check (deps trusted). Anon-only - /// (incompatible with `--meta`). Requires `--ixe`. + /// Comma-separated Lean names to check (Anon-only; each is one guest run). + #[arg(long, value_delimiter = ',')] + consts: Vec, + + /// Additional names from a file (one per line, `#` comments); unions with --consts. #[arg(long)] - constant: Option, + consts_file: Option, - /// Modifies `--constant`: check only the named constant itself, trusting its - /// dependencies (subject-only), instead of re-checking its whole transitive - /// closure. Same flag/semantics as `zisk-host --skip-deps` and the Aiur - /// `bench-typecheck --skip-deps`. - #[arg(long, requires = "constant")] + /// With --consts/--consts-file: check each subject only, trusting its deps. + // Validated in main (not clap `requires = "consts"`): names may come from + // --consts-file alone, which a clap-level `requires` would wrongly reject. + #[arg(long)] skip_deps: bool, + + /// Write per-constant results JSON `{ "": { … } }` (accumulated across --consts). + #[arg(long)] + json: Option, + + /// Enable tracing-texray; with --json, per-phase spans are written to .spans. + #[arg(long)] + texray: bool, } fn load_env_bytes(ixe: Option<&PathBuf>) -> Vec { @@ -112,7 +114,7 @@ fn count_checkable(env_bytes: &[u8], meta_mode: bool) -> usize { } } -/// Resolve `--constant ` to the guest inputs for that one constant: its +/// Resolve one `--consts` name to the guest inputs for that constant: its /// closure sub-env and a check-list. The name resolves through the full env's /// `named` metadata to a constant address, which maps to the `build_anon_work` /// item whose ingress block owns it (standalone → itself; a mutual-block member @@ -181,44 +183,99 @@ fn constant_inputs( } #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> std::process::ExitCode { + match run().await { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + eprintln!("error: {e:#}"); + if e.downcast_ref::().is_some() { + // Reserved exit code: rejection rows are on disk; the caller needs + // no log parsing to tell "kernel said no" from an infra failure. + std::process::ExitCode::from(EXIT_REJECTED) + } else { + std::process::ExitCode::FAILURE + } + }, + } +} + +async fn run() -> Result<()> { sp1_sdk::utils::setup_logger(); let args = Args::parse(); + + // Start the process-tree RSS sampler (accurate peak RAM) and point the + // per-phase timing sink at the drill-down file if requested — both + // independent of the SDK's global tracing logger. + // + // TODO(spans): the sink only receives the coarse `sp1/execute` / `sp1/prove` + // phases we `record_manual` below. For a finer drill-down, install a TeXRay + // subscriber and examine the sp1-sdk's own tracing spans — which requires + // composing it with the SDK's global logger (`sp1_sdk::utils::setup_logger`), + // currently the sole subscriber. + tracing_texray::rss_sampler::start(std::time::Duration::from_millis(50)); + // With --texray + --json, per-phase span timings land at `.spans` as + // JSON Lines — the CI drill-down input. + if args.texray { + if let Some(json) = args.json.as_ref().and_then(|p| p.to_str()) { + let _ = tracing_texray::json_sink::to_file(&format!("{json}.spans")); + } + } + let whole_env_bytes = load_env_bytes(args.ixe.as_ref()); + let client = ProverClient::from_env().await; + let consts = collect_consts(&args.consts, args.consts_file.as_deref())?; + if !consts.is_empty() && args.meta { + bail!("--consts is Anon-only and cannot be combined with --meta"); + } + if consts.is_empty() && args.skip_deps { + bail!("--skip-deps requires constants via --consts or --consts-file"); + } - // `--constant` ships a closure sub-env + a check-list (Anon only); otherwise - // the whole env ships with an empty check-list (= check everything). - let (env_bytes, check_list, const_count) = - if let Some(name) = &args.constant { - if args.meta { - bail!("--constant is Anon-only and cannot be combined with --meta"); - } - constant_inputs(&whole_env_bytes, name, args.skip_deps)? - } else { - let cc = count_checkable(&whole_env_bytes, args.meta); - (whole_env_bytes, Vec::new(), cc) - }; + if consts.is_empty() { + run_one(&client, &args, &whole_env_bytes, None).await?; + } else { + for name in &consts { + run_one(&client, &args, &whole_env_bytes, Some(name)).await?; + } + } + Ok(()) +} + +async fn run_one( + client: &C, + args: &Args, + whole_env_bytes: &[u8], + name: Option<&str>, +) -> Result<()> { + // A name ships a closure sub-env + a check-list (Anon only); otherwise the + // whole env ships with an empty check-list (= check everything). + let (env_bytes, check_list, const_count) = match name { + Some(n) => constant_inputs(whole_env_bytes, n, args.skip_deps)?, + None => { + let cc = count_checkable(whole_env_bytes, args.meta); + (whole_env_bytes.to_vec(), Vec::new(), cc) + }, + }; // Three guest inputs, in order: // 1. 1-byte mode flag (0 = Anon / 1 = Meta). - // 2. Serialized Ixon env (whole env, or a closure sub-env under - // `--constant`). Anon enumerates work in-guest via - // `ix_kernel::anon_work::build_anon_work`; Meta walks `env.named`. - // 3. Check-list of packed primary addresses (`--constant`), or empty - // to check every work item. + // 2. Serialized Ixon env (whole env, or a closure sub-env under --consts). + // 3. Check-list of packed primary addresses (--consts), or empty for all. let mut stdin = SP1Stdin::new(); stdin.write::(&u8::from(args.meta)); stdin.write_vec(env_bytes); stdin.write_vec(check_list); - let client = ProverClient::from_env().await; - if args.execute { let exec_start = Instant::now(); let (output, report) = client.execute(GUEST_ELF, stdin).await.expect("execute"); let exec_duration = exec_start.elapsed(); + tracing_texray::json_sink::record_manual( + "sp1/execute", + exec_duration.as_secs_f64(), + ); let failures = u32::from_le_bytes( output.as_slice()[..4].try_into().expect("output too short"), ); @@ -255,8 +312,35 @@ async fn main() -> Result<()> { // point for profiling SP1 cycles. println!("---- ExecutionReport ----"); println!("{report}"); + if let Some(path) = &args.json { + let cycles = report.total_instruction_count(); + let secs = exec_duration.as_secs_f64(); + let tput = if secs > 0.0 { cycles as f64 / secs } else { 0.0 }; + let key = + name.map(|s| s.to_string()).unwrap_or_else(|| "env".to_string()); + let status = if failures > 0 { Status::Rejected } else { Status::Ok }; + write_row( + path, + &key, + status, + serde_json::json!({ + "cycles": cycles, + "execute-time": (secs * 1e6).round() / 1e6, + "throughput": tput.round(), + // The execute phase's RSS high-water — the only phase this cell + // has, so the name stays bare (phases separate at the testbed). + "peak-rss": peak_rss_bytes(), + }), + )?; + } if failures > 0 { - bail!("kernel typecheck produced {failures} failure(s)"); + return Err( + Rejection { + failures: failures.into(), + ctx: name.map_or_else(|| "env".to_string(), |s| s.to_string()), + } + .into(), + ); } return Ok(()); } @@ -269,6 +353,10 @@ async fn main() -> Result<()> { // var (see the module doc header and `Cargo.toml`). `--execute` doesn't. let proof = client.prove(&pk, stdin).compressed().await.expect("prove"); let prove_duration = start.elapsed(); + tracing_texray::json_sink::record_manual( + "sp1/prove", + prove_duration.as_secs_f64(), + ); let throughput = const_count as f64 / prove_duration.as_secs_f64().max(f64::EPSILON); // `SP1ProofWithPublicValues::bytes()` is the onchain-verifier encoding @@ -285,5 +373,81 @@ async fn main() -> Result<()> { client.verify(&proof, pk.verifying_key(), None).expect("verify"); let verify_duration = verify_start.elapsed(); println!("proof verified in {:.3}s", verify_duration.as_secs_f64()); + if let Some(path) = &args.json { + let key = name.map(|s| s.to_string()).unwrap_or_else(|| "env".to_string()); + write_row( + path, + &key, + Status::Ok, + serde_json::json!({ + "prove-time": (prove_duration.as_secs_f64() * 1e6).round() / 1e6, + "peak-rss": peak_rss_bytes(), + }), + )?; + } Ok(()) } + +#[cfg(test)] +mod cli_tests { + use clap::Parser; + + use super::Args; + + fn parse(argv: &[&str]) -> Args { + Args::try_parse_from( + std::iter::once("sp1-host").chain(argv.iter().copied()), + ) + .expect("parse ok") + } + + #[test] + fn consts_splits_on_comma() { + let a = parse(&["--consts", "Nat.add_comm,Nat.succ"]); + assert_eq!(a.consts, vec!["Nat.add_comm", "Nat.succ"]); + } + + #[test] + fn consts_repeatable_and_comma_lists_stack() { + let a = parse(&["--consts", "a", "--consts", "b,c"]); + assert_eq!(a.consts, vec!["a", "b", "c"]); + } + + #[test] + fn skip_deps_parses_with_consts_file_only() { + // Names may come from --consts-file alone; clap must accept the parse + // (main validates after collect_consts). + let a = parse(&["--consts-file", "names.txt", "--skip-deps"]); + assert!(a.skip_deps); + } + + #[test] + fn json_alone_ok() { + // sp1-host's --json is not gated on --consts (keys by "env" when no name). + let a = parse(&["--json", "out.json"]); + assert_eq!(a.json.as_deref(), Some(std::path::Path::new("out.json"))); + } + + #[test] + fn consts_file_alone_ok() { + let a = parse(&["--consts-file", "names.txt"]); + assert_eq!( + a.consts_file.as_deref(), + Some(std::path::Path::new("names.txt")) + ); + } + + #[test] + fn collect_unions_and_dedups() { + // Grammar/union behavior is tested in ix-bench; this pins the Args + // plumbing (comma splitting stacked with a file) end to end. + let path = std::env::temp_dir().join("sp1_host_cli_test_consts.txt"); + std::fs::write(&path, "a\nb\n# comment\n c \n\na\n").expect("write"); + let a = + parse(&["--consts", "a,d", "--consts-file", path.to_str().unwrap()]); + let got = ix_bench::collect_consts(&a.consts, a.consts_file.as_deref()) + .expect("collect"); + assert_eq!(got, vec!["a", "d", "b", "c"]); + let _ = std::fs::remove_file(&path); + } +} diff --git a/zisk/Cargo.lock b/zisk/Cargo.lock index 57b66b1c7..dd7f4a196 100644 --- a/zisk/Cargo.lock +++ b/zisk/Cargo.lock @@ -2634,6 +2634,14 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ix-bench" +version = "0.1.0" +dependencies = [ + "serde_json", + "tracing-texray", +] + [[package]] name = "ix-common" version = "0.1.0" @@ -4188,7 +4196,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -5410,6 +5418,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -5904,6 +5922,18 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-texray" +version = "0.2.0" +source = "git+https://github.com/argumentcomputer/tracing-texray?rev=bd4faa08a4fa4edb46bde393b4d20c6bd49591d0#bd4faa08a4fa4edb46bde393b4d20c6bd49591d0" +dependencies = [ + "loom", + "parking_lot", + "terminal_size", + "tracing", + "tracing-subscriber", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -6504,15 +6534,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -6546,30 +6567,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows-threading" version = "0.1.0" @@ -6600,12 +6604,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -6618,12 +6616,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -6636,24 +6628,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -6666,12 +6646,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -6684,12 +6658,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -6702,12 +6670,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -6720,12 +6682,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" @@ -7254,10 +7210,13 @@ dependencies = [ "anyhow", "clap", "human-repr", + "ix-bench", "ix-common", "ix-kernel", "ixon", + "serde_json", "tokio", + "tracing-texray", "zisk-sdk", ] diff --git a/zisk/Cargo.toml b/zisk/Cargo.toml index 308731d2a..8165b97f0 100644 --- a/zisk/Cargo.toml +++ b/zisk/Cargo.toml @@ -14,11 +14,8 @@ resolver = "2" zisk-sdk = { git = "https://github.com/argumentcomputer/zisk.git", branch = "blake3-precompile" } ziskos = { git = "https://github.com/argumentcomputer/zisk.git", branch = "blake3-precompile" } -[profile.release] -panic = "abort" - # No local-path patch: the host builds zisk-sdk directly from the # `blake3-precompile` branch above (now that v0.18.0 + the blake3 shim are # pushed). To iterate against a local checkout instead, re-add: # [patch."https://github.com/argumentcomputer/zisk.git"] -# zisk-sdk = { path = "/home/ubuntu/zisk/sdk" } +# zisk-sdk = { path = "/path/to/zisk/sdk" } diff --git a/zisk/host/Cargo.toml b/zisk/host/Cargo.toml index d77c0e7a8..d323eb391 100644 --- a/zisk/host/Cargo.toml +++ b/zisk/host/Cargo.toml @@ -10,6 +10,9 @@ path = "src/main.rs" [dependencies] ixon = { path = "../../crates/ixon" } +# Benchmark results-row contract (status enum, row writer, exit codes) shared +# with sp1-host and `ix bench`. +ix-bench = { path = "../../crates/bench" } ix-common = { path = "../../crates/common" } # Used only to count work items for human-readable output via # `build_anon_work` — same enumeration the guest runs in-circuit. @@ -17,6 +20,12 @@ ix-kernel = { path = "../../crates/kernel" } zisk-sdk = { workspace = true } anyhow = "1" clap = { version = "4.0", features = ["derive"] } +# Per-constant benchmark results JSON (`--json`), merged by the CI bench driver. +serde_json = "1" +# Accurate peak RAM via the process-tree sampler (captures the ASM +# microservices' child-process memory that `/proc/self/status` misses) and the +# per-phase timing sink feeding the CI drill-down. +tracing-texray = { git = "https://github.com/argumentcomputer/tracing-texray", rev = "bd4faa08a4fa4edb46bde393b4d20c6bd49591d0" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } # Throughput formatting (e.g. `42.0 consts/s`). human-repr = "1" diff --git a/zisk/host/src/main.rs b/zisk/host/src/main.rs index 1eb35094b..325ff1cc2 100644 --- a/zisk/host/src/main.rs +++ b/zisk/host/src/main.rs @@ -34,6 +34,9 @@ use std::time::{Duration, Instant}; use anyhow::{Result, bail}; use clap::Parser; use human_repr::{HumanCount, HumanThroughput}; +use ix_bench::{ + EXIT_REJECTED, Rejection, Status, collect_consts, peak_rss_bytes, write_row, +}; use ix_kernel::anon_work::{ AnonWorkItem, block_of_addr, build_anon_work, build_sub_env, work_block_addr, }; @@ -186,40 +189,54 @@ struct Args { #[arg(long)] dump_input: Option, - /// Check a single constant selected by its Lean NAME (e.g. - /// "ByteArray.utf8DecodeChar?_utf8EncodeChar_append"), with no manifest or - /// range plumbing: the name is resolved through the env's `named` metadata to - /// its ingress block, and the guest receives only its closure sub-env. By - /// default this is the **full-closure** typecheck — the constant *and* its - /// whole dependency closure are re-checked (matching `Ix.Claim.check addr - /// none`, the default of the Aiur `bench-typecheck --constant`). Pass - /// `--skip-deps` for a subject-only check (deps trusted). Composes with - /// `--execute` (cycles), plain prove (single leaf, subject-bound + verified), - /// and `--dump-input` (write the stdin for ziskemu profiling). Requires - /// exactly one `--ixe`. Note: a member of a mutual block selects the whole - /// block's work item (the kernel checks blocks atomically). + /// Comma-separated Lean names to check (each: closure sub-env → one leaf). + #[arg( + long, + value_delimiter = ',', + conflicts_with_all = ["shard_plan", "only_shard", "store_dir"] + )] + consts: Vec, + + /// Additional names from a file (one per line, `#` comments); unions with --consts. #[arg(long, conflicts_with_all = ["shard_plan", "only_shard", "store_dir"])] - constant: Option, - - /// Modifies `--constant`: check only the named constant itself, trusting its - /// dependencies (subject-only), instead of re-checking its whole transitive - /// closure. Reserved for constants too expensive to full-closure-check that - /// also can't be sharded. Same flag/semantics as the Aiur - /// `bench-typecheck --skip-deps`. - #[arg(long, requires = "constant")] + consts_file: Option, + + /// With --consts/--consts-file: check each subject only, trusting its deps. + // Validated in main (not clap `requires = "consts"`): names may come from + // --consts-file alone, which a clap-level `requires` would wrongly reject. + #[arg(long)] skip_deps: bool, - /// Cap on resident witness traces during the prove phase, bounding - /// peak host RAM per shard. Zisk's prover queues witnesses up to this - /// count before committing them; peak RAM ≈ N × avg-witness-size + - /// fixed overheads. Zisk's built-in default is 10 (tuned for - /// large-memory boxes); we default to 5 here as a safer fit for - /// ~256 GB machines. Override up to 10 on bigger boxes for maximum - /// parallelism, or down to 3 on smaller ones. See the Zisk section - /// of the top-level README for a per-RAM recommendation table. Has - /// no effect on `--execute` / `--verify-constraints` modes. - #[arg(long, default_value_t = 5)] - max_witness_stored: usize, + /// Write per-constant results JSON `{ "": { … } }` (accumulated across names). + /// With `--shard-plan --execute` it instead gets one env-level row (totals + + /// per-shard cycles breakdown). + #[arg(long)] + json: Option, + + /// Benchmark key for the env-level row `--shard-plan --execute --json` + /// writes (e.g. the CamelCase env slug CI uses). Defaults to the manifest + /// file stem. + #[arg(long, requires = "shard_plan")] + json_name: Option, + + /// Enable tracing-texray; with --json, per-phase spans are written to .spans. + #[arg(long)] + texray: bool, +} + +/// Fail FAST on a guest typecheck failure: a rejected constant rejects the +/// whole workload, so bail before spending cycles (or proofs) on the +/// remaining shards — mirroring the OOM kill, which also cancels the rest. +/// Callers with a `--json` row in flight write it with `status: rejected` +/// BEFORE calling this; `main` maps the typed error to `EXIT_REJECTED`. +fn reject_failures(publics: &ShardPublics, ctx: &str) -> Result<()> { + if publics.failures > 0 { + return Err( + Rejection { failures: publics.failures.into(), ctx: ctx.to_string() } + .into(), + ); + } + Ok(()) } /// 112-byte public output of one shard-guest proof. @@ -761,11 +778,7 @@ fn check_input_coherence( Ok(failures) } -fn build_client( - gpu: bool, - asm: bool, - max_witness_stored: Option, -) -> Result { +fn build_client(gpu: bool, asm: bool) -> Result { // Executor choice. The default is the Assembly executor (`asm = true`, // i.e. no `--emulator`): it is markedly faster at trace generation and is // the prerequisite for the hints stream. It historically broke under our @@ -785,10 +798,8 @@ fn build_client( // docs ("Reduce memory footprint during proving at the cost of // speed"). We have ~94 GB of free GPU memory, so the speed // trade-off is the wrong direction for this hardware. - let mut opts = EmbeddedOpts::default(); - if let Some(n) = max_witness_stored { - opts = opts.max_witness_stored(n); - } + // Zisk's default embedded opts (witness cap 10). + let opts = EmbeddedOpts::default(); let mut builder: EmbeddedClientBuilder = ProverClient::embedded().with_embedded_opts(opts); if asm { @@ -800,7 +811,7 @@ fn build_client( builder.build() } -/// Check a single constant chosen by Lean NAME (the `--constant` path). +/// Check a single constant chosen by Lean NAME (one iteration of `--consts`). /// Resolve name → constant address via the env's `named` metadata, map to its /// ingress block's work item, and ship its closure sub-env. By default the /// check-list is the ENTIRE closure (full-closure typecheck); with @@ -895,18 +906,35 @@ async fn run_constant( // ---- Execute mode: cycles only, no proof. ---- if args.execute { + let t0 = Instant::now(); let result = client.execute(&SHARD_PROGRAM, stdin).run()?.await?; + let execute_secs = t0.elapsed().as_secs_f64(); + tracing_texray::json_sink::record_manual("zisk/execute", execute_secs); let mut buf = [0u8; SHARD_PUBLICS_LEN]; result.get_public_values_slice(&mut buf); let publics = ShardPublics::decode(&buf); - println!( - "cycles: {}, failures: {}", - result.get_execution_steps(), - publics.failures - ); - if publics.failures > 0 { - bail!("kernel typecheck produced {} failure(s)", publics.failures); + let cycles = result.get_execution_steps(); + println!("cycles: {cycles}, failures: {}", publics.failures); + if let Some(path) = &args.json { + let tput = + if execute_secs > 0.0 { cycles as f64 / execute_secs } else { 0.0 }; + let status = + if publics.failures > 0 { Status::Rejected } else { Status::Ok }; + write_row( + path, + name, + status, + serde_json::json!({ + "cycles": cycles, + "execute-time": (execute_secs * 1e6).round() / 1e6, + "throughput": tput.round(), + // The execute phase's RSS high-water — the only phase this cell + // has, so the name stays bare (phases separate at the testbed). + "peak-rss": peak_rss_bytes(), + }), + )?; } + reject_failures(&publics, &format!("constant {name}"))?; return Ok(()); } @@ -916,6 +944,10 @@ async fn run_constant( result.get_public_values_slice(&mut buf); let publics = ShardPublics::decode(&buf); let leaf_ms = result.get_proving_time(); + tracing_texray::json_sink::record_manual( + "zisk/prove", + leaf_ms as f64 / 1000.0, + ); let expected = subject_of_cover(&cover); if *expected.as_bytes() != publics.subject_root { bail!( @@ -938,12 +970,23 @@ async fn run_constant( &expected.hex()[..16], roots.len(), ); - if publics.failures > 0 { - bail!( - "kernel typecheck produced {} failure(s) (proof still verifies)", - publics.failures - ); + if let Some(path) = &args.json { + let status = + if publics.failures > 0 { Status::Rejected } else { Status::Ok }; + write_row( + path, + name, + status, + serde_json::json!({ + "prove-time": (leaf_ms as f64).round() / 1000.0, + "steps": result.get_execution_steps(), + "peak-rss": peak_rss_bytes(), + }), + )?; } + // Proof still verifies on rejection — the guest certifies "checked, with + // failures" — but a rejected constant must land as a red row, not a metric. + reject_failures(&publics, &format!("constant {name}"))?; Ok(()) } @@ -1239,29 +1282,104 @@ async fn run_shard_plan( } // ---- Execute mode: run each novel shard in the VM for cycles (no proof); - // store-covered shards have nothing to execute. ---- + // store-covered shards have nothing to execute. With `--json`, one + // env-level row (keyed by `--json-name`, default the manifest stem) + // carries the totals plus a per-shard cycles breakdown under + // `shard-cycles` — the CI benchmark's per-shard tracking. ---- if args.execute { + let t0 = Instant::now(); let mut total_steps = 0u64; - let mut total_failures = 0u32; + let mut max_shard_cycles = 0u64; + let mut max_shard_peak: Option = None; + let mut shard_cycles = serde_json::Map::new(); + let mut shard_time = serde_json::Map::new(); + let mut shard_peak_rss = serde_json::Map::new(); + // Fail FAST on a rejected shard (skip the rest), but still write the + // row — with the shards measured so far — as `status: rejected`. + let mut rejection: Option = None; for &(idx, g) in &novel { let (check_list, sub_env, _cover) = build_inputs(g)?; let stdin = leaf_stdin(0, 0, &sub_env, &check_list); + // Windowed RAM high-water: reset before each shard so the per-shard + // peaks are independent; the env row's peak-rss is their max. + tracing_texray::rss_sampler::reset_peak_tree_rss(); let result = client.execute(&SHARD_PROGRAM, stdin).run()?.await?; let mut buf = [0u8; SHARD_PUBLICS_LEN]; result.get_public_values_slice(&mut buf); let publics = ShardPublics::decode(&buf); let cycles = result.get_execution_steps(); + let exec_secs = result.get_execution_time() as f64 / 1000.0; + let peak = peak_rss_bytes(); total_steps += cycles; - total_failures = total_failures.saturating_add(publics.failures); + max_shard_cycles = max_shard_cycles.max(cycles); + max_shard_peak = max_shard_peak.max(peak); + // 1-based zero-padded keys: matches --only-shard's numbering and keeps + // the flattened bencher measure list (`shard-cycles:`, …) sorted. + let key = format!("{:02}", idx + 1); + shard_cycles.insert(key.clone(), serde_json::json!(cycles)); + shard_time.insert(key.clone(), serde_json::json!(exec_secs)); + if let Some(p) = peak { + shard_peak_rss.insert(key, serde_json::json!(p)); + } println!( - " [shard {idx}] {} work items, failures={}, cycles={cycles}", + " [shard {idx}] {} work items, failures={}, cycles={cycles}, \ + {exec_secs:.1}s, peak {}", g.len(), publics.failures, + peak.map_or("?".to_string(), |p| format!( + "{:.2} GiB", + p as f64 / (1 << 30) as f64 + )), ); + if publics.failures > 0 { + rejection = Some(Rejection { + failures: publics.failures.into(), + ctx: format!("shard {idx}"), + }); + break; + } + } + let execute_secs = t0.elapsed().as_secs_f64(); + tracing_texray::json_sink::record_manual("zisk/execute", execute_secs); + println!( + "total cycles: {total_steps}, failures: {}", + rejection.as_ref().map_or(0, |r| r.failures) + ); + if let Some(path) = &args.json { + let name = args.json_name.clone().unwrap_or_else(|| { + manifest_path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "env".to_string()) + }); + let tput = if execute_secs > 0.0 { + total_steps as f64 / execute_secs + } else { + 0.0 + }; + let status = + if rejection.is_some() { Status::Rejected } else { Status::Ok }; + write_row( + path, + &name, + status, + serde_json::json!({ + "cycles": total_steps, + "shards": novel.len(), + "max-shard-cycles": max_shard_cycles, + "execute-time": (execute_secs * 1e6).round() / 1e6, + "throughput": tput.round(), + // Max over the per-shard windows == the run's execution-phase + // high-water (setup RAM excluded by the resets above). + "peak-rss": max_shard_peak, + "shard-cycles": shard_cycles, + "shard-time": shard_time, + "shard-peak-rss": shard_peak_rss, + }), + )?; } - println!("total cycles: {total_steps}, failures: {total_failures}"); - if total_failures > 0 { - bail!("kernel typecheck produced {total_failures} failure(s)"); + if let Some(r) = rejection { + return Err(r.into()); } return Ok(()); } @@ -1307,6 +1425,7 @@ async fn run_shard_plan( leaf_ms as f64 / 1000.0, result.get_execution_steps(), ); + reject_failures(&publics, &format!("shard {idx}"))?; // Bind each leaf: its committed subject must equal the env-derived merkle // root over the constants it certified. A guest that proved a different set // than the manifest assigned would commit a different root and fail here. @@ -1598,19 +1717,57 @@ async fn run_shard_plan( covered_union.len(), ); if total_failures > 0 { - bail!( - "kernel typecheck produced {total_failures} failure(s) (proof still verifies)" + return Err( + Rejection { + failures: total_failures.into(), + ctx: "aggregate (proof still verifies)".to_string(), + } + .into(), ); } Ok(()) } #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> std::process::ExitCode { + match run().await { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + eprintln!("error: {e:#}"); + if e.downcast_ref::().is_some() { + // Reserved exit code: rejection rows are on disk; the caller needs + // no log parsing to tell "kernel said no" from an infra failure. + std::process::ExitCode::from(EXIT_REJECTED) + } else { + std::process::ExitCode::FAILURE + } + }, + } +} + +async fn run() -> Result<()> { zisk_sdk::setup_logger(VerboseMode::Info); let args = Args::parse(); + // Start the process-tree RSS sampler so `peak_rss_bytes()` reflects the ASM + // microservices' memory, and point the per-phase timing sink at the drill-down + // file if requested. Both are independent of the SDK's global tracing logger. + // + // TODO(spans): the sink only receives the coarse `zisk/execute` / `zisk/prove` + // phases we `record_manual` below. For a finer drill-down (setup, trace-gen, + // per-microservice), install a TeXRay subscriber and examine the zisk-sdk's + // own tracing spans — which requires composing it with the SDK's global logger + // (`zisk_sdk::setup_logger`), currently the sole subscriber. + tracing_texray::rss_sampler::start(std::time::Duration::from_millis(50)); + // With --texray + --json, per-phase span timings land at `.spans` as + // JSON Lines — the CI drill-down input. + if args.texray { + if let Some(json) = args.json.as_ref().and_then(|p| p.to_str()) { + let _ = tracing_texray::json_sink::to_file(&format!("{json}.spans")); + } + } + // Collect inputs. No `--ixe` → a single empty env (back-compat). let inputs: Vec> = if args.ixe.is_empty() { vec![None] @@ -1633,9 +1790,18 @@ async fn main() -> Result<()> { "--shard-plan requires exactly one --ixe input (the env the manifest was built for)" ); } - // `--constant` selects a named constant from one env. - if args.constant.is_some() && inputs.len() > 1 { - bail!("--constant requires exactly one --ixe input"); + // Named constants (from --consts and/or --consts-file) select from one env. + let consts = collect_consts(&args.consts, args.consts_file.as_deref())?; + if !consts.is_empty() && inputs.len() > 1 { + bail!("--consts/--consts-file requires exactly one --ixe input"); + } + if consts.is_empty() && args.skip_deps { + bail!("--skip-deps requires constants via --consts or --consts-file"); + } + if consts.is_empty() && args.json.is_some() && args.shard_plan.is_none() { + bail!( + "--json requires constants via --consts/--consts-file, or --shard-plan" + ); } // ---- Plan every input up front (parse + shard). ---- @@ -1699,8 +1865,7 @@ async fn main() -> Result<()> { let grand_target_count: usize = plans.iter().map(|p| p.target_count).sum(); let total_leaves: usize = plans.iter().map(|p| p.shards.len()).sum(); - let client = - build_client(args.gpu, !args.emulator, Some(args.max_witness_stored))?; + let client = build_client(args.gpu, !args.emulator)?; client.setup(&SHARD_PROGRAM).run()?.await?; // Skip agg-guest setup unless we'll produce more than one leaf proof. // The shard-plan path sets up the agg program itself, after its leaves. @@ -1729,9 +1894,11 @@ async fn main() -> Result<()> { return Ok(()); } - // ---- Single named constant (no manifest/range). ---- - if let Some(name) = &args.constant { - run_constant(&client, &plans[0], name, &args).await?; + // ---- Named constants (no manifest/range). Loops one leaf per name. ---- + if !consts.is_empty() { + for name in &consts { + run_constant(&client, &plans[0], name, &args).await?; + } return Ok(()); } @@ -1739,7 +1906,6 @@ async fn main() -> Result<()> { if args.execute { let mut total_steps: u64 = 0; let mut total_exec_ms: u64 = 0; - let mut total_failures: u32 = 0; for plan in &plans { let num_shards = plan.shards.len(); for (i, &(start, end)) in plan.shards.iter().enumerate() { @@ -1751,19 +1917,22 @@ async fn main() -> Result<()> { let cycles = result.get_execution_steps(); total_steps += cycles; total_exec_ms += result.get_execution_time(); - total_failures = total_failures.saturating_add(publics.failures); println!( " [{} shard {}/{num_shards}] range [{start}, {end}), failures={}, cycles={cycles}", plan.label, i + 1, publics.failures, ); + reject_failures( + &publics, + &format!("{} shard {}/{num_shards}", plan.label, i + 1), + )?; } } let total_exec = Duration::from_millis(total_exec_ms); let throughput = grand_target_count as f64 / total_exec.as_secs_f64().max(f64::EPSILON); - println!("failures: {total_failures}"); + println!("failures: 0"); println!("cycles: {total_steps}"); println!("inputs: {}", plans.len()); println!("work items: {grand_total_items}"); @@ -1773,9 +1942,6 @@ async fn main() -> Result<()> { total_exec.as_secs_f64(), throughput.human_throughput("consts"), ); - if total_failures > 0 { - bail!("kernel typecheck produced {total_failures} failure(s)"); - } return Ok(()); } @@ -1828,6 +1994,10 @@ async fn main() -> Result<()> { (leaf_ms as f64) / 1000.0, result.get_execution_steps(), ); + reject_failures( + &publics, + &format!("{} leaf {}/{num_shards}", plan.label, i + 1), + )?; leaf_proof_bytes.push(result.get_proof_bytes()?); input_publics.push(publics); last_leaf_result = Some(result); @@ -1947,8 +2117,12 @@ async fn main() -> Result<()> { } println!("verify time: {:.3}s", (verify_ms as f64) / 1000.0); if total_failures > 0 { - bail!( - "kernel typecheck produced {total_failures} failure(s) (proof still verifies)" + return Err( + Rejection { + failures: total_failures.into(), + ctx: "batch (proof still verifies)".to_string(), + } + .into(), ); } Ok(()) @@ -2023,3 +2197,79 @@ mod closure_tests { ); } } + +#[cfg(test)] +mod cli_tests { + use clap::Parser; + + use super::Args; + + fn parse(argv: &[&str]) -> Args { + Args::try_parse_from( + std::iter::once("zisk-host").chain(argv.iter().copied()), + ) + .expect("parse ok") + } + fn parse_err(argv: &[&str]) -> String { + Args::try_parse_from( + std::iter::once("zisk-host").chain(argv.iter().copied()), + ) + .unwrap_err() + .to_string() + } + + #[test] + fn consts_splits_on_comma() { + let a = parse(&["--consts", "Nat.add_comm,Nat.succ,String.append"]); + assert_eq!(a.consts, vec!["Nat.add_comm", "Nat.succ", "String.append"]); + } + + #[test] + fn consts_repeatable_and_comma_lists_stack() { + let a = parse(&["--consts", "a", "--consts", "b,c"]); + assert_eq!(a.consts, vec!["a", "b", "c"]); + } + + #[test] + fn skip_deps_and_json_parse_with_consts_file_only() { + // --skip-deps/--json need names, but names may come from --consts-file + // alone — clap must accept the parse (main validates after + // collect_consts). + let a = parse(&[ + "--consts-file", + "names.txt", + "--skip-deps", + "--json", + "out.json", + ]); + assert!(a.skip_deps); + assert_eq!(a.json.as_deref(), Some(std::path::Path::new("out.json"))); + } + + #[test] + fn consts_conflicts_with_shard_plan() { + let s = parse_err(&["--consts", "a", "--shard-plan", "p.ixes"]); + assert!(s.contains("shard-plan") || s.contains("shard_plan")); + } + + #[test] + fn consts_conflicts_with_only_shard() { + let s = parse_err(&["--consts", "a", "--only-shard", "1"]); + assert!(s.contains("only-shard") || s.contains("only_shard")); + } + + #[test] + fn collect_unions_and_dedups() { + // Grammar/union behavior is tested in ix-bench; this pins the Args + // plumbing (comma splitting stacked with a file) end to end. + let dir = std::env::temp_dir(); + let path = dir.join("zisk_host_cli_test_consts.txt"); + std::fs::write(&path, "a\nb\n# comment\n c \n\na\n").expect("write"); + let a = + parse(&["--consts", "a,d", "--consts-file", path.to_str().unwrap()]); + let got = ix_bench::collect_consts(&a.consts, a.consts_file.as_deref()) + .expect("collect"); + assert_eq!(got, vec!["a", "d", "b", "c"]); + let _ = std::fs::remove_file(&path); + } +}