diff --git a/.github/instructions/benchmarks.instructions.md b/.github/instructions/benchmarks.instructions.md index 7dcbd30ac..b9488830b 100644 --- a/.github/instructions/benchmarks.instructions.md +++ b/.github/instructions/benchmarks.instructions.md @@ -15,8 +15,9 @@ SlangPy has a custom benchmark framework built on pytest. Benchmarks live in `sl | `slangpy/benchmarks/` | Benchmark test files (`test_benchmark_*.py`) and Slang shader files | | `slangpy/benchmarks/conftest.py` | Auto-imports benchmark fixtures and registers plugins | | `slangpy/testing/benchmark/fixtures.py` | Pytest fixtures: `BenchmarkSlangFunction`, `BenchmarkPythonFunction`, `BenchmarkComputeKernel`, `ReportFixture` | -| `slangpy/testing/benchmark/plugin.py` | Pytest plugin adding `--benchmark-save`, `--benchmark-compare`, `--benchmark-upload` CLI options | -| `slangpy/testing/benchmark/report.py` | `BenchmarkReport` / `Report` TypedDicts, serialization, MongoDB upload | +| `slangpy/testing/benchmark/plugin.py` | Pytest plugin adding local report and authenticated BenchView submission options | +| `slangpy/testing/benchmark/benchview.py` | Native BenchView payload construction, batching, and authenticated HTTP submission | +| `slangpy/testing/benchmark/report.py` | Legacy local report serialization and comparison data | | `slangpy/testing/benchmark/table.py` | Terminal table display with color-coded deltas | | `slangpy/testing/benchmark/utils.py` | Machine/GPU/commit info collection, JSON datetime helpers | @@ -90,8 +91,49 @@ pytest slangpy/benchmarks -v --benchmark-compare my_run # List saved runs pytest slangpy/benchmarks --benchmark-list-runs + +# Submit to a local BenchView server (PowerShell) +$env:BENCHVIEW_API_KEY = "" +pytest slangpy/benchmarks -v --benchmark-submit --benchmark-api-url http://localhost:3000 + +# Run the CI wrapper against the nested hosted deployment +$env:BENCHVIEW_API_KEY = "" +python tools/ci.py benchmark-python --run-id --api-url http://rtrci.nvidia.com/benchview +``` + +`BENCHVIEW_API_URL` may supply the base URL when direct pytest commands omit `--benchmark-api-url` or the CI wrapper omits `--api-url`. The URL is the root or nested BenchView application base, not the full submission endpoint. The write key is accepted only through `BENCHVIEW_API_KEY`; it is never a command-line option or printed by the benchmark plugin. `--benchmark-upload` remains an alias for `--benchmark-submit` for existing direct pytest scripts, but it now uses only the HTTP API and never connects to MongoDB. + +The ordinary `.github/workflows/ci-benchmark.yml` workflow is manual and is also dispatched by the nightly scheduler. With no `revision` input it checks out and benchmarks the selected branch tip. With an exact 40-character `revision` it checks out that future-compatible commit while retaining the workflow branch name for BenchView. It always builds the selected source and invokes the same `tools/ci.py benchmark-python` command shown above on the Windows and Linux performance workers. Configure repository variable `BENCHVIEW_API_URL` with the root or nested BenchView base URL and repository secret `BENCHVIEW_API_KEY` with its write key before enabling the workflow. + +## Benchmark action scheduling + +The nightly `.github/workflows/schedule-benchmarks.yml` workflow runs once per day. It uses the authenticated GitHub API client supplied by `actions/github-script`, so it needs no checkout, Python environment, GitHub CLI, or extra secret. It examines one fixed 24-hour UTC interval and dispatches the ordinary workflow once for every `main` commit in that interval, oldest first, through the `revision` input. It deliberately does not inspect or suppress existing runs; manually triggering the scheduler repeats the complete 24-hour interval. + +The local historical backfill controller still uses the official GitHub CLI because it is a resumable operator process rather than a GitHub-hosted workflow. It never accepts a token argument. Verify its prerequisite before any preview or dispatch: + +```powershell +gh --version +gh auth status +``` + +Historical commits use the separate manual `.github/workflows/backfill-benchmark.yml` workflow. Its inclusive supported floor is `f3ad0fd91d8cf4eeb2be3b505765b43482aa952a` from 2 September 2025; older revisions are rejected before setup or build. Each matrix job uses ordinary Git commands to create and synchronize a unique normal recursive clone below the runner's temporary directory, builds the untouched historical checkout, and only then overlays the current `tools/ci.py`, `tools/gpu_clock.py`, and `slangpy/testing/benchmark/` reporting harness. Native PowerShell and Bash cleanup steps validate the resolved clone path before removing it. Submitted observations explicitly identify the historical SHA and branch `main`. + +Preview the supported inventory without creating state or dispatching: + +```powershell +python tools/backfill_benchmarks.py --dry-run +``` + +After the boundary and later historical pilot workflows have passed, start or resume the bounded scheduler with: + +```powershell +python tools/backfill_benchmarks.py ``` +The scheduler stores only commit and workflow-run state in `.temp/benchmark-backfill-state.json`. It publishes `dispatching` state before each request, records GitHub's returned run ID afterward, dispatches at most one oldest commit per minute, and never permits more than four active backfill workflows. Ctrl+C exits with code 130 after the latest atomic state replacement; running the same command again reconciles deterministic `backfill-benchmark: ` titles and continues without duplicating accepted requests. `--once` performs at most one scheduling iteration for a controlled trial. + +Never run two scheduler processes against the same state file. If the scheduler reports incompatible or corrupt state, leave it untouched, archive it manually, and rerun so deterministic GitHub titles can reconstruct already requested commits. + ## Writing New Benchmarks 1. Create a `test_benchmark_*.py` file in `slangpy/benchmarks/`. @@ -118,3 +160,7 @@ Reports are JSON files stored in `.benchmarks/`. Each benchmark entry includes: - `cpu_time` — total wall-clock time including warmup The terminal summary table shows color-coded deltas when comparing: green for >5% improvement, red for >5% regression. + +Local report files retain this legacy shape. In parallel, fixtures accumulate native BenchView observations. Tests whose original pytest function name contains `_cpu` submit `cpu_time`; every other benchmark submits `gpu_time`. Both use milliseconds. This matches the existing imported history, including Python wrappers that synchronize GPU work. The stable test ID is the normalized source file plus the original pytest function, while pytest parameters keep their legacy string values as case dimensions and `DeviceType.cuda` is shortened to `cuda`. Project, commit, machine, OS, CPU, and GPU information use BenchView's dedicated run and environment fields. + +One benchmark process submits observations in API-sized batches. Independent device or machine processes at the same Git revision and build configuration derive the same logical run key. A new benchmark execution uses fresh observation timestamps and execution identity, so it replaces matching test cases normally; retrying an unchanged request body is idempotent. Transient connection and gateway failures use five total attempts with exponential backoff while preserving the exact payload and idempotency key. diff --git a/.github/workflows/backfill-benchmark.yml b/.github/workflows/backfill-benchmark.yml new file mode 100644 index 000000000..3d6426821 --- /dev/null +++ b/.github/workflows/backfill-benchmark.yml @@ -0,0 +1,153 @@ +name: backfill-benchmark +run-name: "backfill-benchmark: ${{ inputs.target_sha }}" + +on: + workflow_dispatch: + inputs: + target_sha: + description: "Exact historical main commit to benchmark" + required: true + type: string + +permissions: + contents: read + checks: write + id-token: write + +jobs: + build: + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + os: [windows, linux] + config: [Release] + python: ["3.10"] + include: + - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "benchmark", runs-on: { labels: [Windows, X64, nvrgfx-perf-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "benchmark", runs-on: { labels: [Linux, X64, nvrgfx-perf-kernelvm-bridge] } } + + env: + CI_OS: ${{ matrix.os }} + CI_PLATFORM: ${{ matrix.platform }} + CI_COMPILER: ${{ matrix.compiler }} + CI_CONFIG: ${{ matrix.config }} + CI_PYTHON: ${{ matrix.python }} + CI_FLAGS: ${{ matrix.flags }} + BACKFILL_TARGET_SHA: ${{ inputs.target_sha }} + BACKFILL_FLOOR_SHA: f3ad0fd91d8cf4eeb2be3b505765b43482aa952a + BACKFILL_CLONE_DIR: ${{ runner.temp }}/slangpy-backfill-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.os }} + BENCHVIEW_API_URL: ${{ vars.BENCHVIEW_API_URL }} + BENCHVIEW_API_KEY: ${{ secrets.BENCHVIEW_API_KEY }} + BENCHVIEW_BENCHMARK_REF: ${{ inputs.target_sha }} + BENCHVIEW_BENCHMARK_BRANCH: main + + steps: + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + + - name: Setup MSVC + uses: step-security/msvc-dev-cmd@v1 + + - name: Setup CMake/Ninja + uses: lukka/get-cmake@latest + + - name: Clone historical source + run: | + git clone --recursive "https://github.com/${{ github.repository }}.git" "${{ env.BACKFILL_CLONE_DIR }}" + git -C "${{ env.BACKFILL_CLONE_DIR }}" checkout --detach "${{ inputs.target_sha }}" + + - name: Validate supported history boundary + run: | + echo "Validating target ${{ inputs.target_sha }} against earliest supported commit ${{ env.BACKFILL_FLOOR_SHA }}" + git -C "${{ env.BACKFILL_CLONE_DIR }}" merge-base --is-ancestor "${{ env.BACKFILL_FLOOR_SHA }}" "${{ inputs.target_sha }}" + + - name: Synchronize historical submodules and LFS + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: | + git submodule sync --recursive + git submodule update --init --recursive + git lfs pull + + - name: Setup historical Python environment + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: | + python -m pip install -r requirements-dev.txt + python -m pip install -r samples/requirements.txt + python -m pip install pytest-github-actions-annotate-failures + + - name: Setup PyTorch environment + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: | + python -m pip install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128 + + - name: Historical setup + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py setup + + - name: Historical configure + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py configure + + - name: Historical build + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py build + + - name: Overlay current BenchView benchmark harness + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: git checkout "${{ github.sha }}" -- tools/ci.py tools/gpu_clock.py slangpy/testing/benchmark + + - name: Install slangpy-torch bridge when present + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py install-slangpy-torch + + - name: Benchmark historical source (Windows) + if: runner.os == 'Windows' + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --api-url "${{ env.BENCHVIEW_API_URL }}" --lock-gpu-clocks + + - name: Benchmark historical source (Linux) + if: runner.os == 'Linux' + working-directory: ${{ env.BACKFILL_CLONE_DIR }} + run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --api-url "${{ env.BENCHVIEW_API_URL }}" --lock-gpu-clocks + + - name: Uninstall slangpy-torch + if: always() + run: python -m pip uninstall slangpy-torch -y + continue-on-error: true + + - name: Safely remove temporary clone (Windows) + if: always() && runner.os == 'Windows' + shell: pwsh + run: | + $candidate = [IO.Path]::GetFullPath($env:BACKFILL_CLONE_DIR).TrimEnd([IO.Path]::DirectorySeparatorChar, [IO.Path]::AltDirectorySeparatorChar) + $runnerTemp = [IO.Path]::GetFullPath($env:RUNNER_TEMP).TrimEnd([IO.Path]::DirectorySeparatorChar, [IO.Path]::AltDirectorySeparatorChar) + $parent = [IO.Directory]::GetParent($candidate) + $name = [IO.Path]::GetFileName($candidate) + if ($null -eq $parent -or -not [StringComparer]::OrdinalIgnoreCase.Equals($parent.FullName, $runnerTemp)) { + throw "Refusing to remove clone outside runner.temp: $candidate" + } + if (-not $name.StartsWith("slangpy-backfill-", [StringComparison]::Ordinal)) { + throw "Refusing to remove unexpected directory name: $name" + } + if (Test-Path -LiteralPath $candidate) { + Remove-Item -LiteralPath $candidate -Recurse -Force + } + + - name: Safely remove temporary clone (Linux) + if: always() && runner.os == 'Linux' + shell: bash + run: | + candidate="$(realpath -m -- "$BACKFILL_CLONE_DIR")" + runner_temp="$(realpath -m -- "$RUNNER_TEMP")" + if [[ "$(dirname -- "$candidate")" != "$runner_temp" ]]; then + echo "Refusing to remove clone outside runner.temp: $candidate" >&2 + exit 1 + fi + if [[ "$(basename -- "$candidate")" != slangpy-backfill-* ]]; then + echo "Refusing to remove unexpected directory name: $(basename -- "$candidate")" >&2 + exit 1 + fi + rm -rf -- "$candidate" diff --git a/.github/workflows/ci-benchmark.yml b/.github/workflows/ci-benchmark.yml index fa36a6c6b..fc3af53c1 100644 --- a/.github/workflows/ci-benchmark.yml +++ b/.github/workflows/ci-benchmark.yml @@ -1,9 +1,13 @@ name: ci-benchmark +run-name: "ci-benchmark: ${{ inputs.revision || github.sha }}" on: - schedule: - - cron: '0 */4 * * *' # run every 4 hours workflow_dispatch: + inputs: + revision: + description: "Optional exact revision to benchmark; defaults to the selected branch tip" + required: false + type: string permissions: contents: read @@ -32,10 +36,15 @@ jobs: CI_CONFIG: ${{ matrix.config }} CI_PYTHON: ${{ matrix.python }} CI_FLAGS: ${{ matrix.flags }} + BENCHVIEW_API_URL: ${{ vars.BENCHVIEW_API_URL }} + BENCHVIEW_API_KEY: ${{ secrets.BENCHVIEW_API_KEY }} + BENCHVIEW_BENCHMARK_REF: ${{ inputs.revision || github.sha }} + BENCHVIEW_BENCHMARK_BRANCH: ${{ github.ref_name }} steps: - uses: actions/checkout@v6 with: + ref: ${{ inputs.revision || github.sha }} submodules: recursive lfs: true @@ -59,7 +68,7 @@ jobs: # Setup PyTorch environment - name: Setup PyTorch environment - if: runner.os != 'macos' && contains(matrix.flags, 'unit-test') + if: runner.os != 'macos' && contains(matrix.flags, 'benchmark') run: | python -m pip install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128 @@ -108,11 +117,22 @@ jobs: - name: Build run: python tools/ci.py build + # Install slangpy-torch extension (requires the completed SlangPy build and PyTorch). + - name: Install slangpy-torch + if: runner.os != 'macos' && contains(matrix.flags, 'benchmark') + run: python tools/ci.py install-slangpy-torch + # Benchmark (Python) - name: Benchmark (Python, Windows, GPU Clock Locked) if: contains(matrix.flags, 'benchmark') && runner.os == 'Windows' - run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --mongodb-connection-string "${{ secrets.BENCHMARK_MONGODB_CONNECTION_STRING }}" --mongodb-database-name "nvr-ci" --lock-gpu-clocks + run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --api-url "${{ env.BENCHVIEW_API_URL }}" --lock-gpu-clocks - - name: Benchmark (Python, Linux, GPU Clock Unlocked) + - name: Benchmark (Python, Linux, GPU Clock Locked) if: contains(matrix.flags, 'benchmark') && runner.os == 'Linux' - run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --mongodb-connection-string "${{ secrets.BENCHMARK_MONGODB_CONNECTION_STRING }}" --mongodb-database-name "nvr-ci" + run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --api-url "${{ env.BENCHVIEW_API_URL }}" --lock-gpu-clocks + + # Cleanup slangpy-torch from persistent self-hosted runners. + - name: Uninstall slangpy-torch + if: always() + run: python -m pip uninstall slangpy-torch -y + continue-on-error: true diff --git a/.github/workflows/schedule-benchmarks.yml b/.github/workflows/schedule-benchmarks.yml new file mode 100644 index 000000000..95eedb127 --- /dev/null +++ b/.github/workflows/schedule-benchmarks.yml @@ -0,0 +1,52 @@ +name: schedule-benchmarks + +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + +concurrency: + group: schedule-benchmarks + cancel-in-progress: false + +permissions: + contents: read + actions: write + +jobs: + schedule: + runs-on: ubuntu-latest + steps: + - name: Dispatch recent main commits + uses: actions/github-script@v9 + with: + script: | + const workflow = "ci-benchmark.yml"; + const branch = "main"; + const until = new Date(); + const since = new Date(until.getTime() - 24 * 60 * 60 * 1000); + const common = { + owner: context.repo.owner, + repo: context.repo.repo, + }; + const commits = await github.paginate(github.rest.repos.listCommits, { + ...common, + sha: branch, + since: since.toISOString(), + until: until.toISOString(), + per_page: 100, + }); + core.info( + `Nightly interval ${since.toISOString()} through ${until.toISOString()}: ` + + `${commits.length} commit(s) to dispatch.`, + ); + for (const commit of commits.reverse()) { + const revision = commit.sha; + await github.rest.actions.createWorkflowDispatch({ + ...common, + workflow_id: workflow, + ref: branch, + inputs: { revision }, + }); + core.info(`Dispatched ${workflow} for ${revision} from ${branch}.`); + } diff --git a/.plans/benchmark-action-scheduling.md b/.plans/benchmark-action-scheduling.md new file mode 100644 index 000000000..c8badb17a --- /dev/null +++ b/.plans/benchmark-action-scheduling.md @@ -0,0 +1,324 @@ +# Separate ordinary, nightly, and historical benchmark workflows + +This ExecPlan is a living document. The sections Progress, Surprises and Discoveries, Decision Log, and Outcomes and Retrospective must be kept up to date as work proceeds. + +This plan follows `.agents/PLANS.md` from the repository root. + +## Purpose / Big Picture + +After this change, SlangPy's normal benchmark workflow continues to build the selected current revision and submit results to BenchView through the existing `tools/ci.py benchmark-python` command. It contains no historical-source patching and can be tested independently before the old four-hour schedule is removed. A nightly scheduler then finds `main` commits from the preceding 24 hours and starts that ordinary workflow once for every returned commit. + +A separate `backfill-benchmark.yml` workflow handles the exceptional task of benchmarking older `main` revisions. It builds an unmodified historical checkout in a temporary clone, overlays the current BenchView reporting files only after compilation, and then runs the normal current `tools/ci.py` from inside that clone. Backfill has an explicit earliest supported commit. Revisions older than that boundary are out of scope; the implementation must advance the boundary if the first pilot proves that supporting it would require additional legacy compatibility code. + +The nightly workflow uses GitHub's own authenticated `actions/github-script` client to dispatch ordinary workflows directly. It needs no repository checkout, Python process, GitHub CLI, extra token, or existing-run lookup. The resumable local backfill scheduler uses the official GitHub `gh` command-line client and no third-party Python package. A developer can observe success by manually running the ordinary workflow, seeing Windows and Linux submissions in BenchView with the expected commit identity, triggering the nightly workflow and previewing local backfill without dispatching, successfully running one boundary backfill, interrupting and resuming a small backfill, and finally allowing the backfill to work forward while never exceeding four active workflow runs. + +## Progress + +- [x] (2026-07-16 15:07Z) Built a working prototype that enumerates live `main` history, dispatches workflows through the REST API, persists atomic backfill state, and submits current benchmark observations to BenchView. +- [x] (2026-07-17 09:43Z) Rejected the prototype's combined ordinary/backfill workflow and separate `tools/run_benchmark_ci.py` runner in favor of the three-path design recorded here. +- [x] (2026-07-17 09:43Z) Selected the official `gh` CLI for GitHub access and selected commit `f3ad0fd91d8cf4eeb2be3b505765b43482aa952a` as the initial, provisional backfill boundary. +- [x] (2026-07-17 09:52Z) Milestone 1 implementation and local validation: restored the ordinary scheduled/manual workflow, retained only the BenchView transport changes in `tools/ci.py`, removed the rejected scheduler/runner prototype, added ordinary-path tests and documentation, built successfully, passed 12 focused tests, passed targeted and full Pyright, and passed pre-commit. +- [x] (2026-07-17 12:02Z) Milestone 1 live acceptance: [workflow run 29577592368](https://github.com/shader-slang/slangpy/actions/runs/29577592368) completed successfully at `7708f86d51ca35d52fc573f9972d9399f40718b6`. Both Windows and Linux jobs configured, built, installed the in-tree `slangpy-torch`, reached `tools/ci.py benchmark-python`, and had BenchView accept benchmark batches. The deployed graph API records passed observations for that exact revision from Windows host `CI-VM` and Linux host `kernelvm-9c2ad50b`. +- [x] (2026-07-17) Repaired and pushed the first live-run failures on `dev/ccummings/benchview`: commit `a022e2e7` installs PyTorch and the in-tree `slangpy-torch` bridge, and commit `c92535aa` resolves `ppisp/extensions.slang` and gives every CUDA-only PPISP benchmark an explicit device dimension. The Windows debug build passed in an MSVC developer environment, 16 focused tests passed, the `nodevice` PPISP run skipped all 24 cases for the explicit CUDA-selection reason, Pyright reported zero findings, pre-commit passed, and `git diff --check` passed. +- [x] (2026-07-17 13:40Z) Audited the Milestone 2/3 starting point at commit `8453b4e1`: the rejected scheduler prototype is absent from the current tree and reachable history, the ordinary workflow still has its four-hour schedule and no revision input, the provisional floor resolves locally at `2025-09-02T14:42:35Z`, and unrelated documentation, CMake, plan, and submodule changes remain preserved. +- [x] (2026-07-17) Implemented the local Milestone 2/3 structure: ordinary CI now selects an optional exact revision without historical logic; nightly scheduling uses `actions/github-script` directly; historical work has a manual temporary-clone workflow with a pre-build floor guard, post-build reporting overlay, exact source override, and guarded cleanup; and the local scheduler has a no-shell typed `gh` adapter, versioned atomic state, deterministic-title recovery, oldest-first dispatch, and a four-run cap. +- [x] (2026-07-17) Completed local Milestone 2/3 validation: the Visual Studio 18 debug build passed, all 38 focused submission and scheduling tests passed under the repository's Python 3.12 environment, targeted Pyright reported zero findings, full pre-commit passed including YAML validation, and `git diff --check` passed. A completion audit additionally bounded minute-by-minute workflow polling, kept uncertain-only state alive through its reconciliation grace period, verified historical source identity and Ctrl+C status, and selected GitHub API version `2026-03-10`. Live dispatch pilots remain pending authentication and publication of the workflows. +- [x] (2026-07-17) Simplified both hosted workflows after user review: nightly scheduling now dispatches every `main` commit from the fixed 24-hour interval without reading existing runs or exposing dry-run behavior, while historical cloning, boundary validation, submodule synchronization, and guarded cleanup use ordinary Git plus native PowerShell/Bash rather than inline Python programs. +- [ ] Milestone 2: disable the old schedule, add exact-revision selection for future commits, and validate direct GitHub Actions scheduling of ordinary workflows. +- [ ] Milestone 3: add the dedicated temporary-clone backfill workflow, prove or advance the compatibility boundary, redirect the resumable local scheduler to it, and validate interruption recovery and the four-run cap. +- [ ] Complete repository build, focused tests, Pyright, pre-commit, workflow syntax checks, documentation, live dispatch pilots, and the final operator handoff. + +## Surprises and Discoveries + +- Observation: a `workflow_dispatch` request selects the workflow by branch or tag, not by an arbitrary detached commit SHA. + Evidence: GitHub's workflow dispatch request requires a `ref` that is a branch or tag. To benchmark an exact older commit while using the current workflow definition, the request must use `ref: main` and pass the desired SHA through a declared workflow input consumed by `actions/checkout`. + +- Observation: current GitHub.com dispatch responses can include the new workflow run ID and URLs immediately. + Evidence: the current API accepts `return_run_details: true` and responds with `workflow_run_id`, `run_url`, and `html_url`. The scheduler can persist that ID without waiting for the new run to appear in a later listing, while deterministic run titles still cover a crash between dispatch and state publication. + +- Observation: commit `f3ad0fd91d8cf4eeb2be3b505765b43482aa952a`, dated 2 September 2025, introduced `--device-types` and the per-device benchmark loop used by current `tools/ci.py`. + Evidence: `git log -S"--device-types" -- slangpy/testing/plugin.py tools/ci.py` identifies that commit. Earlier commits require a different execution shape, so this plan initially excludes them rather than maintaining an additional benchmark runner. + +- Observation: historical revisions still contain the obsolete MongoDB reporter even after the provisional device-filter boundary. + Evidence: the current BenchView files under `slangpy/testing/benchmark/` must be overlaid after historical setup, configure, and build. The current `tools/ci.py` and `tools/gpu_clock.py` must also be overlaid for the benchmark step, but they must not configure or compile the historical source. + +- Observation: retrieving 1,000 workflow runs through the prototype required as many as ten paginated requests and took roughly 35 seconds. + Evidence: live no-write previews on 16 July 2026 took 35 to 42 seconds. The final backfill should do a broad title reconciliation only on startup and periodic refresh, while minute-by-minute capacity polling reads only recent runs. + +- Observation: `gh api --paginate --slurp` downloads every available workflow-run page before Python can apply its `maximum` slice. + Evidence: the audited adapter used that form even for the minute-by-minute `maximum=100` capacity poll. It now requests explicit bounded pages and stops as soon as the requested maximum or the final short page is reached; `test_github_cli_fetches_only_the_requested_workflow_run_pages` locks this behavior. + +- Observation: a state containing only a recent uncertain `dispatching` record is not complete even though it has no immediately pending record. + Evidence: the original loop predicate exited on an empty pending list and incorrectly printed that all commits were requested. The scheduler now remains active while either pending or uncertain records exist, and an offline time-stepped test proves it waits through the grace period before retrying an unmatched request. + +- Observation: the repository contained 355 `main` commits since 1 September 2025 when the prototype was tested. + Evidence: local Git history and the live GitHub commits endpoint agreed. The final supported count will be slightly smaller because the provisional boundary excludes commits before the afternoon of 2 September and may move forward after the pilot. + +- Observation: an unset GitHub repository variable is passed to a workflow command as an explicit empty string. + Evidence: the ordinary workflow always supplies `--api-url "${{ env.BENCHVIEW_API_URL }}"`. `tools/ci.py` now distinguishes an omitted option from an explicitly empty option, forwarding the latter to pytest so `pytest_configure` reports the missing URL/key instead of silently disabling submission. `test_ci_wrapper_forwards_an_explicit_empty_api_url` covers this behavior. + +- Observation: GitHub CLI authentication was unavailable during the first implementation pass but was renewed before final acceptance. + Evidence: `gh auth status` initially reported the `ccummingsNV` token as invalid. During the final audit it identified `ccummingsNV` as the active authenticated account, allowing the successful workflow and job logs to be inspected without exposing the token. + +- Observation: the local `gh` executable is available for the Milestone 3 backfill controller, but its saved authentication became invalid again before this implementation pass. + Evidence: `gh --version` reports 2.94.0, while `gh auth status` reports the active `ccummingsNV` credential as invalid. Fake-runner unit coverage and local code work remain possible, but local backfill previews or dispatch pilots require `gh auth login -h github.com` first. Nightly scheduling is unaffected because `actions/github-script` receives GitHub's job token automatically. + +- Observation: the existing Windows debug build directory was configured with Visual Studio 18, so entering a Visual Studio 2022 developer environment produced standard-library linker mismatches rather than a source failure. + Evidence: the VS2022 attempt failed on unresolved `__std_*` symbols, while the same incremental build under `C:\Program Files (x86)\Microsoft Visual Studio\18\BuildTools\VC\Auxiliary\Build\vcvars64.bat` linked all 23 remaining targets successfully. + +- Observation: the benchmark workflow's copied PyTorch setup condition referenced the `unit-test` flag, while its matrix supplies only the `benchmark` flag, and the workflow did not install the compiled `slangpy-torch` bridge after building. + Evidence: the live CUDA tensor benchmarks raised `PyTorch tensors detected but slangpy-torch is not installed`. The ordinary unit-test workflow already establishes the required order: install PyTorch, build SlangPy, then run `python tools/ci.py install-slangpy-torch`. + +- Observation: `test_benchmark_bwd_diff.py` searched the repository root for `extensions.slang`, but the only supplied file is `slangpy/benchmarks/ppisp/extensions.slang`. + Evidence: the live Slang compiler reported include error E15300, and the local repository inventory plus the corrected include-path regression test identify the PPISP directory as the valid source. + +- Observation: device-isolation classification is driven by the test function's explicit `device_type` parameter, not by a device requested later inside the test body. + Evidence: PPISP tests hard-coded `get_torch_device(DeviceType.cuda)` without declaring the parameter, so the plugin assigned them to the `nodevice` shard. After adding a CUDA parameter to all such tests, the local `--device-types nodevice -rs` run reports 24 skips from `slangpy/testing/plugin.py` with target device `cuda`. + +## Decision Log + +- Decision: keep `.github/workflows/ci-benchmark.yml` as the ordinary benchmark workflow and remove all historical harness overlay behavior from it. + Rationale: scheduled and manually selected current revisions should exercise the same simple path developers already understand. Historical compatibility must not make routine benchmarks harder to test or maintain. + Date/Author: 2026-07-17, Codex. + +- Decision: implement and validate the ordinary BenchView migration before disabling its existing four-hour schedule. + Rationale: this produces a small first change that can be run on the real performance workers and verified in BenchView before scheduling and historical behavior are introduced. + Date/Author: 2026-07-17, Codex. + +- Decision: after ordinary validation, add one optional `revision` input to `ci-benchmark.yml`. + Rationale: dispatching the workflow on a branch naturally benchmarks the branch tip. GitHub does not support a detached SHA as the dispatch `ref`, so the optional input is the smallest generic mechanism for nightly scheduling or manually selecting an exact commit. It is not a legacy compatibility feature and may target only revisions that already contain the BenchView producer. + Date/Author: 2026-07-17, Codex. + +- Decision: use `.github/workflows/backfill-benchmark.yml` exclusively for pre-BenchView historical revisions. + Rationale: the workflow can own temporary cloning, post-build overlays, source identity correction, and the supported-history boundary without leaking those concerns into ordinary CI. + Date/Author: 2026-07-17, Codex. + +- Decision: use `f3ad0fd91d8cf4eeb2be3b505765b43482aa952a` as a provisional inclusive backfill floor and do not support earlier commits. + Rationale: this is the first commit with the device-selection interface required by the normal current `tools/ci.py`. The user explicitly accepts a finite history. If an end-to-end pilot at this SHA reveals another difficult compatibility boundary, move the floor forward and record the first passing SHA instead of adding a special legacy runner. + Date/Author: 2026-07-17, Codex. + +- Decision: build historical source with its own `tools/ci.py`, then overlay the current reporting harness and current `tools/ci.py` only for benchmark execution. + Rationale: historical setup and native compilation must retain the historical build behavior, while the benchmark process needs the current HTTP submission interface. Executing the copied current `tools/ci.py` from inside the temporary clone gives it the correct project paths and removes the need for `tools/run_benchmark_ci.py`. + Date/Author: 2026-07-17, Codex. + +- Decision: use `actions/github-script` directly for nightly scheduling and retain the official `gh` CLI only for the local backfill controller. + Rationale: the nightly scheduler already executes inside GitHub Actions, where the action provides an authenticated Octokit client that can enumerate commits and dispatch `workflow_dispatch` events. Python and `gh` would only add setup there. Backfill remains a long-running, resumable local process, where `gh` supplies authentication and API transport without a custom HTTP client or third-party Python dependency. + Date/Author: 2026-07-17, Codex. + +- Decision: interpret “commits pushed to main” as commits reachable from `main`, filtered by committer timestamp. + Rationale: historical push events are not a durable commit index, whereas GitHub's commits endpoint can enumerate landed history. This creates one benchmark request per landed commit. + Date/Author: 2026-07-17, Codex. + +- Decision: use deterministic titles `ci-benchmark: ` for traceability and `backfill-benchmark: ` for local-state reconciliation, while persisting a write-ahead `dispatching` state for backfill. + Rationale: nightly intentionally dispatches every commit in every invocation and does not deduplicate by title. Backfill titles still reconcile the narrow case where the local process exits after GitHub accepts the request but before its state file is replaced. + Date/Author: 2026-07-17, Codex. + +- Decision: count every non-completed `backfill-benchmark` workflow as active and dispatch at most one oldest pending commit per minute while fewer than four are active. + Rationale: this implements the requested workflow-level pressure limit without depending on the two Windows/Linux jobs inside each matrix run. + Date/Author: 2026-07-17, Codex. + +- Decision: preserve local no-submission use when `--api-url` is omitted, but fail through the pytest plugin when the workflow explicitly passes an empty API URL. + Rationale: local developers may still use `tools/ci.py benchmark-python` without a server, while a misconfigured production workflow must not appear successful after discarding all results. + Date/Author: 2026-07-17, Codex. + +- Decision: make the ordinary benchmark workflow install and remove the in-tree `slangpy-torch` extension using the same post-build command as ordinary unit CI. + Rationale: installing the `torch` wheel alone does not provide SlangPy's tensor bridge. Building the bridge from the checked-out revision keeps it ABI-aligned with that revision, and cleanup prevents contamination of persistent performance runners. + Date/Author: 2026-07-17, Codex. + +- Decision: require CUDA-only benchmark cases to expose `device_type=cuda` through pytest parameterization. + Rationale: this is the routing contract understood by the isolation plugin and also records CUDA as a normal BenchView case dimension. A hard-coded device request inside the test is too late for shard selection. + Date/Author: 2026-07-17, Codex. + +## Outcomes and Retrospective + +The earlier prototype proved that live commit discovery, deterministic run reconciliation, atomic local recovery, and native BenchView submission are feasible. It did not dispatch a real benchmark workflow and its architecture is not the desired result: it places historical checkout and overlay logic in `ci-benchmark.yml`, adds a second benchmark runner, and contains a large custom REST client. + +Milestone 1 is implemented locally. The resulting `ci-benchmark.yml` differs from its repository baseline only by the BenchView URL/key environment and the two API-based `tools/ci.py` invocations. The obsolete MongoDB dependency and uploader are removed, while local report save/compare data remains. The alternate runner, scheduling workflows/scripts, and custom REST client from the rejected prototype are absent. The Windows debug build completed, 12 focused tests passed, targeted and full Pyright reported zero findings, full and explicit-new-file pre-commit passed, and `git diff --check` found no errors. + +Milestone 1 is complete. The first external run exposed stale suite assumptions: PyTorch setup was gated on the wrong matrix flag, `slangpy-torch` was not installed, one Slang include directory was wrong, and CUDA-only PPISP tests were routed to `nodevice`. Commits `a022e2e7` and `c92535aa` repaired those failures, and `7708f86d` restored restricted Linux GPU-clock control without running the Python process as root. Local evidence is a successful Windows debug build, 16 focused tests, 24 correctly skipped PPISP cases in a `nodevice` run, zero Pyright findings, a clean full pre-commit run, and a clean diff check. + +The final external evidence is [GitHub Actions run 29577592368](https://github.com/shader-slang/slangpy/actions/runs/29577592368), completed at 12:02Z on 17 July 2026. Its Windows and Linux matrix jobs both completed successfully, installed the checked-out revision's `slangpy-torch` extension, invoked the ordinary `tools/ci.py benchmark-python` path, and received successful BenchView batch acknowledgements. BenchView's deployed read API shows revision `7708f86d51ca35d52fc573f9972d9399f40718b6` with passed Windows observations on `CI-VM` and passed Linux observations on `kernelvm-9c2ad50b`. The original missing-bridge, missing-include, and `nodevice` routing errors do not recur in the final logs. Per-device execution deliberately continues after a failed shard; both final CUDA shards abort in the autograd suite for an unrelated native failure, while other platform/target batches are durably accepted. The user confirmed the benchmark workflow is operational, and this does not block the Milestone 1 transport and ordinary-workflow acceptance gate. + +The local Milestone 2 and 3 implementation is complete and validated. User review removed unnecessary hosted-workflow machinery: `schedule-benchmarks.yml` now performs recent-commit discovery and unconditionally dispatches every returned commit through `actions/github-script`, while `backfill-benchmark.yml` uses ordinary Git and native shell cleanup rather than embedded Python. Python and `gh` remain only where they add value—the stateful local historical controller. The Visual Studio 18 debug build, 38 focused offline tests, targeted Pyright, full pre-commit, and the diff check all pass. The remaining acceptance work is inherently live: publish the workflows, run the nightly and exact-revision pilots, renew local `gh` authentication, then run the two historical pilots and interruption/capacity trial. + +## Context and Orientation + +Run every command in this plan from `C:\sw\slangpy`. The repository is currently on `dev/ccummings/benchview` with Milestone 1 pushed through `7708f86d`; local documentation, `tests/CMakeLists.txt`, plan changes, and the pre-existing `external/slang-rhi` submodule pointer remain outside that commit. Preserve those unrelated changes. The BenchView producer is committed and remains the transport implementation that the later scheduling workflows need. + +`.github/workflows/ci-benchmark.yml` is the existing Windows/Linux performance workflow. In the repository baseline it runs every four hours and on manual dispatch, checks out the workflow revision, sets up Python and CMake, invokes `python tools/ci.py setup`, `configure`, and `build`, and finally invokes `python tools/ci.py benchmark-python`. The prototype changed it to accept a historical SHA, overlay reporter files, and call `tools/run_benchmark_ci.py`; those historical changes must move out. + +`tools/ci.py` is the normal cross-platform CI entry point. Its `benchmark_python` function selects D3D12, Vulkan, CUDA, Metal, and non-device test shards as appropriate; optionally locks GPU clocks; and invokes pytest. The accepted BenchView change replaces MongoDB arguments with `--api-url` and passes `--benchmark-submit` plus `--benchmark-api-url`. It must remain the only benchmark runner. + +`slangpy/testing/benchmark/benchview.py`, `fixtures.py`, `plugin.py`, `report.py`, and `utils.py` form the current BenchView reporting harness. The pytest plugin collects observations, constructs submissions at session completion, and sends them with `BENCHVIEW_API_KEY`. `plugin.py` also contains `apply_benchmark_source_override`, which lets a historical overlaid checkout report the intended source SHA, branch, and clean state rather than the current harness commit and dirty working tree. Ordinary local runs without override environment variables retain their natural Git identity. + +`tools/benchmark_actions.py` and `tools/backfill_benchmarks.py` implement the local historical scheduler. The first is a small typed adapter around `gh`; the second owns the resumable state machine. Nightly scheduling does not share this machinery because `.github/workflows/schedule-benchmarks.yml` can enumerate and dispatch workflows with GitHub's own authenticated API client. `tools/run_benchmark_ci.py` was a prototype-only duplicate runner and remains deleted. + +A workflow dispatch is a request for GitHub to start a workflow declaring `workflow_dispatch`. The request's `ref` selects a branch or tag containing the workflow definition. An optional workflow input is a named string in that request. Ordinary exact-revision scheduling dispatches `ci-benchmark.yml` from `main` and passes `revision=`; historical scheduling dispatches `backfill-benchmark.yml` from `main` and passes `target_sha=`. + +The local backfill state remains an ignored JSON file at `.temp/benchmark-backfill-state.json`. It records its schema version, repository, branch, workflow, supported lower bound, each commit, dispatch status, and any known run ID and URL. It must never contain GitHub credentials or the BenchView API key. The redesigned state must identify `backfill-benchmark.yml`; an incompatible state written by the prototype must fail with a clear instruction to archive it and start a new state file, never be silently reinterpreted. + +The checked-in `.plans/benchview-benchmark-submission.md` describes the producer-side BenchView work. This plan preserves that producer and changes only normal CI invocation, scheduling, and the historical execution boundary. + +## Plan of Work + +### Milestone 1: prove the ordinary BenchView workflow + +First reduce `.github/workflows/ci-benchmark.yml` to its ordinary purpose. Restore the baseline four-hour cron and input-free manual dispatch. Remove the historical `benchmark_ref`, historical checkout selection, overlay step, hard-coded `main` identity, and every invocation of `tools/run_benchmark_ci.py`. Keep the Windows/Linux matrix and existing setup, configure, and build steps. Gate PyTorch setup on the actual `benchmark` matrix flag, install the in-tree `slangpy-torch` bridge after the SlangPy build, and remove it in an always-run cleanup step because the performance workers are persistent. Add only the `BENCHVIEW_API_URL` repository variable and `BENCHVIEW_API_KEY` repository secret to the job environment, and change the two benchmark steps to invoke `python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --api-url "${{ env.BENCHVIEW_API_URL }}"`, retaining GPU clock locking where supported by the worker configuration. + +Keep the accepted BenchView changes in `tools/ci.py`: remove the MongoDB command-line options, define `--api-url`, obtain a default from `BENCHVIEW_API_URL`, and append the current pytest submission options only when an API URL is present. Preserve the existing device selection, failure continuation, and clock restoration behavior. Remove `pymongo` from `requirements-dev.txt` only if no remaining current code or tests import it. Do not add a compatibility fallback for revisions before the chosen boundary. + +Keep and finish the producer changes in `slangpy/testing/benchmark/`. Move any tests that only exist for `tools/run_benchmark_ci.py` to normal `tools/ci.py` command-construction coverage if they remain useful, then delete `tools/run_benchmark_ci.py`. Update `.github/instructions/benchmarks.instructions.md` so its ordinary instructions describe the API URL, secret, normal `ci.py` command, and current schedule without mentioning backfill. + +Build before testing, run the focused producer and CI-wrapper tests, run the CUDA-only suites through a `nodevice` shard to prove they are excluded before execution, and run a manual ordinary workflow on a revision containing these changes. Acceptance for this milestone requires both matrix jobs to build the in-tree torch bridge, reach `tools/ci.py benchmark-python` without device-routing or include-path failures, have BenchView accept their submissions, and show the selected current commit and correct Windows/Linux hosts. Keep the four-hour schedule in place until that evidence exists. Record the GitHub run URL and BenchView verification here without recording secrets. + +### Milestone 2: add exact future scheduling inside GitHub Actions + +After Milestone 1 passes, remove the four-hour cron from `.github/workflows/ci-benchmark.yml`. Add a single optional string input named `revision`. Set the deterministic run name to `ci-benchmark: ${{ inputs.revision || github.sha }}` and set the checkout ref to the same expression. Export `BENCHVIEW_BENCHMARK_REF` as the selected revision and `BENCHVIEW_BENCHMARK_BRANCH` as `github.ref_name`, so dispatching from `main` with an older SHA still reports branch `main`. This input is for revisions that contain the current BenchView producer; it does not trigger overlays or historical compatibility. + +Keep `.github/workflows/schedule-benchmarks.yml` as a small nightly and manually dispatched job with `contents: read` and `actions: write`. Its only step uses `actions/github-script`. The script fixes one UTC interval at startup, beginning 24 hours earlier, paginates `main` commits in that interval, and dispatches every commit oldest first from workflow ref `main` with input `revision=`. It deliberately does not list existing workflow runs, suppress duplicate titles, or expose a dry-run mode. It needs no checkout, Python setup, `gh` executable, or extra secret because `actions/github-script` receives the repository's job token. GitHub explicitly allows `workflow_dispatch` events created with `GITHUB_TOKEN` to start new workflow runs. Concurrency prevents overlapping scheduler instances. + +Acceptance requires one controlled scheduler invocation to dispatch exactly the commits returned for its 24-hour interval, plus an exact-revision `ci-benchmark.yml` run for a recent non-tip `main` SHA. The resulting BenchView observation must identify that exact SHA and `main`, proving the generic revision input works before it is used nightly. + +### Milestone 3: add bounded historical backfill + +Create `.github/workflows/backfill-benchmark.yml`. It is manual-only, declares required string input `target_sha`, uses deterministic run name `backfill-benchmark: ${{ inputs.target_sha }}`, and uses the same Windows/Linux performance-runner matrix and Python/CMake preparation as the ordinary workflow. The workflow definition is always dispatched from `main`, so `github.sha` identifies the current harness revision while `inputs.target_sha` identifies the historical source revision. + +For each matrix job, use ordinary Git commands to make a unique normal recursive clone below `runner.temp`, check out `target_sha` detached, run `git submodule sync --recursive`, `git submodule update --init --recursive`, and Git LFS retrieval, and install the target revision's development and sample requirements. Do not embed a Python program for these repository operations. Validate that the target is the provisional floor commit or a descendant by running `git merge-base --is-ancestor f3ad0fd91d8cf4eeb2be3b505765b43482aa952a `. If not, stop before setup with a message naming the earliest supported SHA. + +From the temporary clone, invoke the target revision's `python tools/ci.py setup`, `configure`, and `build`. Only after a successful build, overlay the current harness by running `git checkout "${{ github.sha }}" --` for `tools/ci.py`, `tools/gpu_clock.py`, and the required files under `slangpy/testing/benchmark/`. Do not overlay native source, CMake files, the global test plugin, or the build configuration. Then invoke the overlaid `python tools/ci.py benchmark-python` from the temporary clone with the same API arguments as the ordinary workflow. Set `BENCHVIEW_BENCHMARK_REF` to `target_sha` and `BENCHVIEW_BENCHMARK_BRANCH` to `main`; the reporter override must emit `dirty: false` while retaining timestamps and other source facts from historical HEAD. + +Clean the unique temporary clone in OS-specific `if: always()` steps using native PowerShell on Windows and Bash on Linux. Before recursively deleting anything, resolve both the candidate and `runner.temp`, require the candidate's direct parent to equal `runner.temp`, and require its name to start with the workflow's fixed backfill prefix. A failed safety check must leave the directory in place and fail cleanup rather than delete an unexpected path. Do not embed Python for cleanup. + +The first live backfill run must target `f3ad0fd91d8cf4eeb2be3b505765b43482aa952a`. If it builds, runs, and submits valid Windows and Linux observations, record it as the final inclusive floor. If it fails because the reporting harness or benchmark command cannot be overlaid without more compatibility code, inspect later history and advance the floor to the first commit that completes the same pilot. Update the workflow guard, scheduler default, tests, documentation, Decision Log, and this section together. Do not add a second benchmark runner or a pre-device-filter fallback merely to retain older commits. + +Refactor `tools/backfill_benchmarks.py` so its default workflow is `backfill-benchmark.yml` and its default lower bound is the final supported boundary. Preserve the existing oldest-first, one-dispatch-per-minute behavior, four-active-run limit, atomic state replacement, `dispatching` write-ahead marker, deterministic-title reconciliation, transient retry, `--dry-run`, and `--once`. Count only non-completed runs of `backfill-benchmark.yml`; ordinary and nightly `ci-benchmark` runs do not consume backfill capacity. + +Implement `tools/benchmark_actions.py` as a small typed adapter around `gh` for this local controller. It locates `gh` with `shutil.which`, fails with a concise installation/authentication message when unavailable, and executes commands with `subprocess.run` using argument arrays, captured UTF-8 output, `check=False`, and no shell. Authentication comes from `gh auth login`; do not accept or print a token argument. Commit listing uses `gh api --paginate --slurp`; workflow-run listing requests explicit bounded pages and flattens each page's `workflow_runs`; and dispatch sends exact JSON on standard input with `return_run_details: true`. Add current GitHub API version `2026-03-10` to every call, retain typed `Commit`, `WorkflowRun`, and `DispatchResult` records, and inject the command runner in tests. + +Revise the state schema and compatibility validation to bind it to the repository, `main`, `backfill-benchmark.yml`, and supported lower bound. Before calling `gh`, atomically save a record as `dispatching`; after the POST succeeds, save its returned run ID and URL as `dispatched`. On restart, reconcile `dispatching` records against `backfill-benchmark: ` titles. Retry an unmatched uncertain dispatch only after the documented grace period. A prototype state with a different workflow or schema must produce a clear error explaining how to archive it; never mutate it silently. + +Update `slangpy/tests/utils/test_benchmark_action_scheduling.py` and `.github/instructions/benchmarks.instructions.md`. Tests must cover the nightly workflow's direct commit listing and unconditional exact-input dispatch; `gh` JSON parsing and errors through a fake command runner; native historical workflow operations and cleanup guards; backfill-only active counts, oldest-first dispatch, four-run capacity, state mismatch, atomic reload, returned run details, uncertain-title reconciliation, and boundary rejection. Documentation must explain the nightly unconditional direct GitHub API path, then show `gh --version`, `gh auth status`, backfill dry-run and real commands, state location, Ctrl+C/restart behavior, the supported floor, and the prohibition on two simultaneous backfill schedulers sharing one state file. + +Acceptance requires successful pilots at the final floor and at one recent pre-BenchView commit, an interrupted two-or-more-commit scheduler trial that resumes without duplicate dispatch, and observed capacity never exceeding four active backfill workflow runs. + +## Concrete Steps + +Work from the repository root and inspect the dirty tree before each milestone: + + cd C:\sw\slangpy + git status --short + git diff -- .github/workflows/ci-benchmark.yml tools/ci.py requirements-dev.txt slangpy/testing/benchmark tools + +Verify the local backfill client's authentication before running it: + + gh --version + gh auth status + +The first command must print a GitHub CLI version. The second must identify an authenticated GitHub host without printing a token. Nightly scheduling does not use this client. + +Per repository policy, build before running Python tests: + + cmake --build --preset windows-msvc-debug + python -m pytest slangpy/tests/utils/test_benchmark_submission.py slangpy/tests/utils/test_benchmark_action_scheduling.py -v + python -m pyright tools/ci.py tools/benchmark_actions.py tools/backfill_benchmarks.py slangpy/testing/benchmark slangpy/tests/utils/test_benchmark_action_scheduling.py + pre-commit run --all-files + git diff --check + +If pre-commit edits files, repeat the affected tests and pre-commit command. New untracked files must also be passed explicitly to `pre-commit run --files` if the installed pre-commit version does not include them in `--all-files`. + +Manually running `schedule-benchmarks.yml` is a real scheduler invocation. Expected output names the fixed UTC interval and reports one successful `ci-benchmark.yml` dispatch for every `main` commit returned in that interval, oldest first. + +Preview the supported historical inventory: + + python tools/backfill_benchmarks.py --dry-run + +Expected output names `backfill-benchmark.yml`, the final inclusive boundary SHA or timestamp, the number of supported commits, existing deterministic backfill titles, missing commits, and active backfill runs. It must not write `.temp/benchmark-backfill-state.json` and must not dispatch. + +Run or resume the real backfill only after both live pilot workflows pass: + + python tools/backfill_benchmarks.py + +Pressing Ctrl+C must return exit status 130 after the most recent state replacement. Repeating the same command must load that state, reconcile GitHub runs, and continue from the oldest undispatched supported commit. + +## Validation and Acceptance + +Milestone 1 is accepted only when `.github/workflows/ci-benchmark.yml` has no historical input, clone overlay, or alternate runner; both performance matrix jobs call `tools/ci.py`; and a real run produces readable BenchView observations for its selected current SHA. Merely passing unit tests is insufficient because this milestone exists to prove the real worker, GPU, authentication, and server path. + +Milestone 2 is accepted when manual branch dispatch still benchmarks the branch tip with no revision input, exact-revision dispatch benchmarks a selected non-tip commit containing BenchView support, and one scheduler invocation dispatches every `main` commit from its preceding 24-hour interval. The scheduler workflow must contain no existing-run lookup, deduplication, dry-run behavior, checkout, Python setup, `gh` setup, or extra credential. + +Milestone 3 is accepted when the workflow rejects a commit before the supported floor before building, the final floor pilot and a later historical pilot both submit the exact target SHA as branch `main`, and the historical native build occurs before any current files are overlaid. `tools/run_benchmark_ci.py` must no longer exist. Current `tools/ci.py` must not contain a fallback for revisions earlier than the supported floor. + +Scheduler tests must run without a GPU, GitHub authentication, network, or real `gh` executable. With three active backfill runs, one oldest pending commit is dispatched; with four active runs, none is dispatched. A successful dispatch stores its returned run ID. A state left at `dispatching` becomes dispatched when a matching deterministic title appears, and becomes pending only after the grace period if no matching run appears. An incompatible prototype state is rejected without modification. + +The final build and focused tests must pass, targeted Pyright must report zero errors, pre-commit must pass after any automatic edits, and `git diff --check` must report no whitespace errors. Record exact counts and live pilot URLs in Progress and Outcomes and Retrospective as they become available. + +Local Milestone 1 evidence recorded on 17 July 2026 is: + + cmake --build --preset windows-msvc-debug + [0/2] Re-checking globbed directories... + + python -m pytest slangpy/tests/utils/test_benchmark_submission.py -q + 12 passed in 0.05s + + python -m pyright + 0 errors, 0 warnings, 0 informations + + pre-commit run --all-files + Passed + +The first pre-commit pass reformatted `tools/ci.py`; the focused tests and Pyright were rerun afterward before the clean pre-commit result. + +## Idempotence and Recovery + +Ordinary and nightly workflow dispatches use traceable `ci-benchmark: ` titles. The nightly scheduler is stateless and intentionally dispatches every commit returned by each invocation, even when a matching title already exists. Because nightly commit volume is small, it dispatches the complete recent interval oldest first and does not share the backfill four-run throttle. + +The backfill state is replace-only: write a complete sibling temporary JSON file, flush it, and atomically replace the final path. Commit discovery is additive and must not reset dispatched records. The local process saves `dispatching` before calling `gh`, then saves returned run details. If interrupted at any point, repeating the command is safe. Full title reconciliation occurs at startup and periodically; recent-run polling is sufficient between full scans for capacity and newly dispatched runs. + +Do not run two backfill processes against the same state path. The CLI and documentation must warn about this. If the state file is corrupt or incompatible, preserve it, print its path and the reason, and stop. Starting over requires the operator to archive the old file explicitly; the scheduler can then reconstruct known workflow requests from deterministic GitHub titles before dispatching anything new. + +The backfill workflow creates a unique temporary clone per matrix job. Setup, configure, and build may be rerun because the clone is disposable. Reporter overlay happens only after build. Cleanup is allowed only after the path safety checks described above. Failure at any earlier step leaves the authoritative repository and scheduler state unchanged; GitHub's ordinary workflow rerun can repeat the same target SHA. + +## Artifacts and Notes + +The initial provisional boundary is: + + f3ad0fd91d8cf4eeb2be3b505765b43482aa952a 2025-09-02 System to allow tests to be isolated to specific platform (#478) + +Typical ordinary and backfill titles are: + + ci-benchmark: 0123456789abcdef0123456789abcdef01234567 + backfill-benchmark: f3ad0fd91d8cf4eeb2be3b505765b43482aa952a + +A typical exact ordinary dispatch created by `actions/github-script` is conceptually: + + {"ref":"main","inputs":{"revision":"0123456789abcdef0123456789abcdef01234567"},"return_run_details":true} + +A typical historical dispatch body is: + + {"ref":"main","inputs":{"target_sha":"f3ad0fd91d8cf4eeb2be3b505765b43482aa952a"},"return_run_details":true} + +The default state path is: + + .temp/benchmark-backfill-state.json + +The state contains commit identities, scheduling status, and workflow run references only. It contains no GitHub token and no BenchView key. + +## Interfaces and Dependencies + +No new Python package is added. Nightly scheduling uses `actions/github-script@v9` and the job's automatic GitHub token. The local backfill controller depends on GitHub CLI `gh`, authenticated with `gh auth login`, plus Python 3.10 or newer. BenchView configuration remains repository variable `BENCHVIEW_API_URL` and repository secret `BENCHVIEW_API_KEY`. + +`tools.benchmark_actions.GitHubCli` must expose typed methods equivalent to `list_commits(repository: str, branch: str, since: datetime, until: datetime) -> list[Commit]`, `list_workflow_runs(repository: str, workflow: str, maximum: int) -> list[WorkflowRun]`, and `dispatch_workflow(repository: str, workflow: str, workflow_ref: str, inputs: dict[str, str]) -> DispatchResult`. Its constructor accepts an injectable command runner for tests. Every function and method has typed arguments and a documenting docstring, as required by `AGENTS.md`. + +`tools.backfill_benchmarks.main(argv: Optional[list[str]] = None) -> int` is the only local scheduler entry point. `BackfillStateStore` atomically loads and saves the versioned state. Backfill scheduling helpers remain transport-independent so fake `GitHubCli` implementations can prove behavior without executing subprocesses. + +`slangpy.testing.benchmark.plugin.apply_benchmark_source_override(report: Report) -> None` remains the single source-identity override. `tools.ci.benchmark_python(args: Any)` remains the single benchmark runner. There is no `tools/run_benchmark_ci.py` in the completed tree. + +Revision note (2026-07-16, Codex): Created the first plan and prototype around one combined target-aware `ci-benchmark.yml`, a custom standard-library REST client, and a dedicated historical benchmark runner. + +Revision note (2026-07-17, Codex): Replaced that design after user review. The ordinary workflow is now validated first and contains no backfill behavior; nightly scheduling uses an optional exact revision only for commits containing BenchView; historical work moves to `backfill-benchmark.yml`; the normal current `ci.py` is overlaid after historical build; support begins at a demonstrated compatibility boundary; and the initial revision used the official `gh` CLI for all GitHub access. + +Revision note (2026-07-17, Codex): Simplified nightly scheduling after further user review. `schedule-benchmarks.yml` now uses `actions/github-script` and the automatic job token directly, so `tools/nightly_benchmarks.py`, checkout, Python setup, and `gh` setup are absent from the nightly path. The official `gh` adapter remains only for the resumable local backfill controller. + +Revision note (2026-07-17, Codex): Removed the remaining hosted-workflow overengineering after another user review. Nightly now dispatches every `main` commit from the preceding 24 hours without existing-run lookup or dry-run behavior. Historical clone, checkout, boundary, synchronization, and cleanup operations now use Git and native PowerShell/Bash rather than embedded Python. + +Revision note (2026-07-17, Codex): Implemented the local portion of Milestone 1. Restored the baseline scheduled/manual workflow shape, routed both performance jobs through the single current `tools/ci.py`, removed MongoDB and the rejected scheduling/alternate-runner prototype, added configuration and workflow regression tests, and completed all local gates. Left Milestone 1 live acceptance open until the uncommitted changes can run on GitHub's performance workers. diff --git a/.plans/benchview-benchmark-submission.md b/.plans/benchview-benchmark-submission.md new file mode 100644 index 000000000..660715ce1 --- /dev/null +++ b/.plans/benchview-benchmark-submission.md @@ -0,0 +1,165 @@ +# Submit SlangPy benchmark fixtures through the BenchView API + +This ExecPlan is a living document. The sections Progress, Surprises and Discoveries, Decision Log, and Outcomes and Retrospective must be kept up to date as work proceeds. + +This plan follows `.agents/PLANS.md` from the repository root. + +## Purpose / Big Picture + +After this change, SlangPy's pytest benchmark fixtures still measure and display results locally, but an enabled benchmark run sends native BenchView version-one JSON to the configured BenchView HTTP service instead of importing `pymongo` and writing a legacy report directly to MongoDB. Tests whose original pytest function name contains `_cpu` identify their metric as `cpu_time`; every other test uses `gpu_time`, matching the existing imported history. Both use milliseconds. Pytest parameters become legacy-compatible string case dimensions, source file plus test function remains the stable test identity, and machine/VCS facts use BenchView's dedicated environment and run fields. + +A developer can demonstrate the result without a GPU by running focused Python tests that build representative submissions and intercept the HTTP request. In production, `python tools/ci.py benchmark-python` receives a BenchView base URL, reads the write key only from the `BENCHVIEW_API_KEY` environment variable, runs each device shard, and submits batches to `/api/v1/submissions` with a Bearer authorization header. + +## Progress + +- [x] (2026-07-16 13:47Z) Read the repository benchmark instructions, current fixture/report/plugin lifecycle, CI wrapper, and BenchView's frozen version-one contract and existing SlangPy legacy mapping. +- [x] (2026-07-16 13:47Z) Chose the native observation, run identity, batching, authentication, and compatibility design recorded below. +- [x] (2026-07-16 14:05Z) Added native BenchView observation construction, bounded batching, nested-base URL handling, safe Bearer-authenticated HTTP submission, and parallel accumulation that retains local report comparison. +- [x] (2026-07-16 14:05Z) Replaced MongoDB pytest and `tools/ci.py` options with API-oriented options and environment-only key handling; removed the `pymongo` development dependency. +- [x] (2026-07-16 14:05Z) Added nine focused tests and updated benchmark documentation, including compatibility corrections for legacy string dimensions and `_cpu` metric selection. +- [x] (2026-07-16 14:13Z) Completed the Windows debug build, nine focused tests, Pyright with zero findings, the full pre-commit suite, `git diff --check`, CI help inspection, and a generated-payload injection through BenchView's real in-memory API with HTTP 201. +- [x] (2026-07-16 14:35Z) Moved the native BenchView protocol, batching, and HTTP producer into `slangpy/testing/benchmark/benchview.py`, leaving `report.py` responsible only for legacy local reports; revalidated nine focused tests, targeted Pyright, pre-commit, and whitespace checks. + +## Surprises and Discoveries + +- Observation: the existing `run_id` is supplied to each device-specific pytest process and the plugin uses it only as a legacy report field and MongoDB document value. + Evidence: `tools/ci.py` loops over device types and invokes `--benchmark-upload ` for every process, while `slangpy/testing/benchmark/plugin.py` assigns that value to `report["run_id"]` immediately before the direct insert. + +- Observation: timing mechanism alone does not determine the logical metric. Several `BenchmarkPythonFunction` cases synchronize GPU work, while explicitly CPU-oriented tests consistently contain `_cpu` in the original pytest function name. + Evidence: the existing importer used `_cpu` after product-owner clarification, and names in `test_benchmark_interop.py` and `test_benchmark_ppisp.py` distinguish synchronized GPU and CPU-overhead cases this way. + +- Observation: all historical pytest parameters were stringified before legacy storage, so emitting native integers or booleans would create different BenchView observation identities from the imported history. + Evidence: the old `ReportFixture` built `params` with `str(v)`, and the BenchView importer preserved those string types as case dimensions. + +- Observation: the existing Windows CMake cache could not regenerate until Visual Studio's Ninja and developer environment were supplied, then compilation required `/EHsc` because the regenerated cache combined standard-library exception warnings with `/WX`. + Evidence: the first build failed with a missing `CMAKE_MAKE_PROGRAM`, the second found MSVC but failed on C4530, and the same preset completed after configuring the local cache with Visual Studio Ninja and `/EHsc`. No repository build file changed. + +- Observation: a full-repository Pyright rerun after the module split reports seven unrelated profiler/UI attribute errors in `examples/pathtracer/pathtracer.py`. + Evidence: targeted Pyright over `benchview.py`, `report.py`, the fixture, plugin, and focused test reports zero errors and zero warnings; none of the full-run findings reference benchmark code. + +- Observation: saved local reports and comparison tables depend on the legacy `BenchmarkReport` shape. + Evidence: `slangpy/testing/benchmark/table.py` reads top-level `min`, `max`, `mean`, `median`, and `stddev`, and `.benchmarks` files are loaded back into that shape. + +## Decision Log + +- Decision: retain `BenchmarkReport` for local save/compare behavior and accumulate a parallel list of native BenchView observations using the same sample list. + Rationale: this makes the server path native without breaking existing local benchmark workflows or requiring a migration of `.benchmarks` files. + Date/Author: 2026-07-16, Codex. + +- Decision: isolate native BenchView payload and transport code in `slangpy/testing/benchmark/benchview.py`, while `report.py` remains the legacy local-report module. + Rationale: the two formats serve different persistence boundaries and keeping them separate makes it harder for future local-report changes to accidentally alter the API producer. + Date/Author: 2026-07-16, Codex. + +- Decision: use project `slangpy`, suite `python`, test identity `:`, and the existing stringified pytest parameters as case dimensions with `DeviceType.cuda`-style values normalized to `cuda`. + Rationale: these values and scalar types match the accepted legacy importer mapping, so new submissions continue the same graph identities. + Date/Author: 2026-07-16, Codex. + +- Decision: preserve the established `_cpu` function-name marker for metric selection instead of inferring semantics from the wrapper class. + Rationale: Python wrappers measure both synchronized GPU work and CPU dispatch overhead; changing every Python-wrapper result to `cpu_time` would split or mislabel existing histories. + Date/Author: 2026-07-16, Codex. + +- Decision: derive the logical run key from the full Git revision, suite, and the first 16 hexadecimal characters of a deterministic digest of `projectVersion` and `slangBuildTag` run dimensions. + Rationale: the key is independent of host, process, target, timestamp, or random state and matches the run-key policy used for already imported SlangPy data. + Date/Author: 2026-07-16, Codex. + +- Decision: treat the caller-provided run ID as `execution.requestId`, generate a fresh process execution UUID, and derive each batch idempotency key from its complete pre-key JSON payload. + Rationale: distributed processes share the logical run but retain traceable execution identity. Reposting the same in-memory request is an exact retry, while running the benchmarks again naturally creates new observations and therefore intentional replacements. + Date/Author: 2026-07-16, Codex. + +- Decision: use `urllib.request` from the Python standard library and obtain the write key only from `BENCHVIEW_API_KEY`. + Rationale: SlangPy gains no runtime HTTP or MongoDB dependency, and the secret does not appear in command-line arguments or logs. + Date/Author: 2026-07-16, Codex. + +- Decision: retain `--benchmark-upload` as an alias for the clearer `--benchmark-submit`, but remove the MongoDB connection and database options. + Rationale: existing direct pytest invocations keep working after replacing their destination option, while help and new CI commands describe the HTTP behavior accurately. + Date/Author: 2026-07-16, Codex. + +## Outcomes and Retrospective + +SlangPy benchmark fixtures now retain their local report/comparison behavior while also accumulating native BenchView observations. The legacy report model stays in `report.py`, while the native API producer has its own `benchview.py` module. Enabled sessions submit bounded authenticated HTTP batches below root or nested deployment bases, and direct MongoDB code, arguments, and the `pymongo` development dependency are gone. The producer preserves imported test, case, run, and metric histories rather than creating parallel graph identities. The implementation build, focused tests, pre-commit hooks, whitespace checks, CI help, and an in-memory request accepted by the real BenchView API pass; targeted typechecking of every changed benchmark file is clean, while the current full-repository Pyright run has unrelated pathtracer errors recorded above. A live benchmark run and production submission remain an operator action because they require a selected GPU target and the real write key. + +## Context and Orientation + +Before this change, `slangpy/testing/benchmark/fixtures.py` appended only a legacy `BenchmarkReport` to a private pytest configuration context. `slangpy/testing/benchmark/plugin.py` owned local save/compare options and inserted the completed legacy report directly into MongoDB. `slangpy/testing/benchmark/report.py` imported `pymongo` inside `upload_report`, `tools/ci.py` forwarded a credential-bearing connection string and database name, and `requirements-dev.txt` carried `pymongo` only for that path. The implemented `benchview.py` now owns native payload construction and HTTP submission; `report.py` retains only local report serialization. The fixture and plugin connect both paths, and `.github/instructions/benchmarks.instructions.md` is the updated user-facing guide. + +BenchView accepts `POST /api/v1/submissions` below either a root or nested deployment base. One submission contains a project, producer, logical run, concrete execution, and 1 through 500 complete observations. A logical run can be assembled by independent processes because its key excludes machine and process facts. An observation is replaced by a later submission with the same project, run key, test ID, and case dimensions. Exact HTTP retries reuse the same project-scoped idempotency key and body. Every passed observation has at least one explicitly named metric and unit. The request body limit is 8 MiB. + +The checked-in BenchView contract is not copied into SlangPy. This producer emits the small, deliberately selected subset described here and tests exact representative JSON. BenchView remains the authoritative validator at the HTTP boundary. + +## Plan of Work + +Add `slangpy/testing/benchmark/benchview.py` with small `dict[str, Any]` aliases rather than a parallel exhaustive contract hierarchy. Add helpers to normalize RFC 3339 timestamps, paths and scalar dimensions; map the existing machine dictionary into stable environment identity plus volatile telemetry; construct deterministic run fields; greedily batch observations below both a configurable observation count and the 8 MiB request limit; derive printable idempotency keys; and submit JSON with `urllib.request.Request`. Keep legacy report generation, serialization, and comparison helpers in `report.py`, but delete `upload_report` and all `pymongo` use. + +Extend `ReportFixture` in `slangpy/testing/benchmark/fixtures.py` so each measurement appends both its existing local report and a BenchView observation. Use the original pytest function's `_cpu` marker to select `cpu_time` and use `gpu_time` otherwise; both use unit `ms`, direction `lower`, and the raw samples distribution. Normalize the source path, preserve the legacy string parameter representation, and include the adapter name only as observation metadata because full stable GPU identity comes from the session machine record. + +Change the context and lifecycle in `slangpy/testing/benchmark/plugin.py`. Cache a fresh execution UUID and native observations. Validate API URL and `BENCHVIEW_API_KEY` during pytest configuration whenever submission is enabled, so a long benchmark run cannot discover missing configuration only at session end. At session finish, retain local save behavior and then build and post every batch. Print only the destination and accepted/duplicate counts, never the key or authorization header. + +Update `tools/ci.py` so `benchmark-python` accepts `--api-url` instead of MongoDB connection/database arguments and passes `--benchmark-submit` plus `--benchmark-api-url` to pytest. The environment, including `BENCHVIEW_API_KEY`, is inherited by each process. Remove `pymongo` from `requirements-dev.txt`. Update `.github/instructions/benchmarks.instructions.md` with direct pytest and CI examples, nested URL behavior, the environment key, native metric semantics, and retry/replacement behavior. + +Add pure tests under `slangpy/tests/utils/test_benchmark_submission.py`. Cover GPU and CPU observation construction, legacy-compatible dimensions and stable test ID, dual local/native accumulation, deterministic distributed run keys, environment identity/telemetry separation, batch sizing and unique idempotency, root and nested API endpoint construction, Bearer headers, duplicate receipts, and safe HTTP errors that do not disclose the key. These tests must require no GPU, MongoDB, or live BenchView instance. + +## Concrete Steps + +Run all commands from `C:\sw\slangpy`. Per repository policy, build before tests: + + cmake --build --preset windows-msvc-debug + pytest slangpy/tests/utils/test_benchmark_submission.py -v + python tools/ci.py typing-check-python + pre-commit run --all-files + +If pre-commit modifies files, run it again. Inspect `git diff --check` and the focused diff before handoff. A live manual submission, when desired, uses: + + $env:BENCHVIEW_API_KEY = "" + pytest slangpy/benchmarks -v --benchmark-submit --benchmark-api-url http://localhost:3000 + +For the hosted nested deployment, replace the base URL with `http://rtrci.nvidia.com/benchview`; the producer appends `/api/v1/submissions` without discarding `/benchview`. + +## Validation and Acceptance + +The focused tests pass without importing `pymongo` or contacting external services. Their intercepted request has `Content-Type: application/json`, `Authorization: Bearer `, and a URL ending in the correct nested `/api/v1/submissions`. The payload uses schema version 1, project `slangpy`, suite `python`, one stable commit/config run key across distinct execution IDs, normalized device dimensions, stable source/function test IDs, environment identity, and `gpu_time` or `cpu_time` millisecond samples as appropriate. + +Starting a submission-enabled pytest process without `--benchmark-api-url`/`BENCHVIEW_API_URL` or without `BENCHVIEW_API_KEY` fails during configuration with a concise diagnostic. A non-2xx response fails submission with status and a bounded response message but never includes the key. A 200 duplicate receipt and a 201 accepted receipt are both successes. No production test depends on MongoDB or the BenchView repository. + +The Windows debug build completes before tests. The focused pytest suite, Python typing command, pre-commit suite, and `git diff --check` exit zero. `rg pymongo slangpy/testing/benchmark tools/ci.py requirements-dev.txt` finds no remaining direct database upload implementation or dependency. + +Final evidence on 2026-07-16: `cmake --build --preset windows-msvc-debug` completed after supplying Visual Studio's installed Ninja and `/EHsc` to the local regenerated cache; `python -m pytest slangpy/tests/utils/test_benchmark_submission.py -v` passed 9 tests; `python -m pyright` reported 0 errors and 0 warnings; and `python -m pre_commit run --all-files` passed every hook. A representative generated request sent through `apps/api/src/app.ts` with an in-memory BenchView transaction log returned HTTP 201, cursor 0, and observation count 1. That check used a synthetic key and did not connect to MongoDB. + +Separation evidence on 2026-07-16: the focused suite again passed 9 tests, targeted Pyright over the changed benchmark modules and test reported 0 errors and 0 warnings, both the full tracked-file pre-commit run and the explicit new-file run passed, and `git diff --check` completed cleanly. A full Pyright rerun currently reports only the unrelated pathtracer findings recorded under Surprises and Discoveries. + +## Idempotence and Recovery + +Payload generation is side-effect free. The HTTP sender posts immutable serialized bodies; an in-process retry sends the same body and idempotency key. If a process is rerun, its fresh execution UUID and timestamps produce fresh keys, so BenchView treats results as intentional replacements. Batches are independent transactions, so a later batch can fail after earlier batches commit. Rerunning the benchmark safely resubmits a complete new set; retrying a captured failed batch safely reuses its original body. + +Local `.benchmarks` save and comparison files retain their legacy shape. No migration or deletion is performed. The implementation removes only the obsolete direct database code and its unused development dependency. + +## Artifacts and Notes + +The intended native metric core is compact: + + "metrics": [{ + "id": "gpu_time", + "name": "GPU time", + "unit": "ms", + "direction": "lower", + "distribution": {"samples": [...]} + }] + +The logical run key has the form: + + git:/suite:python/config:<16 lowercase digest characters> + +The API key is read from `BENCHVIEW_API_KEY` only and is never persisted in a report or plan. + +## Interfaces and Dependencies + +`slangpy.testing.benchmark.benchview` exposes `BenchViewObservation = dict[str, Any]`, `build_benchview_observation(...)`, `build_benchview_submissions(...)`, and `submit_benchview_submissions(...)`. `slangpy.testing.benchmark.report` continues to expose the legacy `BenchmarkReport`, `Report`, and local serialization helpers. All arguments have type annotations. HTTP uses `urllib.request`; hashing and JSON use `hashlib` and `json`; execution IDs use `uuid`. No third-party dependency is added. + +`ReportFixture.__call__` accepts optional `metric_id` and `metric_name` overrides and otherwise selects the historical `_cpu`/GPU mapping. The pytest context includes `benchmark_observations` and `execution_id`. `tools/ci.py benchmark-python` exposes `--api-url`; `BENCHVIEW_API_KEY` remains an inherited environment variable rather than an argparse option. + +Revision note (2026-07-16, Codex): Created this ExecPlan before implementation because replacing a benchmark storage protocol affects fixture data modeling, pytest lifecycle, CI arguments, security, compatibility, and validation across several SlangPy modules. + +Revision note (2026-07-16, Codex): Corrected the initial native-type and wrapper-based metric assumptions after comparing them with imported production history. Native submissions now preserve string parameter identities and the `_cpu` marker, preventing graph splits or mislabeled synchronized GPU benchmarks. + +Revision note (2026-07-16, Codex): Completed implementation and validation. Added native observation accumulation, deterministic run/idempotency construction, environment mapping, size-bounded batching, nested-base authenticated HTTP, early configuration errors, API-oriented CI options, documentation, and nine focused tests; removed direct MongoDB integration and recorded the successful real BenchView in-memory acceptance check. + +Revision note (2026-07-16, Codex): Separated the native BenchView producer into `benchview.py` and returned `report.py` to its legacy local-report responsibility; updated imports, tests, documentation, and this plan to describe the module boundary. diff --git a/external/slang-rhi b/external/slang-rhi index 1a9768741..7cf010c22 160000 --- a/external/slang-rhi +++ b/external/slang-rhi @@ -1 +1 @@ -Subproject commit 1a9768741246b1f356116d5dfa305933034ce049 +Subproject commit 7cf010c22a561c40e5449c25a0306f057cb173b7 diff --git a/requirements-dev.txt b/requirements-dev.txt index b70573202..89cd6070c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,6 +9,5 @@ deepdiff isort autoflake libcst -pymongo setuptools imageio diff --git a/slangpy/benchmarks/test_benchmark_autograd.py b/slangpy/benchmarks/test_benchmark_autograd.py index 524794d8e..0d955afab 100644 --- a/slangpy/benchmarks/test_benchmark_autograd.py +++ b/slangpy/benchmarks/test_benchmark_autograd.py @@ -183,7 +183,7 @@ def forward(ctx: Any, a: float, b: float, c: float, x: torch.Tensor) -> torch.Te x = x.detach() result = torch.empty_like(x) poly_func(a, b, c, x, _result=result) - ctx.save_for_backward(x) + ctx.save_for_backward(x, result) ctx.a = a ctx.b = b ctx.c = c @@ -193,10 +193,10 @@ def forward(ctx: Any, a: float, b: float, c: float, x: torch.Tensor) -> torch.Te def backward( ctx: Any, grad_output: torch.Tensor ) -> tuple[None, None, None, Optional[torch.Tensor]]: - (x,) = ctx.saved_tensors + x, result = ctx.saved_tensors grad_x = torch.zeros_like(x) x_pair = NativeTorchTensorDiffPair(x, grad_x, 0, True) - result_pair = NativeTorchTensorDiffPair(None, grad_output, 1, False) + result_pair = NativeTorchTensorDiffPair(result, grad_output, 1, False) poly_func.bwds(ctx.a, ctx.b, ctx.c, x_pair, _result=result_pair) return None, None, None, grad_x diff --git a/slangpy/benchmarks/test_benchmark_bwd_diff.py b/slangpy/benchmarks/test_benchmark_bwd_diff.py index bc436fa42..bb08128d7 100644 --- a/slangpy/benchmarks/test_benchmark_bwd_diff.py +++ b/slangpy/benchmarks/test_benchmark_bwd_diff.py @@ -44,7 +44,7 @@ CORRECTNESS_N = 64 BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) -EXTENSIONS_DIR = os.path.join(os.path.dirname(BENCH_DIR), "..").replace("\\", "/") +EXTENSIONS_DIR = os.path.join(BENCH_DIR, "ppisp").replace("\\", "/") W_VAL = 2.0 diff --git a/slangpy/benchmarks/test_benchmark_ppisp.py b/slangpy/benchmarks/test_benchmark_ppisp.py index 0c0da056e..791523a2e 100644 --- a/slangpy/benchmarks/test_benchmark_ppisp.py +++ b/slangpy/benchmarks/test_benchmark_ppisp.py @@ -37,6 +37,7 @@ RESOLUTION_W = 1920 RESOLUTION_H = 1080 BATCH_SIZES = [100_000, 1_000_000] +DEVICE_TYPES = [spy.DeviceType.cuda] # Fixture parameters: 10 outer x 100 inner = 1000 total timed calls ITERATIONS = 10 @@ -146,10 +147,11 @@ def _assert_close( @pytest.mark.skip(reason="Correctness validated; enable manually when needed") @pytest.mark.parametrize("include_pytorch", [False, True], ids=["slang-only", "with-pytorch"]) -def test_ppisp_correctness_forward(include_pytorch: bool) -> None: +@pytest.mark.parametrize("device_type", DEVICE_TYPES) +def test_ppisp_correctness_forward(device_type: spy.DeviceType, include_pytorch: bool) -> None: """Verify forward outputs match across backends.""" _skip_if_no_slangtorch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") torch.manual_seed(42) @@ -188,10 +190,11 @@ def test_ppisp_correctness_forward(include_pytorch: bool) -> None: @pytest.mark.skip(reason="Correctness validated; enable manually when needed") @pytest.mark.parametrize("include_pytorch", [False, True], ids=["slang-only", "with-pytorch"]) -def test_ppisp_correctness_backward(include_pytorch: bool) -> None: +@pytest.mark.parametrize("device_type", DEVICE_TYPES) +def test_ppisp_correctness_backward(device_type: spy.DeviceType, include_pytorch: bool) -> None: """Verify gradients match across backends.""" _skip_if_no_slangtorch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") torch.manual_seed(42) @@ -254,12 +257,14 @@ def test_ppisp_correctness_backward(include_pytorch: bool) -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_forward_pytorch( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_pytorch import PPISPPyTorch @@ -285,12 +290,14 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_forward_slangpy( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangpy import PPISPSlangPy @@ -326,12 +333,14 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_forward_slangtorch( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_slangtorch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangtorch import PPISPSlangtorch @@ -365,12 +374,14 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_backward_pytorch( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_pytorch import PPISPPyTorch @@ -397,12 +408,14 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_backward_slangpy( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangpy import PPISPSlangPy @@ -439,12 +452,14 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_backward_slangtorch( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: _skip_if_no_slangtorch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangtorch import PPISPSlangtorch @@ -474,7 +489,9 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_backward_slangpy_manual_hook( + device_type: spy.DeviceType, batch_size: int, benchmark_python_function: BenchmarkPythonFunction, ) -> None: @@ -484,7 +501,7 @@ def test_ppisp_backward_slangpy_manual_hook( the automatic autograd integration vs doing it manually. """ _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from typing import Any, Optional @@ -515,28 +532,27 @@ def forward( crf: torch.Tensor, ) -> torch.Tensor: # Detach all to avoid triggering SlangPy's automatic autograd - ctx.save_for_backward( - rgb.detach(), - exposure.detach(), - vignetting.detach(), - color.detach(), - crf.detach(), - ) + rgb = rgb.detach() + exposure = exposure.detach() + vignetting = vignetting.detach() + color = color.detach() + crf = crf.detach() result = func( batch_size=rgb.shape[0], num_cameras=NUM_CAMERAS, num_frames=NUM_FRAMES, - exposure_params=exposure.detach(), - vignetting_params=vignetting.detach(), - color_params=color.detach(), - crf_params=crf.detach(), - rgb_pixel=rgb.detach(), + exposure_params=exposure, + vignetting_params=vignetting, + color_params=color, + crf_params=crf, + rgb_pixel=rgb, pixel_coord=pixel_coords, camera_idx=camera_idcs, frame_idx=frame_idcs, resolution_w=float(RESOLUTION_W), resolution_h=float(RESOLUTION_H), ) + ctx.save_for_backward(rgb, exposure, vignetting, color, crf, result) return result @staticmethod @@ -544,7 +560,7 @@ def backward( ctx: Any, grad_output: torch.Tensor, ) -> tuple[Optional[torch.Tensor], ...]: - rgb, exposure, vignetting, color, crf = ctx.saved_tensors + rgb, exposure, vignetting, color, crf, result = ctx.saved_tensors # Build diff pairs: (primal, grad_buffer, index, is_input) exposure_pair = NativeTorchTensorDiffPair(exposure, torch.zeros_like(exposure), 0, True) vignetting_pair = NativeTorchTensorDiffPair( @@ -553,7 +569,7 @@ def backward( color_pair = NativeTorchTensorDiffPair(color, torch.zeros_like(color), 2, True) crf_pair = NativeTorchTensorDiffPair(crf, torch.zeros_like(crf), 3, True) rgb_pair = NativeTorchTensorDiffPair(rgb, torch.zeros_like(rgb), 4, True) - result_pair = NativeTorchTensorDiffPair(None, grad_output, 5, False) + result_pair = NativeTorchTensorDiffPair(result, grad_output, 5, False) func.bwds( batch_size=rgb.shape[0], @@ -619,12 +635,14 @@ def run() -> None: CPU_OVERHEAD_WARMUPS = 10 +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_cpu_overhead_slangpy( + device_type: spy.DeviceType, benchmark_python_function: BenchmarkPythonFunction, ) -> None: """Measure SlangPy CPU dispatch overhead for PPISP (forward + backward).""" _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangpy import PPISPSlangPy @@ -666,12 +684,14 @@ def run() -> None: ) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_cpu_overhead_slangtorch( + device_type: spy.DeviceType, benchmark_python_function: BenchmarkPythonFunction, ) -> None: """Measure slangtorch CPU dispatch overhead for PPISP (forward + backward).""" _skip_if_no_slangtorch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangtorch import PPISPSlangtorch @@ -716,13 +736,15 @@ def run() -> None: @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_gpu_forward_slangpy( + device_type: spy.DeviceType, batch_size: int, benchmark_slang_function: BenchmarkSlangFunction, ) -> None: """GPU-timed SlangPy PPISP forward pass (timestamp queries, no CPU overhead).""" _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangpy import _get_slang_module, _warmup @@ -773,13 +795,15 @@ def test_ppisp_gpu_forward_slangpy( @pytest.mark.parametrize("batch_size", BATCH_SIZES) +@pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_ppisp_gpu_backward_slangpy( + device_type: spy.DeviceType, batch_size: int, benchmark_slang_function: BenchmarkSlangFunction, ) -> None: """GPU-timed SlangPy PPISP backward pass (timestamp queries, no CPU overhead).""" _skip_if_no_torch() - device = helpers.get_torch_device(spy.DeviceType.cuda) + device = helpers.get_torch_device(device_type) torch_device = torch.device("cuda") from slangpy.benchmarks.ppisp.ppisp_slangpy import _get_slang_module, _warmup @@ -816,9 +840,24 @@ def test_ppisp_gpu_backward_slangpy( crf_pair = NativeTorchTensorDiffPair(crf_params, torch.zeros_like(crf_params), 3, True) rgb_pair = NativeTorchTensorDiffPair(rgb, torch.zeros_like(rgb), 4, True) - # Upstream gradient (ones) - result_grad = torch.ones(batch_size, 3, device=torch_device) - result_pair = NativeTorchTensorDiffPair(None, result_grad, 5, False) + # Run an untimed forward pass to provide the output primal needed by bwds(). + result = func( + batch_size=batch_size, + num_cameras=NUM_CAMERAS, + num_frames=NUM_FRAMES, + exposure_params=exposure_params, + vignetting_params=vignetting_params, + color_params=color_params, + crf_params=crf_params, + rgb_pixel=rgb, + pixel_coord=pixel_coords, + camera_idx=camera_idcs, + frame_idx=frame_idcs, + resolution_w=float(RESOLUTION_W), + resolution_h=float(RESOLUTION_H), + ) + result_grad = torch.ones_like(result) + result_pair = NativeTorchTensorDiffPair(result, result_grad, 5, False) benchmark_slang_function( device, diff --git a/slangpy/testing/benchmark/benchview.py b/slangpy/testing/benchmark/benchview.py new file mode 100644 index 000000000..a2290ebb2 --- /dev/null +++ b/slangpy/testing/benchmark/benchview.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from datetime import datetime, timezone +import hashlib +from http.client import HTTPException +import json +import math +import os +from pathlib import Path +from time import sleep +from typing import Any, Optional +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +BENCHVIEW_MAX_BODY_BYTES = 8 * 1024 * 1024 +BENCHVIEW_DEFAULT_BATCH_SIZE = 100 +BENCHVIEW_DEFAULT_MAX_SUBMIT_ATTEMPTS = 5 +BENCHVIEW_DEFAULT_RETRY_DELAY_SECONDS = 1.0 +BENCHVIEW_PROJECT_ID = "slangpy" +BENCHVIEW_REPOSITORY = "https://github.com/shader-slang/slangpy" +BENCHVIEW_SUITE_ID = "python" +BENCHVIEW_RETRYABLE_HTTP_STATUS_CODES = frozenset({408, 429, 500, 502, 503, 504}) + +BenchViewObservation = dict[str, Any] +BenchViewSubmission = dict[str, Any] + + +class BenchmarkSubmissionError(RuntimeError): + """Report a safe BenchView payload or HTTP submission failure.""" + + +def _rfc3339(value: datetime) -> str: + """Convert a datetime to the UTC spelling emitted by native submissions.""" + + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _json_bytes(value: Any) -> bytes: + """Serialize finite JSON deterministically for sizing and producer digests.""" + + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _nonempty_string(value: Any) -> Optional[str]: + """Return a non-empty string representation or omit the value.""" + + if value is None: + return None + text = str(value) + return text if text else None + + +def _normalize_source_path(filename: str) -> str: + """Prefer a repository-relative slash-separated source path for test identity.""" + + source = Path(filename) + if source.is_absolute(): + try: + source = source.resolve().relative_to(Path.cwd().resolve()) + except ValueError: + pass + return source.as_posix().removeprefix("./") + + +def _normalize_dimension(name: str, value: Any) -> str: + """Preserve legacy string dimensions while shortening device enum text.""" + + normalized = str(value) + if name == "device_type": + enum_name = getattr(value, "name", None) + if normalized.startswith("DeviceType."): + normalized = normalized.removeprefix("DeviceType.") + elif isinstance(enum_name, str) and enum_name: + normalized = enum_name + return normalized + + +def build_benchview_observation( + filename: str, + function_name: str, + display_name: str, + parameters: dict[str, Any], + samples: list[float], + observed_at: datetime, + metric_id: str, + metric_name: str, + adapter_name: Optional[str] = None, + source_line: Optional[int] = None, +) -> BenchViewObservation: + """Build one passed native BenchView observation from a measured pytest case.""" + + if not samples or any(not math.isfinite(sample) for sample in samples): + raise BenchmarkSubmissionError("Benchmark samples must be a non-empty finite list.") + source_file = _normalize_source_path(filename) + dimensions = {name: _normalize_dimension(name, value) for name, value in parameters.items()} + source: dict[str, Any] = {"file": source_file, "function": function_name} + if source_line is not None and source_line >= 1: + source["line"] = source_line + observation: BenchViewObservation = { + "test": { + "id": f"{source_file}:{function_name}", + "name": function_name, + "source": source, + }, + "observedAt": _rfc3339(observed_at), + "status": "passed", + "metrics": [ + { + "id": metric_id, + "name": metric_name, + "unit": "ms", + "direction": "lower", + "distribution": {"samples": samples}, + } + ], + } + if dimensions: + observation["case"] = {"dimensions": dimensions} + if display_name != function_name or adapter_name: + metadata: dict[str, Any] = {} + if display_name != function_name: + metadata["pytestName"] = display_name + if adapter_name: + metadata["adapterName"] = adapter_name + observation["metadata"] = metadata + return observation + + +def _build_benchview_environment(machine_info: dict[str, Any]) -> dict[str, Any]: + """Separate stable machine identity from volatile GPU telemetry.""" + + identity: dict[str, Any] = {} + machine = _nonempty_string(machine_info.get("node")) + if machine: + identity["machine"] = machine + + os_identity: dict[str, Any] = {} + for target, source in ( + ("name", "system"), + ("version", "version"), + ("architecture", "machine"), + ): + value = _nonempty_string(machine_info.get(source)) + if value: + os_identity[target] = value + if os_identity: + identity["os"] = os_identity + + cpu: dict[str, Any] = {} + processor = _nonempty_string(machine_info.get("processor")) + architecture = _nonempty_string(machine_info.get("machine")) + if processor: + cpu["model"] = processor + if architecture: + cpu["architecture"] = architecture + logical_cores = os.cpu_count() + if logical_cores is not None and logical_cores > 0: + cpu["logicalCores"] = logical_cores + if cpu: + identity["cpu"] = cpu + + gpu_values = machine_info.get("gpus") + if not isinstance(gpu_values, list): + gpu_values = [] + gpu_identities: list[dict[str, Any]] = [] + gpu_telemetry: list[dict[str, Any]] = [] + for fallback_index, gpu_value in enumerate(gpu_values): + if not isinstance(gpu_value, dict): + continue + name = _nonempty_string(gpu_value.get("name")) + if not name: + continue + index_value = gpu_value.get("index", fallback_index) + index = index_value if isinstance(index_value, int) and index_value >= 0 else fallback_index + gpu: dict[str, Any] = {"index": index, "name": name} + uuid = _nonempty_string(gpu_value.get("uuid")) + driver_version = _nonempty_string(gpu_value.get("driver_version")) + if uuid: + gpu["uuid"] = uuid + if driver_version: + gpu["driverVersion"] = driver_version + memory_total = gpu_value.get("memory_total") + if isinstance(memory_total, (int, float)) and math.isfinite(memory_total): + gpu["memoryBytes"] = max(0, round(memory_total * 1024 * 1024)) + gpu_identities.append(gpu) + + telemetry: dict[str, Any] = {"index": index} + for field in ( + "utilization", + "memory_used", + "clock_current_graphics", + "clock_current_memory", + "clock_max_graphics", + "clock_max_memory", + "temperature", + ): + value = gpu_value.get(field) + if isinstance(value, (str, bool, int)) or ( + isinstance(value, float) and math.isfinite(value) + ): + telemetry[field] = value + if len(telemetry) > 1: + gpu_telemetry.append(telemetry) + if gpu_identities: + identity["gpus"] = gpu_identities + + attributes: dict[str, Any] = {} + for target, source in ( + ("pythonImplementation", "python_implementation"), + ("pythonVersion", "python_version"), + ("pythonCompiler", "python_compiler"), + ("osRelease", "release"), + ): + value = _nonempty_string(machine_info.get(source)) + if value: + attributes[target] = value + for index, gpu_value in enumerate(gpu_values): + if isinstance(gpu_value, dict): + serial = _nonempty_string(gpu_value.get("serial_number")) + if serial: + attributes[f"gpu{index}SerialNumber"] = serial + if attributes: + identity["attributes"] = attributes + + environment: dict[str, Any] = {} + if identity: + environment["identity"] = identity + if gpu_telemetry: + environment["telemetry"] = {"gpus": gpu_telemetry} + return environment + + +def _build_submission_base( + request_id: str, + execution_id: str, + project_info: dict[str, Any], + commit_info: dict[str, Any], +) -> BenchViewSubmission: + """Build fields shared by every batch from one pytest benchmark process.""" + + revision = _nonempty_string(commit_info.get("id")) or "unknown" + dimensions: dict[str, Any] = {} + project_version = _nonempty_string(project_info.get("version")) + slang_build_tag = _nonempty_string(project_info.get("slang_build_tag")) + if project_version: + dimensions["projectVersion"] = project_version + if slang_build_tag: + dimensions["slangBuildTag"] = slang_build_tag + config_digest = hashlib.sha256(_json_bytes(dimensions)).hexdigest()[:16] + + vcs: dict[str, Any] = {"repository": BENCHVIEW_REPOSITORY, "revision": revision} + commit_time = commit_info.get("time") or commit_info.get("author_time") + if isinstance(commit_time, datetime): + vcs["commitTime"] = _rfc3339(commit_time) + elif _nonempty_string(commit_time): + vcs["commitTime"] = str(commit_time) + branch = _nonempty_string(commit_info.get("branch")) + if branch: + vcs["branch"] = branch + if isinstance(commit_info.get("dirty"), bool): + vcs["dirty"] = commit_info["dirty"] + + run: dict[str, Any] = { + "key": f"git:{revision}/suite:{BENCHVIEW_SUITE_ID}/config:{config_digest}", + "suite": {"id": BENCHVIEW_SUITE_ID, "name": "Python benchmarks"}, + "vcs": vcs, + } + if dimensions: + run["dimensions"] = dimensions + return { + "schemaVersion": 1, + "project": {"id": BENCHVIEW_PROJECT_ID, "name": "SlangPy"}, + "producer": {"name": "slangpy-benchmark-plugin", "version": "1.0.0"}, + "run": run, + "execution": {"id": execution_id, "requestId": request_id}, + } + + +def _finalize_submission( + base: BenchViewSubmission, observations: list[BenchViewObservation] +) -> BenchViewSubmission: + """Add deterministic idempotency to one complete native request body.""" + + content = {**base, "observations": observations} + digest = hashlib.sha256(_json_bytes(content)).hexdigest() + return {"schemaVersion": 1, "idempotencyKey": f"slangpy/{digest}", **content} + + +def build_benchview_submissions( + observations: list[BenchViewObservation], + request_id: str, + execution_id: str, + project_info: dict[str, Any], + machine_info: dict[str, Any], + commit_info: dict[str, Any], + batch_size: int = BENCHVIEW_DEFAULT_BATCH_SIZE, + max_body_bytes: int = BENCHVIEW_MAX_BODY_BYTES, +) -> list[BenchViewSubmission]: + """Attach shared run/environment facts and greedily form valid API-sized batches.""" + + if not request_id or not execution_id: + raise BenchmarkSubmissionError("Benchmark request and execution IDs must be non-empty.") + if batch_size < 1 or batch_size > 500: + raise BenchmarkSubmissionError("BenchView batch size must be between 1 and 500.") + if max_body_bytes < 1: + raise BenchmarkSubmissionError("BenchView body limit must be positive.") + + base = _build_submission_base(request_id, execution_id, project_info, commit_info) + environment = _build_benchview_environment(machine_info) + prepared: list[BenchViewObservation] = [] + for observation in observations: + item = dict(observation) + if environment: + item["environment"] = environment + prepared.append(item) + + submissions: list[BenchViewSubmission] = [] + batch: list[BenchViewObservation] = [] + for observation in prepared: + candidate = [*batch, observation] + candidate_submission = _finalize_submission(base, candidate) + if ( + len(candidate) <= batch_size + and len(_json_bytes(candidate_submission)) <= max_body_bytes + ): + batch = candidate + continue + if not batch: + raise BenchmarkSubmissionError("One benchmark observation exceeds the API body limit.") + submissions.append(_finalize_submission(base, batch)) + batch = [observation] + if len(_json_bytes(_finalize_submission(base, batch))) > max_body_bytes: + raise BenchmarkSubmissionError("One benchmark observation exceeds the API body limit.") + if batch: + submissions.append(_finalize_submission(base, batch)) + return submissions + + +def benchview_submission_url(api_base_url: str) -> str: + """Resolve the submission endpoint without discarding a nested deployment path.""" + + try: + parsed = urlsplit(api_base_url) + except ValueError as error: + raise BenchmarkSubmissionError( + "BenchView API base URL must be an absolute HTTP URL." + ) from error + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise BenchmarkSubmissionError("BenchView API base URL must be an absolute HTTP URL.") + if parsed.username is not None or parsed.password is not None: + raise BenchmarkSubmissionError("BenchView API base URL must not contain credentials.") + if parsed.query or parsed.fragment: + raise BenchmarkSubmissionError( + "BenchView API base URL must not contain query or fragment text." + ) + return api_base_url.rstrip("/") + "/api/v1/submissions" + + +def _safe_response_text(data: bytes, write_key: str) -> str: + """Decode a bounded server diagnostic while redacting the configured credential.""" + + text = data[:2048].decode("utf-8", errors="replace") + return text.replace(write_key, "") + + +def _retry_delay(attempt: int, base_delay_seconds: float) -> float: + """Return the deterministic exponential delay after a failed one-based attempt.""" + + return base_delay_seconds * (2 ** (attempt - 1)) + + +def _retry_submission(attempt: int, max_attempts: int, delay_seconds: float, failure: str) -> None: + """Report a transient failure and wait before another idempotent POST.""" + + if attempt < max_attempts: + print( + f"BenchView submission {failure} on attempt {attempt}/{max_attempts}; " + f"retrying in {delay_seconds:g} second(s)." + ) + sleep(delay_seconds) + + +def submit_benchview_submissions( + api_base_url: str, + write_key: str, + submissions: list[BenchViewSubmission], + timeout_seconds: float = 30.0, + max_attempts: int = BENCHVIEW_DEFAULT_MAX_SUBMIT_ATTEMPTS, + retry_delay_seconds: float = BENCHVIEW_DEFAULT_RETRY_DELAY_SECONDS, +) -> list[dict[str, Any]]: + """POST immutable batches, retrying transient failures with the same idempotency key.""" + + if not write_key: + raise BenchmarkSubmissionError("BenchView API write key must be non-empty.") + if timeout_seconds <= 0 or not math.isfinite(timeout_seconds): + raise BenchmarkSubmissionError("BenchView submission timeout must be positive and finite.") + if max_attempts < 1: + raise BenchmarkSubmissionError("BenchView submission attempts must be positive.") + if retry_delay_seconds < 0 or not math.isfinite(retry_delay_seconds): + raise BenchmarkSubmissionError("BenchView retry delay must be non-negative and finite.") + endpoint = benchview_submission_url(api_base_url) + receipts: list[dict[str, Any]] = [] + for submission in submissions: + body = _json_bytes(submission) + for attempt in range(1, max_attempts + 1): + request = Request( + endpoint, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {write_key}", + "Content-Type": "application/json", + }, + ) + try: + with urlopen(request, timeout=timeout_seconds) as response: + status = response.getcode() + response_body = response.read(BENCHVIEW_MAX_BODY_BYTES + 1) + except HTTPError as error: + if error.code in BENCHVIEW_RETRYABLE_HTTP_STATUS_CODES and attempt < max_attempts: + error.close() + _retry_submission( + attempt, + max_attempts, + _retry_delay(attempt, retry_delay_seconds), + f"received HTTP {error.code}", + ) + continue + diagnostic = _safe_response_text(error.read(2049), write_key) + raise BenchmarkSubmissionError( + f"BenchView rejected a benchmark submission with HTTP {error.code} " + f"after {attempt} attempt(s): {diagnostic}" + ) from error + except (URLError, TimeoutError, ConnectionError, HTTPException) as error: + if attempt < max_attempts: + _retry_submission( + attempt, + max_attempts, + _retry_delay(attempt, retry_delay_seconds), + "lost its connection", + ) + continue + reason = error.reason if isinstance(error, URLError) else error + raise BenchmarkSubmissionError( + f"Could not reach the BenchView submission endpoint after {attempt} " + f"attempt(s): {reason}" + ) from error + break + if status < 200 or status >= 300: + diagnostic = _safe_response_text(response_body, write_key) + raise BenchmarkSubmissionError( + f"BenchView rejected a benchmark submission with HTTP {status}: {diagnostic}" + ) + try: + receipt = json.loads(response_body) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise BenchmarkSubmissionError("BenchView returned an invalid JSON receipt.") from error + if not isinstance(receipt, dict) or not isinstance(receipt.get("duplicate"), bool): + raise BenchmarkSubmissionError("BenchView returned an invalid submission receipt.") + receipts.append(receipt) + return receipts diff --git a/slangpy/testing/benchmark/fixtures.py b/slangpy/testing/benchmark/fixtures.py index 7a6d0f457..4250c9468 100644 --- a/slangpy/testing/benchmark/fixtures.py +++ b/slangpy/testing/benchmark/fixtures.py @@ -6,8 +6,9 @@ import numpy as np from typing import Any, Callable, Optional, Union from time import time, sleep -from datetime import datetime +from datetime import datetime, timezone +from .benchview import build_benchview_observation from .report import BenchmarkReport DEFAULT_ITERATIONS = 2000 @@ -26,6 +27,8 @@ def __call__( device: Optional[spy.Device], data: list[float], cpu_time: float, + metric_id: Optional[str] = None, + metric_name: Optional[str] = None, **kwargs: Any, ) -> None: """Generate and store a benchmark report with the given data.""" @@ -50,15 +53,23 @@ def __call__( trimmed_data = sorted_data trimmed_mean = float(np.mean(trimmed_data)) + observed_at = datetime.now(timezone.utc) + samples = [float(d) for d in data] + filename = str(self.node.location[0]).replace("\\", "/") + function_name = self.node.originalname + if metric_id is None: + metric_id = "cpu_time" if "_cpu" in function_name else "gpu_time" + if metric_name is None: + metric_name = "CPU time" if metric_id == "cpu_time" else "GPU time" report: BenchmarkReport = { "name": self.node.name, - "filename": str(self.node.location[0]).replace("\\", "/"), - "function": self.node.originalname, + "filename": filename, + "function": function_name, "params": params, "meta": meta, - "timestamp": datetime.now(), + "timestamp": observed_at, "cpu_time": cpu_time, - "data": [float(d) for d in data], + "data": samples, "min": float(np.min(data)), "max": float(np.max(data)), "mean": trimmed_mean, @@ -67,6 +78,20 @@ def __call__( } self.config._benchmark_context["benchmark_reports"].append(report) # type: ignore + self.config._benchmark_context["benchmark_observations"].append( # type: ignore + build_benchview_observation( + filename=filename, + function_name=function_name, + display_name=self.node.name, + parameters=params, + samples=samples, + observed_at=observed_at, + metric_id=metric_id, + metric_name=metric_name, + adapter_name=meta.get("adapter_name"), + source_line=int(self.node.location[1]) + 1, + ) + ) class BenchmarkSlangFunction: @@ -106,7 +131,14 @@ def __call__( cpu_time = end_time - start_time # Use the report fixture to generate and store the report - self.report_fixture(device, deltas, cpu_time, **kwargs) + self.report_fixture( + device, + deltas, + cpu_time, + metric_id="gpu_time", + metric_name="GPU time", + **kwargs, + ) class BenchmarkComputeKernel: @@ -147,7 +179,14 @@ def __call__( cpu_time = end_time - start_time # Use the report fixture to generate and store the report - self.report_fixture(device, deltas, cpu_time, **kwargs) + self.report_fixture( + device, + deltas, + cpu_time, + metric_id="gpu_time", + metric_name="GPU time", + **kwargs, + ) class BenchmarkPythonFunction: diff --git a/slangpy/testing/benchmark/plugin.py b/slangpy/testing/benchmark/plugin.py index 15817fe9f..686f939fd 100644 --- a/slangpy/testing/benchmark/plugin.py +++ b/slangpy/testing/benchmark/plugin.py @@ -2,17 +2,25 @@ import pytest from datetime import datetime +import os from pathlib import Path - +from uuid import uuid4 + +from .benchview import ( + BenchViewObservation, + BenchmarkSubmissionError, + benchview_submission_url, + build_benchview_submissions, + submit_benchview_submissions, +) from .report import ( - Report, BenchmarkReport, + Report, generate_report, generate_run_id, list_report_ids, - write_report, load_report, - upload_report, + write_report, ) from .table import display @@ -24,7 +32,9 @@ class Context(TypedDict): timestamp: datetime benchmark_reports: list[BenchmarkReport] + benchmark_observations: list[BenchViewObservation] compare_run_id: Optional[str] + execution_id: str def get_context(config: pytest.Config) -> Context: @@ -32,15 +42,46 @@ def get_context(config: pytest.Config) -> Context: context: Context = { "timestamp": datetime.now(), "benchmark_reports": [], + "benchmark_observations": [], "compare_run_id": None, + "execution_id": str(uuid4()), } setattr(config, "_benchmark_context", context) return getattr(config, "_benchmark_context") +def apply_benchmark_source_override(report: Report) -> None: + """Replace Git metadata when CI overlays the current reporter on historical source. + + :param report: Mutable local report whose commit facts feed native submissions. + """ + + revision = os.environ.get("BENCHVIEW_BENCHMARK_REF") + if not revision: + return + report["commit_info"]["id"] = revision + report["commit_info"]["dirty"] = False + branch = os.environ.get("BENCHVIEW_BENCHMARK_BRANCH") + if branch: + report["commit_info"]["branch"] = branch + + def pytest_configure(config: pytest.Config): # Make sure context is initialized get_context(config) + submit = config.getoption("benchmark_submit") + if submit: + api_url = config.getoption("benchmark_api_url") or os.environ.get("BENCHVIEW_API_URL") + if not api_url: + raise pytest.UsageError( + "--benchmark-submit requires --benchmark-api-url or BENCHVIEW_API_URL." + ) + try: + benchview_submission_url(api_url) + except BenchmarkSubmissionError as error: + raise pytest.UsageError(str(error)) from error + if not os.environ.get("BENCHVIEW_API_KEY"): + raise pytest.UsageError("--benchmark-submit requires BENCHVIEW_API_KEY.") def pytest_sessionstart(session: pytest.Session): @@ -51,6 +92,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): # Generate benchmark report context = get_context(session.config) report = generate_report(context["timestamp"], "", context["benchmark_reports"]) + apply_benchmark_source_override(report) # Save report save = session.config.getoption("--benchmark-save") @@ -62,14 +104,26 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): BENCHMARK_DIR.mkdir(parents=True, exist_ok=True) write_report(report, path, strip_data=True) - # Upload report to MongoDB - upload = session.config.getoption("--benchmark-upload") - if upload: - report["run_id"] = upload - print("Uploading benchmark report to MongoDB") - connection_string = session.config.getoption("--benchmark-mongodb-connection-string") - database_name = session.config.getoption("--benchmark-mongodb-database-name") - upload_report(report, connection_string, database_name) + # Submit native observations through the authenticated BenchView HTTP boundary. + submit = session.config.getoption("benchmark_submit") + if submit: + api_url = session.config.getoption("benchmark_api_url") or os.environ["BENCHVIEW_API_URL"] + write_key = os.environ["BENCHVIEW_API_KEY"] + submissions = build_benchview_submissions( + context["benchmark_observations"], + request_id=submit, + execution_id=context["execution_id"], + project_info=report["project_info"], + machine_info=report["machine_info"], + commit_info=report["commit_info"], + ) + print(f"Submitting {len(submissions)} benchmark batch(es) to BenchView at {api_url}") + receipts = submit_benchview_submissions(api_url, write_key, submissions) + duplicate_count = sum(1 for receipt in receipts if receipt["duplicate"]) + print( + f"BenchView accepted {len(receipts) - duplicate_count} batch(es); " + f"{duplicate_count} exact retry batch(es)." + ) def pytest_addoption(parser: pytest.Parser): @@ -97,25 +151,21 @@ def pytest_addoption(parser: pytest.Parser): help="List the IDs of all saved benchmark runs.", ) group.addoption( + "--benchmark-submit", "--benchmark-upload", + dest="benchmark_submit", action="store", default=False, - metavar="ID", - help="Upload benchmark report to a MongoDB with the specified run ID.", - ) - group.addoption( - "--benchmark-mongodb-connection-string", - action="store", - default="mongodb://localhost:27017", - metavar="CONNECTION_STRING", - help="MongoDB connection string.", + metavar="REQUEST_ID", + help="Submit native observations to BenchView with the specified request ID.", ) group.addoption( - "--benchmark-mongodb-database-name", + "--benchmark-api-url", + dest="benchmark_api_url", action="store", - default="nvr-ci", - metavar="NAME", - help="MongoDB database name.", + default=None, + metavar="URL", + help="BenchView root or nested base URL; defaults to BENCHVIEW_API_URL.", ) diff --git a/slangpy/testing/benchmark/report.py b/slangpy/testing/benchmark/report.py index 0be11b248..2bc237c14 100644 --- a/slangpy/testing/benchmark/report.py +++ b/slangpy/testing/benchmark/report.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -from typing import TypedDict, Any -import json from datetime import datetime +import json from pathlib import Path +from typing import Any, TypedDict -from .utils import get_project_info, get_machine_info, get_commit_info, to_json, from_json +from .utils import from_json, get_commit_info, get_machine_info, get_project_info, to_json class BenchmarkReport(TypedDict): @@ -34,6 +34,8 @@ class Report(TypedDict): def generate_report(timestamp: datetime, run_id: str, benchmarks: list[BenchmarkReport]) -> Report: + """Collect local project, machine, commit, and benchmark data into a legacy report.""" + return { "timestamp": timestamp, "run_id": run_id, @@ -45,6 +47,8 @@ def generate_report(timestamp: datetime, run_id: str, benchmarks: list[Benchmark def generate_run_id(report: Report) -> str: + """Derive a readable local report ID from its timestamp and commit state.""" + timestamp = report["timestamp"].strftime("%Y%m%d-%H%M%S") commit_id = report["commit_info"].get("id", "unknown") commit_dirty = "dirty" if report["commit_info"].get("dirty", False) else "clean" @@ -52,6 +56,8 @@ def generate_run_id(report: Report) -> str: def strip_benchmark_data(report: Report) -> Report: + """Copy a local report while removing its raw benchmark sample arrays.""" + stripped_benchmarks = [] for benchmark in report["benchmarks"]: stripped_benchmark = benchmark.copy() @@ -63,6 +69,8 @@ def strip_benchmark_data(report: Report) -> Report: def write_report(report: Report, path: Path, strip_data: bool = False) -> None: + """Serialize a legacy local report, optionally omitting its raw samples.""" + if strip_data: report = strip_benchmark_data(report) with open(path, "w") as f: @@ -70,22 +78,16 @@ def write_report(report: Report, path: Path, strip_data: bool = False) -> None: def load_report(path: Path) -> Report: + """Load a legacy local report from a JSON file.""" + with open(path, "r") as f: return from_json(json.load(f)) def list_report_ids(dir: Path) -> list[str]: + """List saved local report IDs from newest to oldest.""" + files = list(dir.iterdir()) - # sort by file date (descending) + # Sort reports by file date in descending order. files.sort(key=lambda f: f.stat().st_mtime, reverse=True) - # get ids - ids = [f.stem for f in files if f.suffix == ".json"] - return ids - - -def upload_report(report: Report, connection_string: str, database_name: str): - from pymongo import MongoClient - - client = MongoClient(connection_string) - db = client[database_name] - db["benchmark"].insert_one(report) + return [f.stem for f in files if f.suffix == ".json"] diff --git a/slangpy/tests/slangpy_tests/test_torchintegration.py b/slangpy/tests/slangpy_tests/test_torchintegration.py index a8d6d8afc..17a990c08 100644 --- a/slangpy/tests/slangpy_tests/test_torchintegration.py +++ b/slangpy/tests/slangpy_tests/test_torchintegration.py @@ -3,9 +3,10 @@ import pytest import sys import numpy as np +from typing import Any, Optional from slangpy import DeviceType, Device, Module, grid -from slangpy.core.native import NativeCallDataCache, SignatureBuilder +from slangpy.core.native import NativeCallDataCache, NativeTorchTensorDiffPair, SignatureBuilder from slangpy.testing import helpers try: @@ -234,6 +235,55 @@ def test_polynomial_multiple_calls(device_type: DeviceType): compare_tensors(2 * a * x + b, x.grad) # type: ignore +@pytest.mark.parametrize("device_type", DEVICE_TYPES) +def test_polynomial_manual_autograd_hook(device_type: DeviceType): + """A manual autograd hook must preserve input and output primals for bwds().""" + + module = load_test_module(device_type) + + a = 2.0 + b = 4.0 + c = 1.0 + x = torch.randn((10,), dtype=torch.float32, device=torch.device("cuda"), requires_grad=True) + + class PolynomialManualHook(torch.autograd.Function): + @staticmethod + def forward(ctx: Any, x: torch.Tensor) -> torch.Tensor: + detached_x = x.detach() + result = torch.empty_like(detached_x) + module.polynomial(a, b, c, detached_x, _result=result) + ctx.save_for_backward(detached_x, result) + return result + + @staticmethod + def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[Optional[torch.Tensor]]: + detached_x, result = ctx.saved_tensors + grad_x = torch.zeros_like(detached_x) + x_pair = NativeTorchTensorDiffPair(detached_x, grad_x, 0, True) + result_pair = NativeTorchTensorDiffPair(result, grad_output, 1, False) + module.polynomial.bwds(a, b, c, x_pair, _result=result_pair) + return (grad_x,) + + result = PolynomialManualHook.apply(x) + compare_tensors(a * x * x + b * x + c, result) + + result.backward(torch.ones_like(result)) + compare_tensors(2 * a * x + b, x.grad) # type: ignore[arg-type] + + +@pytest.mark.parametrize("device_type", DEVICE_TYPES) +def test_polynomial_manual_bwds_requires_result_primal(device_type: DeviceType): + """Reject a missing output primal before it becomes a null GPU address.""" + + module = load_test_module(device_type) + x = torch.randn((10,), dtype=torch.float32, device=torch.device("cuda")) + x_pair = NativeTorchTensorDiffPair(x, torch.zeros_like(x), 0, True) + result_pair = NativeTorchTensorDiffPair(None, torch.ones_like(x), 1, False) + + with pytest.raises(RuntimeError, match="primal tensor cannot be None"): + module.polynomial.bwds(2.0, 4.0, 1.0, x_pair, _result=result_pair) + + @pytest.mark.parametrize("device_type", DEVICE_TYPES) def test_polynomial_outparam(device_type: DeviceType): diff --git a/slangpy/tests/utils/test_benchmark_action_scheduling.py b/slangpy/tests/utils/test_benchmark_action_scheduling.py new file mode 100644 index 000000000..8592e7d5d --- /dev/null +++ b/slangpy/tests/utils/test_benchmark_action_scheduling.py @@ -0,0 +1,727 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Offline coverage for ordinary, nightly, and historical benchmark scheduling.""" + +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import subprocess +from typing import Any, Optional, Sequence + +import pytest + +from tools import backfill_benchmarks as backfill +from tools import benchmark_actions as actions + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +NOW = datetime(2026, 7, 17, 12, 0, tzinfo=timezone.utc) + + +class FakeCommandRunner: + """Return queued ``gh`` results while retaining exact arguments and stdin.""" + + def __init__(self, responses: Sequence[subprocess.CompletedProcess[str]]) -> None: + """Copy deterministic responses so each invocation consumes one result.""" + + super().__init__() + self.responses = list(responses) + self.calls: list[tuple[list[str], Optional[str]]] = [] + + def __call__( + self, arguments: Sequence[str], input_text: Optional[str] + ) -> subprocess.CompletedProcess[str]: + """Capture one no-shell invocation and return its next queued response.""" + + self.calls.append((list(arguments), input_text)) + if not self.responses: + raise AssertionError("Unexpected GitHub CLI invocation.") + return self.responses.pop(0) + + +class FakeGitHub(actions.GitHubCli): + """Provide in-memory commit, run, and dispatch behavior to scheduler tests.""" + + def __init__( + self, + commits: Sequence[actions.Commit], + runs: Sequence[actions.WorkflowRun] = (), + ) -> None: + """Seed fake GitHub history without locating or executing a real ``gh`` binary.""" + + super().__init__(command_runner=FakeCommandRunner([]), executable="gh-test") + self.commits = list(commits) + self.runs = list(runs) + self.dispatches: list[tuple[str, str, str, dict[str, str]]] = [] + self.next_run_id = 9000 + + def list_commits( + self, + repository: str, + branch: str, + since: datetime, + until: datetime, + ) -> list[actions.Commit]: + """Return seeded commits inside the scheduler's explicit UTC interval.""" + + del repository, branch + return [commit for commit in self.commits if since <= commit.committed_at <= until] + + def list_workflow_runs( + self, repository: str, workflow: str, maximum: int + ) -> list[actions.WorkflowRun]: + """Return the newest requested number of seeded workflow runs.""" + + del repository, workflow + return self.runs[:maximum] + + def dispatch_workflow( + self, + repository: str, + workflow: str, + workflow_ref: str, + inputs: dict[str, str], + ) -> actions.DispatchResult: + """Record one dispatch and expose it to subsequent title reconciliation.""" + + self.dispatches.append((repository, workflow, workflow_ref, dict(inputs))) + sha = inputs.get("revision") or inputs.get("target_sha") + if sha is None: + raise AssertionError("Benchmark workflow dispatch lacks its revision input.") + title_prefix = "ci-benchmark" if "revision" in inputs else "backfill-benchmark" + result = actions.DispatchResult( + run_id=self.next_run_id, + run_url=f"https://api.github.test/runs/{self.next_run_id}", + html_url=f"https://github.test/runs/{self.next_run_id}", + ) + self.runs.insert( + 0, + make_run( + run_id=result.run_id, + title=f"{title_prefix}: {sha}", + status="queued", + sha=sha, + ), + ) + self.next_run_id += 1 + return result + + +class FailingDispatchGitHub(FakeGitHub): + """Simulate an ambiguous transport failure after write-ahead state publication.""" + + def dispatch_workflow( + self, + repository: str, + workflow: str, + workflow_ref: str, + inputs: dict[str, str], + ) -> actions.DispatchResult: + """Fail without revealing whether GitHub accepted the request.""" + + del repository, workflow, workflow_ref, inputs + raise actions.GitHubCliError("connection lost") + + +def completed_process( + stdout: str, returncode: int = 0, stderr: str = "" +) -> subprocess.CompletedProcess[str]: + """Create one captured subprocess result for a fake command runner.""" + + return subprocess.CompletedProcess( + args=["gh"], + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + +def make_commit(index: int, committed_at: Optional[datetime] = None) -> actions.Commit: + """Create one stable 40-character commit record for scheduler scenarios.""" + + sha = f"{index:040x}" + return actions.Commit( + sha=sha, + committed_at=committed_at or NOW + timedelta(minutes=index), + message=f"Commit {index}\n\nDetails", + html_url=f"https://github.test/commit/{sha}", + ) + + +def floor_commit() -> actions.Commit: + """Create the exact inclusive compatibility-boundary commit.""" + + return actions.Commit( + sha=backfill.SUPPORTED_FLOOR_SHA, + committed_at=backfill.SUPPORTED_FLOOR_TIME, + message="Device-isolated benchmark support", + html_url=f"https://github.test/commit/{backfill.SUPPORTED_FLOOR_SHA}", + ) + + +def make_run( + run_id: int, + title: str, + status: str, + sha: str = "f" * 40, +) -> actions.WorkflowRun: + """Create one workflow run with stable reconciliation and capacity fields.""" + + return actions.WorkflowRun( + run_id=run_id, + title=title, + status=status, + conclusion=None if status != "completed" else "success", + html_url=f"https://github.test/runs/{run_id}", + head_sha=sha, + created_at=NOW + timedelta(seconds=run_id), + updated_at=NOW + timedelta(seconds=run_id), + ) + + +def make_run_document(index: int) -> dict[str, Any]: + """Create one GitHub workflow-run response object for adapter pagination tests.""" + + sha = f"{index:040x}" + return { + "id": index, + "display_title": f"backfill-benchmark: {sha}", + "status": "completed", + "conclusion": "success", + "html_url": f"https://github.test/runs/{index}", + "head_sha": sha, + "created_at": "2026-07-16T10:00:00Z", + "updated_at": "2026-07-16T11:00:00Z", + } + + +def make_store(path: Path) -> backfill.BackfillStateStore: + """Create one test state store bound to the production backfill configuration.""" + + return backfill.BackfillStateStore( + path, + repository=backfill.DEFAULT_REPOSITORY, + branch=backfill.DEFAULT_BRANCH, + workflow=backfill.DEFAULT_WORKFLOW, + lower_bound=backfill.SUPPORTED_FLOOR_SHA, + ) + + +def test_github_cli_parses_paginated_commits_and_uses_versioned_api() -> None: + """Flatten commit pages while passing all filters through an argument-array command.""" + + first_sha = "1" * 40 + second_sha = "2" * 40 + response = [ + [ + { + "sha": first_sha, + "html_url": f"https://github.test/commit/{first_sha}", + "commit": { + "message": "First", + "committer": {"date": "2026-07-16T10:00:00Z"}, + }, + } + ], + [ + { + "sha": second_sha, + "html_url": f"https://github.test/commit/{second_sha}", + "commit": { + "message": "Second", + "committer": {"date": "2026-07-16T11:00:00Z"}, + }, + } + ], + ] + runner = FakeCommandRunner([completed_process(json.dumps(response))]) + github = actions.GitHubCli(command_runner=runner, executable="gh-test") + + commits = github.list_commits( + "shader-slang/slangpy", + "main", + datetime(2026, 7, 16, tzinfo=timezone.utc), + datetime(2026, 7, 17, tzinfo=timezone.utc), + ) + + assert [commit.sha for commit in commits] == [first_sha, second_sha] + command, input_text = runner.calls[0] + assert command[:2] == ["gh-test", "api"] + assert f"X-GitHub-Api-Version: {actions.GITHUB_API_VERSION}" in command + assert "--paginate" in command + assert "--slurp" in command + assert "sha=main" in command + assert input_text is None + + +def test_github_cli_parses_runs_and_dispatches_json_on_stdin() -> None: + """Parse workflow pages and return immediate run details from exact dispatch JSON.""" + + run_page = { + "workflow_runs": [ + { + "id": 42, + "display_title": "ci-benchmark: " + "a" * 40, + "status": "completed", + "conclusion": "success", + "html_url": "https://github.test/runs/42", + "head_sha": "b" * 40, + "created_at": "2026-07-16T10:00:00Z", + "updated_at": "2026-07-16T11:00:00Z", + } + ] + } + dispatch = { + "workflow_run_id": 43, + "run_url": "https://api.github.test/runs/43", + "html_url": "https://github.test/runs/43", + } + runner = FakeCommandRunner( + [completed_process(json.dumps(run_page)), completed_process(json.dumps(dispatch))] + ) + github = actions.GitHubCli(command_runner=runner, executable="gh-test") + + runs = github.list_workflow_runs("shader-slang/slangpy", "ci-benchmark.yml", maximum=10) + result = github.dispatch_workflow( + "shader-slang/slangpy", + "ci-benchmark.yml", + workflow_ref="main", + inputs={"revision": "a" * 40}, + ) + + assert runs[0].run_id == 42 + assert runs[0].title == "ci-benchmark: " + "a" * 40 + assert result.run_id == 43 + command, input_text = runner.calls[1] + assert "--method" in command and "POST" in command + assert "--input" in command and "-" in command + assert json.loads(input_text or "") == { + "ref": "main", + "inputs": {"revision": "a" * 40}, + "return_run_details": True, + } + + +def test_github_cli_fetches_only_the_requested_workflow_run_pages() -> None: + """Bound capacity polling instead of downloading the workflow's complete history.""" + + first_page = {"workflow_runs": [make_run_document(index) for index in range(1, 101)]} + second_page = {"workflow_runs": [make_run_document(101)]} + runner = FakeCommandRunner( + [completed_process(json.dumps(first_page)), completed_process(json.dumps(second_page))] + ) + github = actions.GitHubCli(command_runner=runner, executable="gh-test") + + runs = github.list_workflow_runs("shader-slang/slangpy", "backfill-benchmark.yml", maximum=101) + + assert len(runs) == 101 + assert len(runner.calls) == 2 + first_command, _ = runner.calls[0] + second_command, _ = runner.calls[1] + assert "per_page=100" in first_command + assert "page=1" in first_command + assert "page=2" in second_command + assert "--paginate" not in first_command + assert f"X-GitHub-Api-Version: {actions.GITHUB_API_VERSION}" in first_command + + +def test_github_cli_reports_command_and_installation_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Give operators actionable errors without requiring a real binary or printing tokens.""" + + runner = FakeCommandRunner( + [completed_process("", returncode=1, stderr="authentication failed")] + ) + github = actions.GitHubCli(command_runner=runner, executable="gh-test") + with pytest.raises(actions.GitHubCliError, match="gh auth status"): + github.list_workflow_runs("shader-slang/slangpy", "ci-benchmark.yml", maximum=1) + + monkeypatch.setattr(actions.shutil, "which", lambda executable: None) + with pytest.raises(actions.GitHubCliError, match="gh auth login"): + actions.GitHubCli() + + +def test_default_gh_runner_never_uses_a_shell(monkeypatch: pytest.MonkeyPatch) -> None: + """Lock the real subprocess boundary to captured UTF-8 argument-array execution.""" + + captured: dict[str, Any] = {} + + def fake_run(arguments: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + """Capture subprocess options without starting an external program.""" + + captured["arguments"] = arguments + captured.update(kwargs) + return completed_process("{}") + + monkeypatch.setattr(actions.subprocess, "run", fake_run) + actions._default_command_runner(["gh", "api", "rate_limit"], None) + + assert captured["arguments"] == ["gh", "api", "rate_limit"] + assert captured["shell"] is False + assert captured["check"] is False + assert captured["capture_output"] is True + assert captured["encoding"] == "utf-8" + + +def test_ordinary_workflow_selects_tip_or_exact_revision_without_backfill_logic() -> None: + """Keep exact future selection generic and historical overlays out of ordinary CI.""" + + workflow = (REPOSITORY_ROOT / ".github/workflows/ci-benchmark.yml").read_text(encoding="utf-8") + + assert "cron:" not in workflow + assert 'run-name: "ci-benchmark: ${{ inputs.revision || github.sha }}"' in workflow + assert "revision:" in workflow + assert "ref: ${{ inputs.revision || github.sha }}" in workflow + assert "BENCHVIEW_BENCHMARK_REF: ${{ inputs.revision || github.sha }}" in workflow + assert "BENCHVIEW_BENCHMARK_BRANCH: ${{ github.ref_name }}" in workflow + assert "target_sha" not in workflow + assert "Overlay current BenchView benchmark harness" not in workflow + assert "run_benchmark_ci.py" not in workflow + + +def test_nightly_workflow_dispatches_recent_commits_with_github_script() -> None: + """Keep nightly fan-out inside GitHub Actions without Python or GitHub CLI setup.""" + + workflow = (REPOSITORY_ROOT / ".github/workflows/schedule-benchmarks.yml").read_text( + encoding="utf-8" + ) + + assert "actions/github-script@v9" in workflow + assert "github.rest.repos.listCommits" in workflow + assert "github.rest.actions.createWorkflowDispatch" in workflow + assert 'const workflow = "ci-benchmark.yml"' in workflow + assert 'const branch = "main"' in workflow + assert "inputs: { revision }" in workflow + assert "commits.reverse()" in workflow + assert "github.rest.actions.listWorkflowRuns" not in workflow + assert "existingTitles" not in workflow + assert "dry_run" not in workflow + assert "python" not in workflow.lower() + assert "gh --version" not in workflow + assert "actions/checkout" not in workflow + + +def test_backfill_workflow_guards_boundary_builds_before_overlay_and_cleans_safely() -> None: + """Lock the historical workflow ordering, source override, and deletion guard.""" + + workflow = (REPOSITORY_ROOT / ".github/workflows/backfill-benchmark.yml").read_text( + encoding="utf-8" + ) + + assert 'run-name: "backfill-benchmark: ${{ inputs.target_sha }}"' in workflow + assert "required: true" in workflow + assert ( + "${{ runner.temp }}/slangpy-backfill-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.os }}" + in workflow + ) + assert 'git clone --recursive "https://github.com/${{ github.repository }}.git"' in workflow + assert 'checkout --detach "${{ inputs.target_sha }}"' in workflow + assert "git submodule sync --recursive" in workflow + assert "git submodule update --init --recursive" in workflow + assert "git lfs pull" in workflow + assert "merge-base --is-ancestor" in workflow + assert backfill.SUPPORTED_FLOOR_SHA in workflow + assert workflow.index("Validate supported history boundary") < workflow.index( + "Historical setup" + ) + assert workflow.index("Historical build") < workflow.index( + "Overlay current BenchView benchmark harness" + ) + assert workflow.index("Overlay current BenchView benchmark harness") < workflow.index( + "Benchmark historical source" + ) + assert ( + 'git checkout "${{ github.sha }}" -- tools/ci.py tools/gpu_clock.py ' + "slangpy/testing/benchmark" + ) in workflow + assert "python tools/ci.py benchmark-python" in workflow + assert "BENCHVIEW_BENCHMARK_REF: ${{ inputs.target_sha }}" in workflow + assert "BENCHVIEW_BENCHMARK_BRANCH: main" in workflow + assert "shell: python" not in workflow + assert "[StringComparer]::OrdinalIgnoreCase.Equals($parent.FullName, $runnerTemp)" in workflow + assert '$name.StartsWith("slangpy-backfill-"' in workflow + assert "Remove-Item -LiteralPath $candidate -Recurse -Force" in workflow + assert '"$(dirname -- "$candidate")" != "$runner_temp"' in workflow + assert '"$(basename -- "$candidate")" != slangpy-backfill-*' in workflow + assert 'rm -rf -- "$candidate"' in workflow + + +def test_boundary_rejection_excludes_history_before_the_floor() -> None: + """Reject an inventory that cannot prove the configured inclusive floor.""" + + with pytest.raises(backfill.BackfillStateError, match=backfill.SUPPORTED_FLOOR_SHA): + backfill.supported_commits([make_commit(1)], backfill.SUPPORTED_FLOOR_SHA) + + +def test_backfill_with_three_active_runs_dispatches_one_oldest_commit( + tmp_path: Path, +) -> None: + """Fill the fourth workflow slot with exactly one oldest pending revision.""" + + commits = [floor_commit(), make_commit(2, backfill.SUPPORTED_FLOOR_TIME + timedelta(days=1))] + active_runs = [ + make_run(index, f"unrelated backfill {index}", "in_progress") for index in range(3) + ] + github = FakeGitHub(commits, active_runs) + store = make_store(tmp_path / "state.json") + + result = backfill.run_backfill_scheduler( + github, + store, + now_provider=lambda: NOW, + sleep=lambda seconds: None, + poll_seconds=60, + once=True, + dry_run=False, + output=lambda line: None, + ) + + assert result == 0 + assert len(github.dispatches) == 1 + assert github.dispatches[0][3] == {"target_sha": backfill.SUPPORTED_FLOOR_SHA} + reloaded = store.load() + floor = reloaded.records[backfill.SUPPORTED_FLOOR_SHA] + assert floor.status == "dispatched" + assert floor.run_id == 9000 + assert floor.run_url == "https://github.test/runs/9000" + assert list(tmp_path.glob("state.json.*.tmp")) == [] + + +def test_backfill_with_four_active_runs_dispatches_nothing(tmp_path: Path) -> None: + """Hold all pending history when the four-workflow pressure limit is full.""" + + commits = [floor_commit(), make_commit(2, backfill.SUPPORTED_FLOOR_TIME + timedelta(days=1))] + active_runs = [make_run(index, f"unrelated backfill {index}", "queued") for index in range(4)] + github = FakeGitHub(commits, active_runs) + store = make_store(tmp_path / "state.json") + + backfill.run_backfill_scheduler( + github, + store, + now_provider=lambda: NOW, + sleep=lambda seconds: None, + poll_seconds=60, + once=True, + dry_run=False, + output=lambda line: None, + ) + + assert github.dispatches == [] + assert store.load().records[backfill.SUPPORTED_FLOOR_SHA].status == "pending" + + +def test_backfill_restart_preserves_the_one_minute_dispatch_interval(tmp_path: Path) -> None: + """Prevent a quick restart from bypassing the persisted dispatch cadence.""" + + second = make_commit(2, backfill.SUPPORTED_FLOOR_TIME + timedelta(days=1)) + commits = [floor_commit(), second] + existing = make_run( + 70, + backfill.backfill_run_title(backfill.SUPPORTED_FLOOR_SHA), + "in_progress", + backfill.SUPPORTED_FLOOR_SHA, + ) + github = FakeGitHub(commits, [existing]) + store = make_store(tmp_path / "state.json") + state = store.load() + backfill.merge_discovered_commits(state, commits) + floor = state.records[backfill.SUPPORTED_FLOOR_SHA] + floor.status = "dispatched" + floor.dispatch_started_at = NOW - timedelta(seconds=30) + floor.run_id = existing.run_id + floor.run_url = existing.html_url + store.save(state) + + backfill.run_backfill_scheduler( + github, + store, + now_provider=lambda: NOW, + sleep=lambda seconds: None, + poll_seconds=60, + once=True, + dry_run=False, + output=lambda line: None, + ) + assert github.dispatches == [] + + backfill.run_backfill_scheduler( + github, + store, + now_provider=lambda: NOW + timedelta(seconds=31), + sleep=lambda seconds: None, + poll_seconds=60, + once=True, + dry_run=False, + output=lambda line: None, + ) + assert [dispatch[3] for dispatch in github.dispatches] == [{"target_sha": second.sha}] + + +def test_incompatible_state_is_rejected_without_modification(tmp_path: Path) -> None: + """Require explicit archival of a prototype state bound to another workflow.""" + + path = tmp_path / "state.json" + path.write_text( + json.dumps( + { + "schemaVersion": 1, + "repository": backfill.DEFAULT_REPOSITORY, + "branch": backfill.DEFAULT_BRANCH, + "workflow": "ci-benchmark.yml", + "lowerBound": backfill.SUPPORTED_FLOOR_SHA, + "commits": [], + } + ), + encoding="utf-8", + ) + original = path.read_bytes() + + with pytest.raises(backfill.BackfillStateError, match="Archive"): + make_store(path).load() + + assert path.read_bytes() == original + + +def test_uncertain_dispatch_reconciles_by_title_or_expires_after_grace(tmp_path: Path) -> None: + """Recover a crash after POST without duplicating an accepted workflow request.""" + + store = make_store(tmp_path / "state.json") + state = store.load() + commit = floor_commit() + backfill.merge_discovered_commits(state, [commit]) + record = state.records[commit.sha] + record.status = "dispatching" + record.dispatch_started_at = NOW + store.save(state) + + accepted = make_run(77, backfill.backfill_run_title(commit.sha), "queued", commit.sha) + assert backfill.reconcile_records( + state, [accepted], NOW + timedelta(minutes=1), backfill.DEFAULT_DISPATCH_GRACE + ) + assert record.status == "dispatched" + assert record.run_id == 77 + + record.status = "dispatching" + record.dispatch_started_at = NOW + record.run_id = None + record.run_url = None + assert not backfill.reconcile_records( + state, [], NOW + timedelta(minutes=9), backfill.DEFAULT_DISPATCH_GRACE + ) + assert record.status == "dispatching" + assert backfill.reconcile_records( + state, [], NOW + timedelta(minutes=10), backfill.DEFAULT_DISPATCH_GRACE + ) + assert record.status == "pending" + + +def test_scheduler_waits_for_an_uncertain_only_record_then_retries(tmp_path: Path) -> None: + """Keep running through the grace window when no immediately pending commit remains.""" + + commit = floor_commit() + github = FakeGitHub([commit]) + store = make_store(tmp_path / "state.json") + state = store.load() + backfill.merge_discovered_commits(state, [commit]) + record = state.records[commit.sha] + record.status = "dispatching" + record.dispatch_started_at = NOW + store.save(state) + times = iter( + [ + NOW + timedelta(minutes=1), + NOW + timedelta(minutes=1), + NOW + timedelta(minutes=11), + ] + ) + sleeps: list[float] = [] + + result = backfill.run_backfill_scheduler( + github, + store, + now_provider=lambda: next(times), + sleep=sleeps.append, + poll_seconds=60, + once=False, + dry_run=False, + output=lambda line: None, + ) + + assert result == 0 + assert sleeps == [60] + assert [dispatch[3] for dispatch in github.dispatches] == [{"target_sha": commit.sha}] + assert store.load().records[commit.sha].status == "dispatched" + + +def test_failed_dispatch_leaves_write_ahead_state_for_restart_reconciliation( + tmp_path: Path, +) -> None: + """Preserve an ambiguous request until a deterministic GitHub title resolves it.""" + + store = make_store(tmp_path / "state.json") + state = store.load() + commit = floor_commit() + backfill.merge_discovered_commits(state, [commit]) + github = FailingDispatchGitHub([commit]) + + with pytest.raises(actions.GitHubCliError, match="connection lost"): + backfill.dispatch_oldest_pending(github, store, state, NOW) + + reloaded = store.load() + assert reloaded.records[commit.sha].status == "dispatching" + accepted = make_run(88, backfill.backfill_run_title(commit.sha), "queued", commit.sha) + assert backfill.reconcile_records( + reloaded, [accepted], NOW + timedelta(minutes=1), backfill.DEFAULT_DISPATCH_GRACE + ) + assert reloaded.records[commit.sha].status == "dispatched" + assert reloaded.records[commit.sha].run_id == 88 + + +def test_backfill_dry_run_does_not_create_state_or_dispatch(tmp_path: Path) -> None: + """Preview the complete supported inventory without any local or GitHub write.""" + + commits = [floor_commit(), make_commit(2, backfill.SUPPORTED_FLOOR_TIME + timedelta(days=1))] + github = FakeGitHub(commits) + path = tmp_path / "state.json" + output: list[str] = [] + + backfill.run_backfill_scheduler( + github, + make_store(path), + now_provider=lambda: NOW, + sleep=lambda seconds: None, + poll_seconds=60, + once=False, + dry_run=True, + output=output.append, + ) + + assert not path.exists() + assert github.dispatches == [] + assert any(backfill.DEFAULT_WORKFLOW in line for line in output) + assert any(backfill.SUPPORTED_FLOOR_SHA in line for line in output) + + +def test_backfill_main_returns_130_after_keyboard_interrupt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Translate an operator Ctrl+C into the documented durable-restart exit status.""" + + def make_github() -> FakeGitHub: + """Avoid locating or contacting a real GitHub CLI during the command test.""" + + return FakeGitHub([floor_commit()]) + + def interrupt_scheduler(*args: Any, **kwargs: Any) -> int: + """Interrupt after command setup as if the operator pressed Ctrl+C.""" + + del args, kwargs + raise KeyboardInterrupt + + monkeypatch.setattr(backfill, "GitHubCli", make_github) + monkeypatch.setattr(backfill, "run_backfill_scheduler", interrupt_scheduler) + + result = backfill.main(["--state-file", str(tmp_path / "state.json")]) + + assert result == 130 diff --git a/slangpy/tests/utils/test_benchmark_submission.py b/slangpy/tests/utils/test_benchmark_submission.py new file mode 100644 index 000000000..33ecc1fb6 --- /dev/null +++ b/slangpy/tests/utils/test_benchmark_submission.py @@ -0,0 +1,736 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import ast +from datetime import datetime, timezone +from email.message import Message +from io import BytesIO +from importlib import import_module +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast, Optional, Type +from urllib.error import HTTPError, URLError +from urllib.request import Request + +import pytest + +from slangpy.testing.benchmark.fixtures import ReportFixture + +benchmark_api = import_module("slangpy.testing.benchmark.benchview") +benchmark_plugin = import_module("slangpy.testing.benchmark.plugin") +ci = import_module("tools.ci") +gpu_clock = import_module("tools.gpu_clock") + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] + + +class FakeDeviceType: + """Provide the enum-name surface used by real SlangPy device types.""" + + name = "cuda" + + +class FakeResponse: + """Act as the small urllib response surface consumed by the sender.""" + + def __init__(self, status: int, body: bytes): + super().__init__() + self.status = status + self.body = body + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__( + self, + exception_type: Optional[Type[BaseException]], + exception: Optional[BaseException], + traceback: Any, + ) -> None: + return None + + def getcode(self) -> int: + return self.status + + def read(self, amount: int = -1) -> bytes: + return self.body if amount < 0 else self.body[:amount] + + +class FakePytestConfig: + """Provide the option and context surface used during plugin configuration.""" + + def __init__(self, submit: Any, api_url: Optional[str]): + super().__init__() + self.options = {"benchmark_submit": submit, "benchmark_api_url": api_url} + + def getoption(self, name: str) -> Any: + return self.options[name] + + +def make_observation(metric_id: str = "gpu_time") -> dict[str, Any]: + """Build one representative native observation for payload tests.""" + + return benchmark_api.build_benchview_observation( + filename="slangpy/benchmarks/test_benchmark_tensor.py", + function_name="test_tensor_sum", + display_name="test_tensor_sum[cuda-1024]", + parameters={"device_type": FakeDeviceType(), "element_count": 1024, "contiguous": True}, + samples=[1.25, 1.5, 1.0], + observed_at=datetime(2026, 7, 16, 12, 30, tzinfo=timezone.utc), + metric_id=metric_id, + metric_name="GPU time" if metric_id == "gpu_time" else "CPU time", + adapter_name="NVIDIA Test GPU", + source_line=42, + ) + + +def submission_context() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + """Return deterministic project, machine, and commit facts for batching tests.""" + + project_info = { + "name": "slangpy", + "version": "0.41.0", + "slang_build_tag": "v2026.1", + } + machine_info = { + "node": "benchmark-host", + "processor": "Test CPU", + "machine": "AMD64", + "system": "Windows", + "release": "11", + "version": "10.0.26100", + "python_compiler": "MSC v.1944", + "python_implementation": "CPython", + "python_version": "3.12.8", + "gpus": [ + { + "index": 0, + "uuid": "GPU-test", + "name": "NVIDIA Test GPU", + "driver_version": "600.00", + "memory_total": 1024.0, + "memory_used": 128.0, + "temperature": 45.0, + } + ], + } + commit_info = { + "id": "a" * 40, + "time": datetime(2026, 7, 16, 12, tzinfo=timezone.utc), + "branch": "main", + "dirty": False, + } + return project_info, machine_info, commit_info + + +def test_historical_source_override_preserves_time_and_replaces_git_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Report the historical target as clean main source after overlaying current Python files.""" + + historical_time = datetime(2025, 9, 2, 14, 42, 35, tzinfo=timezone.utc) + report: dict[str, Any] = { + "commit_info": { + "id": "overlay-tree", + "time": historical_time, + "branch": "detached", + "dirty": True, + } + } + target = "f3ad0fd91d8cf4eeb2be3b505765b43482aa952a" + monkeypatch.setenv("BENCHVIEW_BENCHMARK_REF", target) + monkeypatch.setenv("BENCHVIEW_BENCHMARK_BRANCH", "main") + + benchmark_plugin.apply_benchmark_source_override(cast(Any, report)) + + assert report["commit_info"] == { + "id": target, + "time": historical_time, + "branch": "main", + "dirty": False, + } + + +@pytest.mark.parametrize("metric_id", ["gpu_time", "cpu_time"]) +def test_build_observation_uses_native_identity_and_metric(metric_id: str) -> None: + """Verify typed dimensions and explicit fixture timing semantics.""" + + observation = make_observation(metric_id) + + assert observation["test"] == { + "id": "slangpy/benchmarks/test_benchmark_tensor.py:test_tensor_sum", + "name": "test_tensor_sum", + "source": { + "file": "slangpy/benchmarks/test_benchmark_tensor.py", + "function": "test_tensor_sum", + "line": 42, + }, + } + assert observation["case"]["dimensions"] == { + "device_type": "cuda", + "element_count": "1024", + "contiguous": "True", + } + assert observation["metrics"][0] == { + "id": metric_id, + "name": "GPU time" if metric_id == "gpu_time" else "CPU time", + "unit": "ms", + "direction": "lower", + "distribution": {"samples": [1.25, 1.5, 1.0]}, + } + + +def test_report_fixture_accumulates_local_and_native_results() -> None: + """Keep local comparisons while fixtures build the API-ready observation.""" + + context: dict[str, Any] = {"benchmark_reports": [], "benchmark_observations": []} + config = SimpleNamespace(_benchmark_context=context) + node = SimpleNamespace( + name="test_cpu_case[cuda]", + originalname="test_cpu_case", + location=("slangpy/benchmarks/test_cpu.py", 9, "test_cpu_case"), + callspec=SimpleNamespace(params={"device_type": FakeDeviceType()}), + ) + + ReportFixture(cast(pytest.Config, config), node)( + None, + [2.0, 1.0, 3.0], + 0.25, + ) + + assert len(context["benchmark_reports"]) == 1 + assert context["benchmark_reports"][0]["data"] == [2.0, 1.0, 3.0] + assert len(context["benchmark_observations"]) == 1 + assert context["benchmark_observations"][0]["metrics"][0]["id"] == "cpu_time" + + +def test_build_submissions_shares_run_and_separates_environment_telemetry() -> None: + """Prove distributed run identity, batching, and stable/volatile environment mapping.""" + + project_info, machine_info, commit_info = submission_context() + first = make_observation() + second = make_observation("cpu_time") + second["test"] = { + "id": "slangpy/benchmarks/test_benchmark_tensor.py:test_tensor_sum_cpu", + "name": "test_tensor_sum_cpu", + } + + submissions = benchmark_api.build_benchview_submissions( + [first, second], + request_id="gitlab-pipeline-123", + execution_id="execution-a", + project_info=project_info, + machine_info=machine_info, + commit_info=commit_info, + batch_size=1, + ) + repeated = benchmark_api.build_benchview_submissions( + [first, second], + request_id="gitlab-pipeline-123", + execution_id="execution-a", + project_info=project_info, + machine_info=machine_info, + commit_info=commit_info, + batch_size=1, + ) + other_execution = benchmark_api.build_benchview_submissions( + [first], + request_id="gitlab-pipeline-123", + execution_id="execution-b", + project_info=project_info, + machine_info=machine_info, + commit_info=commit_info, + ) + + assert submissions == repeated + assert len(submissions) == 2 + assert submissions[0]["run"]["key"] == submissions[1]["run"]["key"] + assert submissions[0]["run"]["key"] == other_execution[0]["run"]["key"] + assert submissions[0]["idempotencyKey"] != submissions[1]["idempotencyKey"] + assert submissions[0]["idempotencyKey"] != other_execution[0]["idempotencyKey"] + environment = submissions[0]["observations"][0]["environment"] + assert environment["identity"]["machine"] == "benchmark-host" + assert environment["identity"]["gpus"][0]["memoryBytes"] == 1024 * 1024 * 1024 + assert "temperature" not in environment["identity"]["gpus"][0] + assert environment["telemetry"]["gpus"][0]["temperature"] == 45.0 + + +def test_submission_url_preserves_arbitrary_nested_base() -> None: + """Keep API traffic inside root and arbitrary-depth nginx mount paths.""" + + assert ( + benchmark_api.benchview_submission_url("http://localhost:3000") + == "http://localhost:3000/api/v1/submissions" + ) + assert ( + benchmark_api.benchview_submission_url("http://host/foo/bar/hello/") + == "http://host/foo/bar/hello/api/v1/submissions" + ) + with pytest.raises(benchmark_api.BenchmarkSubmissionError, match="credentials"): + benchmark_api.benchview_submission_url("https://user:password@host/benchview") + + +def test_build_submissions_splits_at_the_body_limit() -> None: + """Split large sessions without allowing one oversize observation through.""" + + project_info, machine_info, commit_info = submission_context() + first = make_observation() + second = make_observation("cpu_time") + second["test"] = {"id": "tests:second", "name": "second"} + common = { + "request_id": "request", + "execution_id": "execution", + "project_info": project_info, + "machine_info": machine_info, + "commit_info": commit_info, + } + single_sizes = [ + len( + json.dumps( + benchmark_api.build_benchview_submissions([observation], **common)[0], + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ) + for observation in (first, second) + ] + body_limit = max(single_sizes) + + submissions = benchmark_api.build_benchview_submissions( + [first, second], max_body_bytes=body_limit, **common + ) + assert len(submissions) == 2 + with pytest.raises(benchmark_api.BenchmarkSubmissionError, match="exceeds"): + benchmark_api.build_benchview_submissions( + [first], max_body_bytes=single_sizes[0] - 1, **common + ) + + +def test_plugin_rejects_missing_api_configuration_early( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Fail before an expensive benchmark session when URL or key configuration is absent.""" + + monkeypatch.delenv("BENCHVIEW_API_URL", raising=False) + monkeypatch.setenv("BENCHVIEW_API_KEY", "key") + with pytest.raises(pytest.UsageError, match="benchmark-api-url"): + benchmark_plugin.pytest_configure(cast(pytest.Config, FakePytestConfig("request", None))) + + monkeypatch.delenv("BENCHVIEW_API_KEY", raising=False) + with pytest.raises(pytest.UsageError, match="BENCHVIEW_API_KEY"): + benchmark_plugin.pytest_configure( + cast(pytest.Config, FakePytestConfig("request", "http://localhost:3000")) + ) + + monkeypatch.setenv("BENCHVIEW_API_KEY", "key") + with pytest.raises(pytest.UsageError, match="absolute HTTP URL"): + benchmark_plugin.pytest_configure( + cast(pytest.Config, FakePytestConfig("request", "not-a-url")) + ) + + +def test_ci_wrapper_passes_benchview_options_to_pytest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep the ordinary CI entry point as the single benchmark runner.""" + + commands: list[list[str]] = [] + + def capture_command( + command: list[str], + shell: bool = True, + env: Optional[dict[str, str]] = None, + ) -> None: + """Capture the generated command without starting benchmark subprocesses.""" + + assert shell is True + assert env is not None + commands.append(command) + + monkeypatch.setattr(ci, "get_os", lambda: "linux") + monkeypatch.setattr(ci, "run_command", capture_command) + ci.benchmark_python( + SimpleNamespace( + device_type="cuda", + lock_gpu_clocks=False, + api_url="http://host/benchview", + run_id="workflow-123", + ) + ) + + assert len(commands) == 1 + assert commands[0][:6] == [ + "pytest", + "slangpy/benchmarks", + "-ra", + "--device-types", + "cuda", + f"--basetemp={ci.PYTEST_BASE_TEMP_DIR}", + ] + assert commands[0][-4:] == [ + "--benchmark-submit", + "workflow-123", + "--benchmark-api-url", + "http://host/benchview", + ] + + +def test_ci_wrapper_forwards_an_explicit_empty_api_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Let pytest reject a missing workflow variable instead of skipping submission.""" + + commands: list[list[str]] = [] + + def capture_command( + command: list[str], + shell: bool = True, + env: Optional[dict[str, str]] = None, + ) -> None: + """Capture the command generated for an explicitly configured API URL.""" + + commands.append(command) + + monkeypatch.setattr(ci, "get_os", lambda: "linux") + monkeypatch.setattr(ci, "run_command", capture_command) + ci.benchmark_python( + SimpleNamespace( + device_type="cuda", + lock_gpu_clocks=False, + api_url="", + run_id="workflow-123", + ) + ) + + assert commands[0][-4:] == [ + "--benchmark-submit", + "workflow-123", + "--benchmark-api-url", + "", + ] + + +def test_linux_ci_wrapper_does_not_run_gpu_clock_python_as_root( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Elevate only nvidia-smi mutations inside the clock helper.""" + + commands: list[list[str]] = [] + + def capture_command( + command: list[str], + shell: bool = True, + env: Optional[dict[str, str]] = None, + ) -> None: + del shell, env + commands.append(command) + + monkeypatch.setattr(ci, "get_os", lambda: "linux") + monkeypatch.setattr(ci, "run_command", capture_command) + ci.benchmark_python( + SimpleNamespace( + device_type="cuda", + lock_gpu_clocks=True, + api_url=None, + run_id="workflow-123", + ) + ) + + assert commands[0][:2] == ["python", str(ci.PROJECT_DIR / "tools/gpu_clock.py")] + assert commands[0][2:] == ["lock", "--ratio", "0.7"] + assert commands[-1][:2] == ["python", str(ci.PROJECT_DIR / "tools/gpu_clock.py")] + assert commands[-1][2:] == ["unlock"] + assert all(command[0] != "sudo" for command in (commands[0], commands[-1])) + + +def test_linux_gpu_clock_elevates_only_nvidia_smi_mutations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + commands: list[list[str]] = [] + + def capture_command(command: list[str]) -> str: + commands.append(command) + return "Test GPU" + + monkeypatch.setattr(gpu_clock.platform, "system", lambda: "Linux") + monkeypatch.setattr(gpu_clock, "NVIDIA_SMI", "nvidia-smi") + monkeypatch.setattr(gpu_clock, "run_command", capture_command) + + assert gpu_clock.nvidia_smi_mutation_command(["-i", "2", "--lock-gpu-clocks=1234"]) == [ + "sudo", + "-n", + "--", + "nvidia-smi", + "-i", + "2", + "--lock-gpu-clocks=1234", + ] + assert gpu_clock.get_gpu_name(2) == "Test GPU" + assert commands == [ + [ + "nvidia-smi", + "-i", + "2", + "--query-gpu=name", + "--format=csv,noheader,nounits", + ] + ] + + +def test_ordinary_workflow_uses_ci_wrapper_without_historical_logic() -> None: + """Keep branch-tip and exact future benchmarks on the normal current producer path.""" + + workflow = (REPOSITORY_ROOT / ".github/workflows/ci-benchmark.yml").read_text(encoding="utf-8") + + assert "cron:" not in workflow + assert "workflow_dispatch:" in workflow + assert 'run-name: "ci-benchmark: ${{ inputs.revision || github.sha }}"' in workflow + assert "ref: ${{ inputs.revision || github.sha }}" in workflow + assert workflow.count("python tools/ci.py benchmark-python") == 2 + assert workflow.count("--lock-gpu-clocks") == 2 + assert "Benchmark (Python, Linux, GPU Clock Locked)" in workflow + assert "BENCHVIEW_API_URL" in workflow + assert "BENCHVIEW_API_KEY" in workflow + assert workflow.count("contains(matrix.flags, 'benchmark')") >= 4 + assert "contains(matrix.flags, 'unit-test')" not in workflow + assert "python tools/ci.py install-slangpy-torch" in workflow + assert "python -m pip uninstall slangpy-torch -y" in workflow + assert "target_sha" not in workflow + assert "Overlay current BenchView benchmark harness" not in workflow + assert "run_benchmark_ci.py" not in workflow + assert "mongodb" not in workflow.lower() + + +def test_cuda_only_ppisp_benchmarks_declare_the_device_dimension() -> None: + """Keep CUDA-only PPISP tests out of the non-device benchmark shard.""" + + source_path = REPOSITORY_ROOT / "slangpy/benchmarks/test_benchmark_ppisp.py" + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + device_tests: list[ast.FunctionDef] = [] + for node in tree.body: + if not isinstance(node, ast.FunctionDef) or not node.name.startswith("test_"): + continue + if any( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Attribute) + and child.func.attr == "get_torch_device" + for child in ast.walk(node) + ): + device_tests.append(node) + + assert device_tests + for test_function in device_tests: + parameter_names = [parameter.arg for parameter in test_function.args.args] + assert "device_type" in parameter_names, test_function.name + get_device_calls = [ + child + for child in ast.walk(test_function) + if isinstance(child, ast.Call) + and isinstance(child.func, ast.Attribute) + and child.func.attr == "get_torch_device" + ] + assert all( + call.args and isinstance(call.args[0], ast.Name) and call.args[0].id == "device_type" + for call in get_device_calls + ), test_function.name + + +def test_backward_diff_benchmark_uses_the_available_extensions_include() -> None: + """Keep extensions.slang and the benchmark include directory aligned.""" + + benchmark_directory = REPOSITORY_ROOT / "slangpy/benchmarks" + benchmark_source = (benchmark_directory / "test_benchmark_bwd_diff.py").read_text( + encoding="utf-8" + ) + + assert (benchmark_directory / "ppisp/extensions.slang").is_file() + assert 'os.path.join(BENCH_DIR, "ppisp")' in benchmark_source + + +def test_submit_posts_bearer_authenticated_json(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify the real urllib boundary without contacting a live service.""" + + project_info, machine_info, commit_info = submission_context() + submissions = benchmark_api.build_benchview_submissions( + [make_observation()], + request_id="request-1", + execution_id="execution-1", + project_info=project_info, + machine_info=machine_info, + commit_info=commit_info, + ) + captured: dict[str, Any] = {} + + def fake_urlopen(request: Request, timeout: float) -> FakeResponse: + captured["request"] = request + captured["timeout"] = timeout + return FakeResponse( + 201, + json.dumps({"duplicate": False, "transactionId": "tx", "cursor": "0"}).encode(), + ) + + monkeypatch.setattr(benchmark_api, "urlopen", fake_urlopen) + receipts = benchmark_api.submit_benchview_submissions( + "http://host/benchview", "secret-write-key", submissions + ) + + request = captured["request"] + assert isinstance(request, Request) + assert request.full_url == "http://host/benchview/api/v1/submissions" + assert request.get_header("Authorization") == "Bearer secret-write-key" + assert request.get_header("Content-type") == "application/json" + assert isinstance(request.data, bytes) + assert json.loads(request.data)["idempotencyKey"].startswith("slangpy/") + assert receipts == [{"duplicate": False, "transactionId": "tx", "cursor": "0"}] + + +def test_submit_retries_connection_resets_with_the_identical_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Recover from ambiguous connection resets by safely repeating one idempotent batch.""" + + attempts: list[bytes] = [] + delays: list[float] = [] + + def flaky_urlopen(request: Request, timeout: float) -> FakeResponse: + """Reset two connections before accepting the byte-identical third request.""" + + assert timeout == 30.0 + assert isinstance(request.data, bytes) + attempts.append(request.data) + if len(attempts) < 3: + raise URLError(ConnectionResetError(104, "Connection reset by peer")) + return FakeResponse( + 200, + json.dumps({"duplicate": True, "transactionId": "tx", "cursor": "0"}).encode(), + ) + + monkeypatch.setattr(benchmark_api, "urlopen", flaky_urlopen) + monkeypatch.setattr( + benchmark_api, + "sleep", + lambda delay: delays.append(delay), + ) + receipts = benchmark_api.submit_benchview_submissions( + "http://host/benchview", + "secret-write-key", + [{"schemaVersion": 1, "idempotencyKey": "stable-key"}], + max_attempts=3, + retry_delay_seconds=0.25, + ) + + assert len(attempts) == 3 + assert attempts[0] == attempts[1] == attempts[2] + assert delays == [0.25, 0.5] + assert receipts == [{"duplicate": True, "transactionId": "tx", "cursor": "0"}] + + +def test_submit_retries_a_transient_gateway_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Retry temporary proxy failures while leaving permanent HTTP failures terminal.""" + + attempts = 0 + delays: list[float] = [] + + def flaky_urlopen(request: Request, timeout: float) -> FakeResponse: + """Return one retryable gateway error followed by a normal receipt.""" + + nonlocal attempts + attempts += 1 + if attempts == 1: + raise HTTPError( + request.full_url, + 503, + "Service unavailable", + hdrs=Message(), + fp=BytesIO(b"temporary"), + ) + return FakeResponse( + 201, + json.dumps({"duplicate": False, "transactionId": "tx", "cursor": "0"}).encode(), + ) + + monkeypatch.setattr(benchmark_api, "urlopen", flaky_urlopen) + monkeypatch.setattr( + benchmark_api, + "sleep", + lambda delay: delays.append(delay), + ) + receipts = benchmark_api.submit_benchview_submissions( + "http://host/benchview", + "secret-write-key", + [{"schemaVersion": 1, "idempotencyKey": "stable-key"}], + retry_delay_seconds=0.5, + ) + + assert attempts == 2 + assert delays == [0.5] + assert receipts == [{"duplicate": False, "transactionId": "tx", "cursor": "0"}] + + +def test_submit_stops_after_the_configured_connection_attempts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bound persistent connection failures instead of retrying a CI submission forever.""" + + attempts = 0 + delays: list[float] = [] + + def failing_urlopen(request: Request, timeout: float) -> FakeResponse: + """Reset every connection to exercise the terminal retry path.""" + + nonlocal attempts + attempts += 1 + raise URLError(ConnectionResetError(104, "Connection reset by peer")) + + monkeypatch.setattr(benchmark_api, "urlopen", failing_urlopen) + monkeypatch.setattr( + benchmark_api, + "sleep", + lambda delay: delays.append(delay), + ) + with pytest.raises(benchmark_api.BenchmarkSubmissionError) as error: + benchmark_api.submit_benchview_submissions( + "http://host/benchview", + "secret-write-key", + [{"schemaVersion": 1, "idempotencyKey": "stable-key"}], + max_attempts=3, + retry_delay_seconds=0.25, + ) + + assert attempts == 3 + assert delays == [0.25, 0.5] + assert "after 3 attempt(s)" in str(error.value) + + +def test_submit_redacts_key_from_http_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Prevent a malicious or reflected server diagnostic from disclosing credentials.""" + + key = "never-print-this-key" + + attempts = 0 + + def failing_urlopen(request: Request, timeout: float) -> FakeResponse: + """Return a permanent authentication failure that must not be retried.""" + + nonlocal attempts + attempts += 1 + raise HTTPError( + request.full_url, + 401, + "Unauthorized", + hdrs=Message(), + fp=BytesIO(f"invalid {key}".encode()), + ) + + monkeypatch.setattr(benchmark_api, "urlopen", failing_urlopen) + with pytest.raises(benchmark_api.BenchmarkSubmissionError) as error: + benchmark_api.submit_benchview_submissions( + "http://host/benchview", + key, + [{"schemaVersion": 1}], + ) + assert key not in str(error.value) + assert "" in str(error.value) + assert attempts == 1 diff --git a/src/slangpy_ext/utils/slangpytorchtensor.cpp b/src/slangpy_ext/utils/slangpytorchtensor.cpp index 8c4d4b56f..8ea497895 100644 --- a/src/slangpy_ext/utils/slangpytorchtensor.cpp +++ b/src/slangpy_ext/utils/slangpytorchtensor.cpp @@ -270,6 +270,12 @@ void NativeTorchTensorMarshall::write_shader_cursor_pre_dispatch( NativeTorchTensorDiffPair* pair; if (nb::try_cast(value, pair)) { // NativeTorchTensorDiffPair case + if (pair->primal.is_none()) { + SGL_THROW( + "NativeTorchTensorDiffPair primal tensor cannot be None. " + "Save the forward primal and pass it to the backward call because Slang replays the forward pass." + ); + } primal_value = pair->primal; grad_value = pair->grad; } else { diff --git a/src/slangpy_ext/utils/slangpytorchtensor.h b/src/slangpy_ext/utils/slangpytorchtensor.h index ab91eb69e..8f7a64a20 100644 --- a/src/slangpy_ext/utils/slangpytorchtensor.h +++ b/src/slangpy_ext/utils/slangpytorchtensor.h @@ -28,7 +28,7 @@ namespace sgl::slangpy { /// - grad: tensor to receive computed gradients (written by kernel) /// /// For outputs in backwards pass: -/// - primal: can be None (not needed) +/// - primal: the original output tensor value (read or rewritten by the backward replay) /// - grad: the upstream gradient from autograd (read by kernel) /// /// The index and is_input fields are used by the autograd hook to track @@ -45,7 +45,7 @@ class NativeTorchTensorDiffPair : public NativeObject { { } - /// The primal (value) tensor. May be None for output gradients in backwards. + /// The primal (value) tensor. Required because Slang's backward pass replays the forward pass. nb::object primal; /// The gradient tensor. For inputs: written by kernel. For outputs: read by kernel. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5cd43401f..c75049ea2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -45,7 +45,10 @@ if(SGL_BUILD_TESTS) endif() target_include_directories(sgl_tests BEFORE PRIVATE sgl) target_link_libraries(sgl_tests PRIVATE sgl header_only) - target_compile_definitions(sgl_tests PRIVATE SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") + target_compile_definitions(sgl_tests PRIVATE + DOCTEST_CONFIG_USE_STD_HEADERS + SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" + ) set_target_properties(sgl_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${SGL_RUNTIME_OUTPUT_DIRECTORY} diff --git a/tools/backfill_benchmarks.py b/tools/backfill_benchmarks.py new file mode 100644 index 000000000..5058da4a6 --- /dev/null +++ b/tools/backfill_benchmarks.py @@ -0,0 +1,580 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Safely dispatch bounded historical benchmark workflows with resumable state.""" + +import argparse +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import sys +import tempfile +import time +from typing import Any, Callable, Literal, Optional, Sequence, TypeVar + +try: + from tools.benchmark_actions import ( + Commit, + DispatchResult, + GitHubCli, + GitHubCliError, + WorkflowRun, + ) +except ModuleNotFoundError: + from benchmark_actions import Commit, DispatchResult, GitHubCli, GitHubCliError, WorkflowRun + +DEFAULT_REPOSITORY = "shader-slang/slangpy" +DEFAULT_BRANCH = "main" +DEFAULT_WORKFLOW = "backfill-benchmark.yml" +DEFAULT_STATE_PATH = Path(".temp/benchmark-backfill-state.json") +SUPPORTED_FLOOR_SHA = "f3ad0fd91d8cf4eeb2be3b505765b43482aa952a" +SUPPORTED_FLOOR_TIME = datetime(2025, 9, 2, 14, 42, 35, tzinfo=timezone.utc) +STATE_SCHEMA_VERSION = 2 +MAX_ACTIVE_RUNS = 4 +DEFAULT_POLL_SECONDS = 60.0 +DEFAULT_DISPATCH_GRACE = timedelta(minutes=10) + +BackfillStatus = Literal["pending", "dispatching", "dispatched"] +ResultType = TypeVar("ResultType") + + +class BackfillStateError(RuntimeError): + """Report corrupt or incompatible state without rewriting the source file.""" + + +@dataclass +class BackfillRecord: + """Track scheduling and returned workflow details for one supported commit.""" + + sha: str + committed_at: datetime + message: str + html_url: str + status: BackfillStatus = "pending" + dispatch_started_at: Optional[datetime] = None + run_id: Optional[int] = None + run_url: Optional[str] = None + + +@dataclass +class BackfillState: + """Bind additive commit scheduling state to one exact backfill configuration.""" + + schema_version: int + repository: str + branch: str + workflow: str + lower_bound: str + records: dict[str, BackfillRecord] + + +def _utc_text(value: datetime) -> str: + """Serialize an aware timestamp in stable RFC 3339 UTC form.""" + + if value.tzinfo is None: + raise BackfillStateError("Backfill timestamps must be timezone-aware.") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse_time(value: Any, field: str) -> datetime: + """Parse a required state timestamp with a configuration-focused error.""" + + if not isinstance(value, str): + raise BackfillStateError(f"State field {field!r} must be a timestamp string.") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise BackfillStateError(f"State field {field!r} is not a valid timestamp.") from error + if parsed.tzinfo is None: + raise BackfillStateError(f"State field {field!r} must include a timezone.") + return parsed.astimezone(timezone.utc) + + +def _optional_time(value: Any, field: str) -> Optional[datetime]: + """Parse an optional state timestamp while preserving an absent value.""" + + return None if value is None else _parse_time(value, field) + + +def _required_string(value: Any, field: str) -> str: + """Return one non-empty state string or identify the malformed field.""" + + if not isinstance(value, str) or not value: + raise BackfillStateError(f"State field {field!r} must be a non-empty string.") + return value + + +def _new_state(repository: str, branch: str, workflow: str, lower_bound: str) -> BackfillState: + """Create one empty state bound to the requested scheduler configuration.""" + + return BackfillState( + schema_version=STATE_SCHEMA_VERSION, + repository=repository, + branch=branch, + workflow=workflow, + lower_bound=lower_bound, + records={}, + ) + + +class BackfillStateStore: + """Load and atomically replace one versioned local scheduler state file.""" + + def __init__( + self, + path: Path, + repository: str, + branch: str, + workflow: str, + lower_bound: str, + ) -> None: + """Retain the exact configuration that every loaded state must match.""" + + super().__init__() + self.path = path + self.repository = repository + self.branch = branch + self.workflow = workflow + self.lower_bound = lower_bound + + def load(self) -> BackfillState: + """Load compatible state or return a new in-memory state when absent.""" + + if not self.path.exists(): + return _new_state( + self.repository, + self.branch, + self.workflow, + self.lower_bound, + ) + try: + document = json.loads(self.path.read_text(encoding="utf-8")) + return self._parse_document(document) + except (OSError, json.JSONDecodeError, BackfillStateError) as error: + raise BackfillStateError( + f"Cannot use backfill state {self.path}: {error}. Archive that file before " + "starting a new scheduler; it was not modified." + ) from error + + def _parse_document(self, document: Any) -> BackfillState: + """Validate state identity and materialize its commit records.""" + + if not isinstance(document, dict): + raise BackfillStateError("state root must be a JSON object") + expected = { + "schemaVersion": STATE_SCHEMA_VERSION, + "repository": self.repository, + "branch": self.branch, + "workflow": self.workflow, + "lowerBound": self.lower_bound, + } + mismatches = [ + f"{field}={document.get(field)!r} (expected {value!r})" + for field, value in expected.items() + if document.get(field) != value + ] + if mismatches: + raise BackfillStateError("incompatible state configuration: " + ", ".join(mismatches)) + raw_records = document.get("commits") + if not isinstance(raw_records, list): + raise BackfillStateError("state commits must be a JSON array") + records: dict[str, BackfillRecord] = {} + for index, raw_record in enumerate(raw_records): + record = self._parse_record(raw_record, index) + if record.sha in records: + raise BackfillStateError(f"duplicate state commit {record.sha}") + records[record.sha] = record + return BackfillState( + schema_version=STATE_SCHEMA_VERSION, + repository=self.repository, + branch=self.branch, + workflow=self.workflow, + lower_bound=self.lower_bound, + records=records, + ) + + def _parse_record(self, value: Any, index: int) -> BackfillRecord: + """Parse one commit record and enforce status-specific required fields.""" + + if not isinstance(value, dict): + raise BackfillStateError(f"state commit {index} must be a JSON object") + status = value.get("status") + if status not in ("pending", "dispatching", "dispatched"): + raise BackfillStateError(f"state commit {index} has invalid status {status!r}") + run_id = value.get("runId") + if run_id is not None and (not isinstance(run_id, int) or isinstance(run_id, bool)): + raise BackfillStateError(f"state commit {index} runId must be an integer or null") + record = BackfillRecord( + sha=_required_string(value.get("sha"), f"commits[{index}].sha"), + committed_at=_parse_time(value.get("committedAt"), f"commits[{index}].committedAt"), + message=_required_string(value.get("message"), f"commits[{index}].message"), + html_url=_required_string(value.get("htmlUrl"), f"commits[{index}].htmlUrl"), + status=status, + dispatch_started_at=_optional_time( + value.get("dispatchStartedAt"), f"commits[{index}].dispatchStartedAt" + ), + run_id=run_id, + run_url=value.get("runUrl"), + ) + if record.run_url is not None and not isinstance(record.run_url, str): + raise BackfillStateError(f"state commit {index} runUrl must be a string or null") + if record.status == "dispatching" and record.dispatch_started_at is None: + raise BackfillStateError(f"state commit {index} dispatching status lacks a timestamp") + if record.status == "dispatched" and (record.run_id is None or not record.run_url): + raise BackfillStateError(f"state commit {index} dispatched status lacks run details") + return record + + def save(self, state: BackfillState) -> None: + """Flush a complete sibling file and atomically replace the visible state.""" + + self.path.parent.mkdir(parents=True, exist_ok=True) + document = { + "schemaVersion": state.schema_version, + "repository": state.repository, + "branch": state.branch, + "workflow": state.workflow, + "lowerBound": state.lower_bound, + "commits": [ + { + "sha": record.sha, + "committedAt": _utc_text(record.committed_at), + "message": record.message, + "htmlUrl": record.html_url, + "status": record.status, + "dispatchStartedAt": ( + _utc_text(record.dispatch_started_at) + if record.dispatch_started_at is not None + else None + ), + "runId": record.run_id, + "runUrl": record.run_url, + } + for record in sorted( + state.records.values(), key=lambda item: (item.committed_at, item.sha) + ) + ], + } + data = (json.dumps(document, indent=2, sort_keys=False) + "\n").encode("utf-8") + temporary_path: Optional[Path] = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=self.path.parent, + prefix=f"{self.path.name}.", + suffix=".tmp", + delete=False, + ) as temporary: + temporary_path = Path(temporary.name) + temporary.write(data) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, self.path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def backfill_run_title(sha: str) -> str: + """Return the deterministic title used by the historical workflow.""" + + return f"backfill-benchmark: {sha}" + + +def supported_commits(commits: Sequence[Commit], lower_bound: str) -> list[Commit]: + """Return commits from the inclusive boundary forward in chronological order.""" + + ordered = sorted(commits, key=lambda commit: (commit.committed_at, commit.sha)) + for index, commit in enumerate(ordered): + if commit.sha == lower_bound: + return ordered[index:] + raise BackfillStateError( + f"Supported lower-bound commit {lower_bound} was not returned for main history." + ) + + +def merge_discovered_commits(state: BackfillState, commits: Sequence[Commit]) -> bool: + """Add newly discovered commits without changing any existing scheduling record.""" + + changed = False + for commit in commits: + if commit.sha in state.records: + continue + state.records[commit.sha] = BackfillRecord( + sha=commit.sha, + committed_at=commit.committed_at, + message=commit.message.splitlines()[0], + html_url=commit.html_url, + ) + changed = True + return changed + + +def reconcile_records( + state: BackfillState, + runs: Sequence[WorkflowRun], + now: datetime, + grace: timedelta, +) -> bool: + """Resolve deterministic titles and release expired uncertain dispatch markers.""" + + matching_runs: dict[str, WorkflowRun] = {} + for run in sorted(runs, key=lambda item: (item.created_at, item.run_id), reverse=True): + if run.title.startswith("backfill-benchmark: "): + matching_runs.setdefault(run.title, run) + changed = False + for record in state.records.values(): + matching = matching_runs.get(backfill_run_title(record.sha)) + if matching is not None: + if ( + record.status != "dispatched" + or record.run_id != matching.run_id + or record.run_url != matching.html_url + ): + record.status = "dispatched" + record.run_id = matching.run_id + record.run_url = matching.html_url + changed = True + continue + if ( + record.status == "dispatching" + and record.dispatch_started_at is not None + and now - record.dispatch_started_at >= grace + ): + record.status = "pending" + record.dispatch_started_at = None + changed = True + return changed + + +def active_backfill_count( + state: BackfillState, + runs: Sequence[WorkflowRun], + now: datetime, + grace: timedelta, +) -> int: + """Count active workflow runs plus recent dispatches not visible in GitHub yet.""" + + listed_run_ids = {run.run_id for run in runs} + active = sum(1 for run in runs if run.status != "completed") + for record in state.records.values(): + if record.status not in ("dispatching", "dispatched"): + continue + if record.run_id is not None and record.run_id in listed_run_ids: + continue + if record.dispatch_started_at is not None and now - record.dispatch_started_at < grace: + active += 1 + return active + + +def pending_records(state: BackfillState) -> list[BackfillRecord]: + """Return pending commits in stable oldest-first dispatch order.""" + + return sorted( + (record for record in state.records.values() if record.status == "pending"), + key=lambda record: (record.committed_at, record.sha), + ) + + +def incomplete_records(state: BackfillState) -> list[BackfillRecord]: + """Return commits that are pending or awaiting uncertain-dispatch reconciliation.""" + + return sorted( + (record for record in state.records.values() if record.status != "dispatched"), + key=lambda record: (record.committed_at, record.sha), + ) + + +def dispatch_cooldown_remaining( + state: BackfillState, + now: datetime, + minimum_interval_seconds: float, +) -> float: + """Return seconds before another dispatch is allowed across process restarts.""" + + dispatch_times = [ + record.dispatch_started_at + for record in state.records.values() + if record.dispatch_started_at is not None + ] + if not dispatch_times: + return 0.0 + elapsed = (now - max(dispatch_times)).total_seconds() + return max(0.0, minimum_interval_seconds - elapsed) + + +def retry_read_operation( + operation: Callable[[], ResultType], + description: str, + attempts: int = 3, + delay_seconds: float = 2.0, + sleep: Callable[[float], None] = time.sleep, +) -> ResultType: + """Retry read-only GitHub operations without duplicating workflow dispatches.""" + + if attempts < 1: + raise ValueError("Read retry attempts must be positive.") + for attempt in range(1, attempts + 1): + try: + return operation() + except GitHubCliError: + if attempt == attempts: + raise + print(f"{description} failed; retrying in {delay_seconds:g} second(s).") + sleep(delay_seconds) + raise AssertionError("unreachable read retry state") + + +def dispatch_oldest_pending( + github: GitHubCli, + store: BackfillStateStore, + state: BackfillState, + now: datetime, + output: Callable[[str], None] = print, +) -> Optional[DispatchResult]: + """Publish write-ahead state, dispatch one oldest commit, and save returned details.""" + + pending = pending_records(state) + if not pending: + return None + record = pending[0] + record.status = "dispatching" + record.dispatch_started_at = now + record.run_id = None + record.run_url = None + store.save(state) + result = github.dispatch_workflow( + state.repository, + state.workflow, + workflow_ref=state.branch, + inputs={"target_sha": record.sha}, + ) + record.status = "dispatched" + record.run_id = result.run_id + record.run_url = result.html_url + store.save(state) + output(f"Dispatched {record.sha}: {result.html_url}") + return result + + +def run_backfill_scheduler( + github: GitHubCli, + store: BackfillStateStore, + now_provider: Callable[[], datetime], + sleep: Callable[[float], None], + poll_seconds: float, + once: bool, + dry_run: bool, + output: Callable[[str], None] = print, +) -> int: + """Discover history, reconcile state, and dispatch at most one commit per interval.""" + + if poll_seconds < 0: + raise ValueError("Backfill poll interval must be non-negative.") + now = now_provider().astimezone(timezone.utc) + commits = retry_read_operation( + lambda: github.list_commits( + store.repository, + store.branch, + SUPPORTED_FLOOR_TIME, + now, + ), + "Commit discovery", + sleep=sleep, + ) + supported = supported_commits(commits, store.lower_bound) + state = store.load() + merge_discovered_commits(state, supported) + runs = retry_read_operation( + lambda: github.list_workflow_runs(store.repository, store.workflow, maximum=1000), + "Workflow reconciliation", + sleep=sleep, + ) + reconcile_records(state, runs, now, DEFAULT_DISPATCH_GRACE) + output( + f"Backfill {store.workflow} from {store.lower_bound}: {len(supported)} supported " + f"commit(s), {len(runs)} existing run(s), {len(pending_records(state))} pending." + ) + if dry_run: + for record in pending_records(state): + output(f"Would dispatch {store.workflow} for {record.sha} from {store.branch}.") + output( + f"Active backfill workflow count: " + f"{active_backfill_count(state, runs, now, DEFAULT_DISPATCH_GRACE)}." + ) + return 0 + store.save(state) + while incomplete_records(state): + now = now_provider().astimezone(timezone.utc) + reconcile_records(state, runs, now, DEFAULT_DISPATCH_GRACE) + store.save(state) + if not incomplete_records(state): + break + active = active_backfill_count(state, runs, now, DEFAULT_DISPATCH_GRACE) + output(f"Active backfill workflow count: {active}/{MAX_ACTIVE_RUNS}.") + cooldown = dispatch_cooldown_remaining(state, now, poll_seconds) + if active < MAX_ACTIVE_RUNS and cooldown <= 0: + dispatch_oldest_pending(github, store, state, now, output) + elif active < MAX_ACTIVE_RUNS: + output(f"Next backfill dispatch is allowed in {cooldown:g} second(s).") + if once: + return 0 + if not incomplete_records(state): + break + sleep(poll_seconds) + runs = retry_read_operation( + lambda: github.list_workflow_runs(store.repository, store.workflow, maximum=100), + "Workflow capacity poll", + sleep=sleep, + ) + output("All supported commits have been requested.") + return 0 + + +def _parser() -> argparse.ArgumentParser: + """Create the resumable backfill scheduler command-line parser.""" + + parser = argparse.ArgumentParser(description="Dispatch bounded historical benchmark workflows.") + parser.add_argument("--repository", default=DEFAULT_REPOSITORY) + parser.add_argument("--branch", default=DEFAULT_BRANCH) + parser.add_argument("--workflow", default=DEFAULT_WORKFLOW) + parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE_PATH) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--once", action="store_true") + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + """Run the backfill scheduler with durable Ctrl+C and operator error behavior.""" + + args = _parser().parse_args(argv) + store = BackfillStateStore( + args.state_file, + repository=args.repository, + branch=args.branch, + workflow=args.workflow, + lower_bound=SUPPORTED_FLOOR_SHA, + ) + print(f"State file: {store.path}") + print("Do not run another scheduler against this state file at the same time.") + try: + return run_backfill_scheduler( + GitHubCli(), + store, + now_provider=lambda: datetime.now(timezone.utc), + sleep=time.sleep, + poll_seconds=DEFAULT_POLL_SECONDS, + once=args.once, + dry_run=args.dry_run, + ) + except KeyboardInterrupt: + print("Backfill interrupted; durable state is ready for restart.", file=sys.stderr) + return 130 + except (BackfillStateError, GitHubCliError, OSError, ValueError) as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_actions.py b/tools/benchmark_actions.py new file mode 100644 index 000000000..c3996b133 --- /dev/null +++ b/tools/benchmark_actions.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Typed GitHub Actions operations implemented through the official ``gh`` CLI.""" + +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import shutil +import subprocess +from typing import Any, Optional, Protocol, Sequence + +GITHUB_API_VERSION = "2026-03-10" + + +class GitHubCliError(RuntimeError): + """Report an unavailable, unauthenticated, or malformed GitHub CLI operation.""" + + +class CommandRunner(Protocol): + """Describe the injectable subprocess boundary used by :class:`GitHubCli`.""" + + def __call__( + self, arguments: Sequence[str], input_text: Optional[str] + ) -> subprocess.CompletedProcess[str]: + """Execute one argument-array command and return its captured result.""" + + ... + + +@dataclass(frozen=True) +class Commit: + """Describe one commit reachable from the selected repository branch.""" + + sha: str + committed_at: datetime + message: str + html_url: str + + +@dataclass(frozen=True) +class WorkflowRun: + """Describe the GitHub fields needed for title reconciliation and capacity.""" + + run_id: int + title: str + status: str + conclusion: Optional[str] + html_url: str + head_sha: str + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True) +class DispatchResult: + """Describe the workflow run returned directly by a successful dispatch.""" + + run_id: int + run_url: str + html_url: str + + +def _default_command_runner( + arguments: Sequence[str], input_text: Optional[str] +) -> subprocess.CompletedProcess[str]: + """Run ``gh`` without a shell while capturing deterministic UTF-8 text output.""" + + return subprocess.run( + list(arguments), + input=input_text, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + shell=False, + ) + + +def _utc_text(value: datetime) -> str: + """Serialize an aware timestamp in the RFC 3339 UTC spelling expected by GitHub.""" + + if value.tzinfo is None: + raise ValueError("GitHub time bounds must be timezone-aware.") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse_datetime(value: Any, field: str) -> datetime: + """Parse one required GitHub timestamp and identify malformed response fields.""" + + if not isinstance(value, str): + raise GitHubCliError(f"GitHub response field {field!r} must be a timestamp string.") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise GitHubCliError( + f"GitHub response field {field!r} is not a valid timestamp." + ) from error + if parsed.tzinfo is None: + raise GitHubCliError(f"GitHub response field {field!r} must include a timezone.") + return parsed.astimezone(timezone.utc) + + +def _required_string(value: Any, field: str) -> str: + """Return a required non-empty GitHub string or raise a response-shape error.""" + + if not isinstance(value, str) or not value: + raise GitHubCliError(f"GitHub response field {field!r} must be a non-empty string.") + return value + + +def _flatten_pages(value: Any, collection_field: Optional[str]) -> list[Any]: + """Flatten the page arrays emitted by ``gh api --paginate --slurp``.""" + + if not isinstance(value, list): + raise GitHubCliError("GitHub CLI pagination output must be a JSON array.") + flattened: list[Any] = [] + for page in value: + collection = page + if collection_field is not None: + if not isinstance(page, dict): + raise GitHubCliError("GitHub workflow-run page must be a JSON object.") + collection = page.get(collection_field) + if not isinstance(collection, list): + raise GitHubCliError("GitHub CLI page does not contain the expected JSON array.") + flattened.extend(collection) + return flattened + + +class GitHubCli: + """Expose the GitHub operations needed by benchmark scheduling through ``gh``.""" + + def __init__( + self, + command_runner: Optional[CommandRunner] = None, + executable: Optional[str] = None, + ) -> None: + """Locate ``gh`` and retain an injectable no-shell command boundary.""" + + super().__init__() + resolved = executable if executable is not None else shutil.which("gh") + if resolved is None: + raise GitHubCliError( + "GitHub CLI 'gh' was not found. Install it and run 'gh auth login' before retrying." + ) + self._executable = resolved + self._command_runner = command_runner or _default_command_runner + + def _invoke(self, arguments: Sequence[str], input_text: Optional[str] = None) -> Any: + """Execute one versioned ``gh api`` call and decode its JSON response.""" + + command = [ + self._executable, + "api", + "--header", + f"X-GitHub-Api-Version: {GITHUB_API_VERSION}", + *arguments, + ] + result = self._command_runner(command, input_text) + if result.returncode != 0: + diagnostic = (result.stderr or result.stdout or "unknown error").strip()[:2048] + raise GitHubCliError( + f"GitHub CLI request failed with exit code {result.returncode}: {diagnostic}. " + "Run 'gh auth status' to verify authentication." + ) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + raise GitHubCliError("GitHub CLI returned invalid JSON.") from error + + def list_commits( + self, + repository: str, + branch: str, + since: datetime, + until: datetime, + ) -> list[Commit]: + """List branch commits in the inclusive committer-time interval.""" + + if until < since: + raise ValueError("GitHub commit interval end must not precede its start.") + response = self._invoke( + [ + "--method", + "GET", + "--paginate", + "--slurp", + f"repos/{repository}/commits", + "-f", + f"sha={branch}", + "-f", + f"since={_utc_text(since)}", + "-f", + f"until={_utc_text(until)}", + "-f", + "per_page=100", + ] + ) + commits: list[Commit] = [] + for index, item in enumerate(_flatten_pages(response, None)): + if not isinstance(item, dict): + raise GitHubCliError(f"GitHub commit at index {index} must be a JSON object.") + commit = item.get("commit") + if not isinstance(commit, dict): + raise GitHubCliError(f"GitHub commit at index {index} lacks commit data.") + committer = commit.get("committer") + if not isinstance(committer, dict): + raise GitHubCliError(f"GitHub commit at index {index} lacks committer data.") + commits.append( + Commit( + sha=_required_string(item.get("sha"), "sha"), + committed_at=_parse_datetime(committer.get("date"), "commit.committer.date"), + message=_required_string(commit.get("message"), "commit.message"), + html_url=_required_string(item.get("html_url"), "html_url"), + ) + ) + return commits + + def list_workflow_runs(self, repository: str, workflow: str, maximum: int) -> list[WorkflowRun]: + """List up to ``maximum`` newest runs for one workflow definition.""" + + if maximum < 1: + raise ValueError("Workflow run maximum must be positive.") + per_page = min(maximum, 100) + runs: list[WorkflowRun] = [] + page = 1 + while len(runs) < maximum: + response = self._invoke( + [ + "--method", + "GET", + f"repos/{repository}/actions/workflows/{workflow}/runs", + "-f", + f"per_page={per_page}", + "-f", + f"page={page}", + ] + ) + if not isinstance(response, dict): + raise GitHubCliError("GitHub workflow-run page must be a JSON object.") + items = response.get("workflow_runs") + if not isinstance(items, list): + raise GitHubCliError("GitHub workflow-run page lacks the workflow_runs JSON array.") + for item in items: + index = len(runs) + if not isinstance(item, dict): + raise GitHubCliError(f"GitHub workflow run at index {index} must be an object.") + run_id = item.get("id") + conclusion = item.get("conclusion") + if not isinstance(run_id, int) or isinstance(run_id, bool): + raise GitHubCliError("GitHub workflow run id must be an integer.") + if conclusion is not None and not isinstance(conclusion, str): + raise GitHubCliError("GitHub workflow run conclusion must be a string or null.") + runs.append( + WorkflowRun( + run_id=run_id, + title=_required_string(item.get("display_title"), "display_title"), + status=_required_string(item.get("status"), "status"), + conclusion=conclusion, + html_url=_required_string(item.get("html_url"), "html_url"), + head_sha=_required_string(item.get("head_sha"), "head_sha"), + created_at=_parse_datetime(item.get("created_at"), "created_at"), + updated_at=_parse_datetime(item.get("updated_at"), "updated_at"), + ) + ) + if len(runs) == maximum: + break + if len(items) < per_page or len(runs) == maximum: + break + page += 1 + return runs + + def dispatch_workflow( + self, + repository: str, + workflow: str, + workflow_ref: str, + inputs: dict[str, str], + ) -> DispatchResult: + """Dispatch one workflow and return GitHub's immediate run identity and URLs.""" + + body = json.dumps( + { + "ref": workflow_ref, + "inputs": inputs, + "return_run_details": True, + }, + separators=(",", ":"), + sort_keys=True, + ) + response = self._invoke( + [ + "--method", + "POST", + "--input", + "-", + f"repos/{repository}/actions/workflows/{workflow}/dispatches", + ], + body, + ) + if not isinstance(response, dict): + raise GitHubCliError("GitHub workflow dispatch response must be a JSON object.") + run_id = response.get("workflow_run_id") + if not isinstance(run_id, int) or isinstance(run_id, bool): + raise GitHubCliError("GitHub workflow dispatch response lacks workflow_run_id.") + return DispatchResult( + run_id=run_id, + run_url=_required_string(response.get("run_url"), "run_url"), + html_url=_required_string(response.get("html_url"), "html_url"), + ) diff --git a/tools/ci.py b/tools/ci.py index 7734e8ee9..41076caf0 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -196,8 +196,6 @@ def benchmark_python(args: Any): # Lock GPU clocks if args.lock_gpu_clocks: cmd = ["python", str(PROJECT_DIR / "tools/gpu_clock.py"), "lock", "--ratio", "0.7"] - if os_name == "linux": - cmd = ["sudo"] + cmd run_command(cmd) # Run benchmarks for each device type @@ -205,11 +203,12 @@ def benchmark_python(args: Any): print(f"Running benchmarks for device type: {device_type}") cmd = pytest_command("slangpy/benchmarks", "-ra", "--device-types", device_type) - if args.mongodb_connection_string: - cmd += ["--benchmark-upload", args.run_id] - cmd += ["--benchmark-mongodb-connection-string", args.mongodb_connection_string] - if args.mongodb_database_name: - cmd += ["--benchmark-mongodb-database-name", args.mongodb_database_name] + api_url = ( + args.api_url if args.api_url is not None else os.environ.get("BENCHVIEW_API_URL") + ) + if api_url is not None: + cmd += ["--benchmark-submit", args.run_id] + cmd += ["--benchmark-api-url", api_url] try: run_command(cmd, env=env) @@ -222,8 +221,6 @@ def benchmark_python(args: Any): # Unlock GPU clocks if args.lock_gpu_clocks: cmd = ["python", str(PROJECT_DIR / "tools/gpu_clock.py"), "unlock"] - if os_name == "linux": - cmd = ["sudo"] + cmd run_command(cmd) @@ -306,12 +303,14 @@ def main(): parser_benchmark_python = commands.add_parser( "benchmark-python", help="run benchmarks (python)" ) - parser_benchmark_python.add_argument("-r", "--run-id", type=str, required=True, help="Run ID") parser_benchmark_python.add_argument( - "-c", "--mongodb-connection-string", type=str, help="MongoDB connection string" + "-r", "--run-id", type=str, required=True, help="Traceable BenchView request ID" ) parser_benchmark_python.add_argument( - "-d", "--mongodb-database-name", type=str, help="MongoDB database name" + "-u", + "--api-url", + type=str, + help="BenchView base URL; defaults to BENCHVIEW_API_URL and uses BENCHVIEW_API_KEY", ) parser_benchmark_python.add_argument( "--device-type", diff --git a/tools/gpu_clock.py b/tools/gpu_clock.py index 7ccf892b9..86237336d 100755 --- a/tools/gpu_clock.py +++ b/tools/gpu_clock.py @@ -32,6 +32,20 @@ def run_command(cmd: list[str]) -> str: return subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True).strip() +def nvidia_smi_mutation_command(arguments: list[str]) -> list[str]: + """ + Build an nvidia-smi command for an operation that changes GPU state. + + Linux CI grants ci-runner passwordless sudo only for the exact clock + lock/reset argument forms used below. The Python process and read-only + nvidia-smi queries remain unprivileged. + """ + command = [NVIDIA_SMI, *arguments] + if platform.system() == "Linux": + command = ["sudo", "-n", "--", *command] + return command + + def get_gpu_name(device_index: int): """ Return the name of the GPU. @@ -113,10 +127,14 @@ def lock_gpu_clocks(device_index: int, ratio: float, conservative: bool): print(f"Selected gpu clock: {locked_gpu_clock} MHz ({locked_gpu_clock / max_gpu_clock:.1%}):") print("Locking mem clock:") - cmd = [NVIDIA_SMI, "-i", str(device_index), f"--lock-memory-clocks={locked_mem_clock}"] + cmd = nvidia_smi_mutation_command( + ["-i", str(device_index), f"--lock-memory-clocks={locked_mem_clock}"] + ) print(run_command(cmd)) print("Locking gpu clock") - cmd = [NVIDIA_SMI, "-i", str(device_index), f"--lock-gpu-clocks={locked_gpu_clock}"] + cmd = nvidia_smi_mutation_command( + ["-i", str(device_index), f"--lock-gpu-clocks={locked_gpu_clock}"] + ) print(run_command(cmd)) @@ -126,9 +144,11 @@ def unlock_gpu_clocks(device_index: int): """ print(f"Selected GPU: {get_gpu_name(device_index)}") print("Unlocking mem clock:") - print(run_command([NVIDIA_SMI, "-i", str(device_index), "--reset-memory-clocks"])) + print( + run_command(nvidia_smi_mutation_command(["-i", str(device_index), "--reset-memory-clocks"])) + ) print("Unlocking gpu clock:") - print(run_command([NVIDIA_SMI, "-i", str(device_index), "--reset-gpu-clocks"])) + print(run_command(nvidia_smi_mutation_command(["-i", str(device_index), "--reset-gpu-clocks"]))) def main():