CleanBench full suite #74
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CleanBench full suite | |
| on: | |
| schedule: | |
| - cron: "30 4 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| cleanbench: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| issues: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Install | |
| run: | | |
| python -m pip install -U pip | |
| pip install -e ".[bench,cli]" | |
| - name: Restore perf baseline | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: benchmarks/cleanbench/results/baseline_v1.json | |
| key: cleanbench-baseline-${{ runner.os }}-v1 | |
| - name: Full CleanBench T1-T5 (release gates + baselines) | |
| run: python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline | |
| - name: Verify committed result is reproducible | |
| if: always() | |
| run: python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json | |
| - name: Audit README trust claims | |
| if: always() | |
| run: python -m benchmarks.cleanbench.reproducibility audit-readme | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: cleanbench-results | |
| path: benchmarks/cleanbench/results/latest.* | |
| retention-days: 30 | |
| - name: Open/refresh alert issue on failure | |
| if: failure() | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const title = "nightly CleanBench suite failing"; | |
| const body = `The scheduled CleanBench full suite failed.\n\n` + | |
| `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + | |
| `/actions/runs/${context.runId}`; | |
| const open = await github.rest.issues.listForRepo({ | |
| ...context.repo, state: "open", labels: "nightly-failure", | |
| }); | |
| const existing = open.data.find(i => i.title === title); | |
| if (existing) { | |
| await github.rest.issues.createComment({ | |
| ...context.repo, issue_number: existing.number, body, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| ...context.repo, title, body, labels: ["nightly-failure"], | |
| }); | |
| } |