Performance regression #78
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance regression | |
| # Same-run A/B perf gate on the CleanBench T5 workload: HEAD is measured against | |
| # a base commit on the same runner, interleaved, and fails only when it is >20% | |
| # slower or uses >15% more peak memory than base AND a confirmation re-run | |
| # reproduces the breach. No absolute numbers from other machines are compared. | |
| # | |
| # pull_request PR head vs the PR base commit (blocking) | |
| # daily main vs main as of ~26 hours earlier (blocking, alert issue) | |
| # weekly main vs the latest release tag (report only) | |
| # manual any base_ref; accept_regression=true reports without failing | |
| on: | |
| schedule: | |
| - cron: "0 5 * * *" | |
| - cron: "30 5 * * 1" | |
| pull_request: | |
| paths: | |
| - "src/freshdata/**" | |
| - "benchmarks/cleanbench/**" | |
| - ".github/workflows/perf-regression.yml" | |
| workflow_dispatch: | |
| inputs: | |
| base_ref: | |
| description: "Base ref to compare against (default: main as of 26h ago)" | |
| type: string | |
| default: "" | |
| accept_regression: | |
| description: "Report only; do not fail on a regression" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: perf-regression-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| perf: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 # the base commit and release tags must be resolvable | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| - name: Install | |
| run: | | |
| python -m pip install -U pip | |
| pip install -e ".[bench]" | |
| - name: Resolve base commit | |
| id: base | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| SCHEDULE: ${{ github.event.schedule }} | |
| PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| INPUT_BASE_REF: ${{ inputs.base_ref }} | |
| INPUT_ACCEPT: ${{ inputs.accept_regression }} | |
| run: | | |
| set -euo pipefail | |
| mode=gate | |
| if [ "$EVENT" = "pull_request" ]; then | |
| ref="$PR_BASE_SHA" | |
| elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_BASE_REF" ]; then | |
| ref="$INPUT_BASE_REF" | |
| elif [ "$SCHEDULE" = "30 5 * * 1" ]; then | |
| ref="$(git describe --tags --abbrev=0 --match 'v*' origin/main)" | |
| mode=report | |
| else | |
| ref="$(git rev-list -1 --before='26 hours ago' origin/main)" | |
| fi | |
| if [ "$INPUT_ACCEPT" = "true" ]; then mode=report; fi | |
| sha="$(git rev-parse --verify "${ref}^{commit}")" | |
| { | |
| echo "ref=$ref" | |
| echo "sha=$sha" | |
| echo "mode=$mode" | |
| } >> "$GITHUB_OUTPUT" | |
| if [ "$sha" = "$(git rev-parse HEAD)" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "Base \`$ref\` is HEAD; nothing to compare." >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| git worktree add --detach "$RUNNER_TEMP/perf-base" "$sha" | |
| fi | |
| - name: Same-run A/B perf gate | |
| if: steps.base.outputs.skip != 'true' | |
| env: | |
| BASE_REF: ${{ steps.base.outputs.ref }} | |
| BASE_SHA: ${{ steps.base.outputs.sha }} | |
| MODE: ${{ steps.base.outputs.mode }} | |
| run: | | |
| GATE="" | |
| if [ "$MODE" = "gate" ]; then GATE="--check-gates"; fi | |
| python -m benchmarks.cleanbench.ab \ | |
| --base-src "$RUNNER_TEMP/perf-base/src" \ | |
| --head-src src \ | |
| --base-label "$BASE_REF (${BASE_SHA:0:7})" \ | |
| --head-label "HEAD (${GITHUB_SHA:0:7})" \ | |
| --output benchmarks/cleanbench/results/latest.ab.json \ | |
| --summary "$GITHUB_STEP_SUMMARY" \ | |
| $GATE | |
| - name: Upload results | |
| if: always() && steps.base.outputs.skip == 'false' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: perf-ab-results | |
| path: benchmarks/cleanbench/results/latest.ab.json | |
| retention-days: 30 | |
| - name: Open/refresh alert issue on failure | |
| if: failure() && github.event_name == 'schedule' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const title = "nightly perf-regression gate failing"; | |
| const body = `The scheduled same-run A/B performance gate failed: main ran ` + | |
| `>20% slower or used >15% more peak memory than main from ~26 hours ` + | |
| `earlier, reproduced on a confirmation run (or the job errored).\n\n` + | |
| `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + | |
| `/actions/runs/${context.runId}`; | |
| const open = await github.rest.issues.listForRepo({ | |
| ...context.repo, state: "open", labels: "nightly-failure", | |
| }); | |
| const existing = open.data.find(i => i.title === title); | |
| if (existing) { | |
| await github.rest.issues.createComment({ | |
| ...context.repo, issue_number: existing.number, body, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| ...context.repo, title, body, labels: ["nightly-failure"], | |
| }); | |
| } |