diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml new file mode 100644 index 00000000..034ff5a3 --- /dev/null +++ b/.github/workflows/benchmark-report.yml @@ -0,0 +1,21 @@ +name: Python SDK benchmark report + +on: + workflow_run: + workflows: ['Python SDK benchmarks'] + types: [completed] + +permissions: {} + +jobs: + report: + permissions: + contents: read + actions: read + pull-requests: write + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@0ee11ac0d55505c626d9ecb987ffeebf812c9e44 # SDK benchmark workflow (PostHog/.github PR #84) + with: + workflow-path: .github/workflows/benchmarks.yml + artifact-name: python-benchmark-results + report-key: posthog-python + sdk: posthog-python diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index eecd9849..6f441dc8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,14 +1,6 @@ name: Python SDK benchmarks -on: - pull_request: - paths: - - 'posthog/**' - - 'benchmarks/**' - - 'pyproject.toml' - - 'setup.py' - - 'uv.lock' - - '.github/workflows/benchmarks.yml' +on: [pull_request] permissions: {} @@ -61,6 +53,7 @@ jobs: run: .venv-benchmark/bin/python -B benchmarks/test_benchmarks.py - name: Measure paired base and head + id: measure run: | .venv-benchmark/bin/python -B benchmarks/run.py \ --base "$BASE_WORKTREE" \ @@ -68,6 +61,25 @@ jobs: --output "$RESULTS" \ --repetitions 6 + - name: Render benchmark comparison + if: ${{ !cancelled() && (steps.measure.outcome == 'success' || steps.measure.outcome == 'failure') }} + uses: PostHog/.github/.github/actions/benchmark-report@a9b17fcb0e7d962e924d33053f2e40257d674629 # SDK benchmark reporting (PostHog/.github PR #84) + with: + mode: render + source-status: ${{ steps.measure.outcome }} + base-results: ${{ runner.temp }}/python-benchmark-results/base.json + head-results: ${{ runner.temp }}/python-benchmark-results/head.json + output-directory: ${{ runner.temp }}/python-benchmark-report + + - name: Upload rendered comparison + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-benchmark-report + path: ${{ runner.temp }}/python-benchmark-report + if-no-files-found: ignore + retention-days: 14 + - name: Upload raw benchmark results if: always() id: results diff --git a/benchmarks/README.md b/benchmarks/README.md index 8ca37a75..b5561ffd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -110,8 +110,8 @@ Each worker has a two-minute timeout and an externally monitored **512 MiB RSS c ## Results and CI -The pilot JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. Shared schema finalization and rendering are separate work. +The shared JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. The contract, semantic validation, comparisons, and rendering live in [PostHog/.github](https://github.com/PostHog/.github/blob/main/.github/workflows/sdk-benchmark-report.md). -The PR workflow builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and uploads JSON artifacts linked from the job summary. +The workflow runs for every pull request opened, reopened, or updated in `PostHog/posthog-python`. It builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and invokes the SHA-pinned reporting action in `PostHog/.github` to render validated comparisons in the job summary. Raw JSON and rendered reports are uploaded as separate artifacts. -Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Shared comparisons, noise classification, and PR comments remain separate. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor. +Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Comparisons expose descriptive medians and variability rather than calibrated regression verdicts. A separate `workflow_run` caller uses the SHA-pinned reusable workflow in `PostHog/.github` to verify source-run/artifact/PR identities and publish comments without executing PR code with write permissions. This caller must land on the default branch before automatic comments are active, including for fork PRs. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor.