From 9d5ad57b67915897deb9412f32f1c4035176b796 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 18:04:42 -0400 Subject: [PATCH 1/4] ci: integrate shared Python benchmark reports --- .github/workflows/benchmark-report.yml | 21 +++++++++++++++++++++ .github/workflows/benchmarks.yml | 20 ++++++++++++++++++++ benchmarks/README.md | 6 +++--- 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/benchmark-report.yml diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml new file mode 100644 index 00000000..7d014ab6 --- /dev/null +++ b/.github/workflows/benchmark-report.yml @@ -0,0 +1,21 @@ +name: Python SDK benchmark report + +on: + workflow_run: + workflows: ['Python SDK benchmarks'] + types: [completed] + +permissions: {} + +jobs: + report: + permissions: + contents: read + actions: read + pull-requests: write + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@ced4b9694b90c9221bc277b1c8d9f013eafe56c5 # SDK benchmark workflow (PostHog/.github PR #84) + with: + workflow-path: .github/workflows/benchmarks.yml + artifact-name: python-benchmark-results + report-key: posthog-python + sdk: posthog-python diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index eecd9849..e55ec98f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -9,6 +9,7 @@ on: - 'setup.py' - 'uv.lock' - '.github/workflows/benchmarks.yml' + - '.github/workflows/benchmark-report.yml' permissions: {} @@ -61,6 +62,7 @@ jobs: run: .venv-benchmark/bin/python -B benchmarks/test_benchmarks.py - name: Measure paired base and head + id: measure run: | .venv-benchmark/bin/python -B benchmarks/run.py \ --base "$BASE_WORKTREE" \ @@ -68,6 +70,24 @@ jobs: --output "$RESULTS" \ --repetitions 6 + - name: Render benchmark comparison + if: ${{ !cancelled() && (steps.measure.outcome == 'success' || steps.measure.outcome == 'failure') }} + uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@a558711336bd3b81be4a820ed10f26372fc568ed # SDK benchmark reporting (harness PR #93) + with: + mode: render + base-results: ${{ runner.temp }}/python-benchmark-results/base.json + head-results: ${{ runner.temp }}/python-benchmark-results/head.json + output-directory: ${{ runner.temp }}/python-benchmark-report + + - name: Upload rendered comparison + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-benchmark-report + path: ${{ runner.temp }}/python-benchmark-report + if-no-files-found: ignore + retention-days: 14 + - name: Upload raw benchmark results if: always() id: results diff --git a/benchmarks/README.md b/benchmarks/README.md index 8ca37a75..1a5fc8c6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -110,8 +110,8 @@ Each worker has a two-minute timeout and an externally monitored **512 MiB RSS c ## Results and CI -The pilot JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. Shared schema finalization and rendering are separate work. +The shared JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. The contract, semantic validation, comparisons, and rendering live in [posthog-sdk-test-harness](https://github.com/PostHog/posthog-sdk-test-harness/tree/main/benchmarks). -The PR workflow builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and uploads JSON artifacts linked from the job summary. +The PR workflow builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and invokes the SHA-pinned harness action to render validated comparisons in the job summary. Raw JSON and rendered reports are uploaded as separate artifacts. -Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Shared comparisons, noise classification, and PR comments remain separate. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor. +Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Comparisons expose descriptive medians and variability rather than calibrated regression verdicts. A separate `workflow_run` caller uses the SHA-pinned reusable workflow in `PostHog/.github` to verify source-run/artifact/PR identities and publish comments without executing PR code with write permissions. This caller must land on the default branch before automatic comments are active, including for fork PRs. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor. From 3435499ebbef08cb4fbab392caecf580ce81e912 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 18:19:31 -0400 Subject: [PATCH 2/4] fix(ci): report aborted benchmark suites and retain pending publishers --- .github/workflows/benchmark-report.yml | 2 +- .github/workflows/benchmarks.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml index 7d014ab6..8ecccc08 100644 --- a/.github/workflows/benchmark-report.yml +++ b/.github/workflows/benchmark-report.yml @@ -13,7 +13,7 @@ jobs: contents: read actions: read pull-requests: write - uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@ced4b9694b90c9221bc277b1c8d9f013eafe56c5 # SDK benchmark workflow (PostHog/.github PR #84) + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@b03aeaffe0b3c69a93ab366ee89eb42048d985b6 # SDK benchmark workflow (PostHog/.github PR #84) with: workflow-path: .github/workflows/benchmarks.yml artifact-name: python-benchmark-results diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e55ec98f..ca1a60f3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -72,9 +72,10 @@ jobs: - name: Render benchmark comparison if: ${{ !cancelled() && (steps.measure.outcome == 'success' || steps.measure.outcome == 'failure') }} - uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@a558711336bd3b81be4a820ed10f26372fc568ed # SDK benchmark reporting (harness PR #93) + uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@ef2507cce61b81327033aaa6ae5bcc8d96698121 # SDK benchmark reporting (harness PR #93) with: mode: render + source-status: ${{ steps.measure.outcome }} base-results: ${{ runner.temp }}/python-benchmark-results/base.json head-results: ${{ runner.temp }}/python-benchmark-results/head.json output-directory: ${{ runner.temp }}/python-benchmark-report From ddb839316bb9ad3fea0f514ce52564b674a6bff7 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 19:17:25 -0400 Subject: [PATCH 3/4] fix(ci): run every PR benchmark with organization-owned reporting --- .github/workflows/benchmark-report.yml | 2 +- .github/workflows/benchmarks.yml | 13 ++----------- benchmarks/README.md | 4 ++-- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml index 8ecccc08..89b30782 100644 --- a/.github/workflows/benchmark-report.yml +++ b/.github/workflows/benchmark-report.yml @@ -13,7 +13,7 @@ jobs: contents: read actions: read pull-requests: write - uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@b03aeaffe0b3c69a93ab366ee89eb42048d985b6 # SDK benchmark workflow (PostHog/.github PR #84) + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@7eec66436a29a83ec39e7416937c2b66d4e31e2a # SDK benchmark workflow (PostHog/.github PR #84) with: workflow-path: .github/workflows/benchmarks.yml artifact-name: python-benchmark-results diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ca1a60f3..647b89ff 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,15 +1,6 @@ name: Python SDK benchmarks -on: - pull_request: - paths: - - 'posthog/**' - - 'benchmarks/**' - - 'pyproject.toml' - - 'setup.py' - - 'uv.lock' - - '.github/workflows/benchmarks.yml' - - '.github/workflows/benchmark-report.yml' +on: [pull_request] permissions: {} @@ -72,7 +63,7 @@ jobs: - name: Render benchmark comparison if: ${{ !cancelled() && (steps.measure.outcome == 'success' || steps.measure.outcome == 'failure') }} - uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@ef2507cce61b81327033aaa6ae5bcc8d96698121 # SDK benchmark reporting (harness PR #93) + uses: PostHog/.github/.github/actions/benchmark-report@62bbf249cd14175adfa6eab19477b3c5df08245a # SDK benchmark reporting (PostHog/.github PR #84) with: mode: render source-status: ${{ steps.measure.outcome }} diff --git a/benchmarks/README.md b/benchmarks/README.md index 1a5fc8c6..b5561ffd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -110,8 +110,8 @@ Each worker has a two-minute timeout and an externally monitored **512 MiB RSS c ## Results and CI -The shared JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. The contract, semantic validation, comparisons, and rendering live in [posthog-sdk-test-harness](https://github.com/PostHog/posthog-sdk-test-harness/tree/main/benchmarks). +The shared JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. The contract, semantic validation, comparisons, and rendering live in [PostHog/.github](https://github.com/PostHog/.github/blob/main/.github/workflows/sdk-benchmark-report.md). -The PR workflow builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and invokes the SHA-pinned harness action to render validated comparisons in the job summary. Raw JSON and rendered reports are uploaded as separate artifacts. +The workflow runs for every pull request opened, reopened, or updated in `PostHog/posthog-python`. It builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and invokes the SHA-pinned reporting action in `PostHog/.github` to render validated comparisons in the job summary. Raw JSON and rendered reports are uploaded as separate artifacts. Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Comparisons expose descriptive medians and variability rather than calibrated regression verdicts. A separate `workflow_run` caller uses the SHA-pinned reusable workflow in `PostHog/.github` to verify source-run/artifact/PR identities and publish comments without executing PR code with write permissions. This caller must land on the default branch before automatic comments are active, including for fork PRs. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor. From f6e7a01f71a18f6d0417c5b6b80aeafe5ee5804d Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 19:19:34 -0400 Subject: [PATCH 4/4] chore(ci): advance benchmark reporting pins --- .github/workflows/benchmark-report.yml | 2 +- .github/workflows/benchmarks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml index 89b30782..034ff5a3 100644 --- a/.github/workflows/benchmark-report.yml +++ b/.github/workflows/benchmark-report.yml @@ -13,7 +13,7 @@ jobs: contents: read actions: read pull-requests: write - uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@7eec66436a29a83ec39e7416937c2b66d4e31e2a # SDK benchmark workflow (PostHog/.github PR #84) + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@0ee11ac0d55505c626d9ecb987ffeebf812c9e44 # SDK benchmark workflow (PostHog/.github PR #84) with: workflow-path: .github/workflows/benchmarks.yml artifact-name: python-benchmark-results diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 647b89ff..6f441dc8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -63,7 +63,7 @@ jobs: - name: Render benchmark comparison if: ${{ !cancelled() && (steps.measure.outcome == 'success' || steps.measure.outcome == 'failure') }} - uses: PostHog/.github/.github/actions/benchmark-report@62bbf249cd14175adfa6eab19477b3c5df08245a # SDK benchmark reporting (PostHog/.github PR #84) + uses: PostHog/.github/.github/actions/benchmark-report@a9b17fcb0e7d962e924d33053f2e40257d674629 # SDK benchmark reporting (PostHog/.github PR #84) with: mode: render source-status: ${{ steps.measure.outcome }}