From ced4b9694b90c9221bc277b1c8d9f013eafe56c5 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 18:03:27 -0400 Subject: [PATCH 01/14] ci: add trusted SDK benchmark report workflow --- .../scripts/sdk-benchmark-workflow.test.mjs | 45 +++++++++++++ .../workflows/sdk-benchmark-report-tests.yml | 30 +++++++++ .github/workflows/sdk-benchmark-report.md | 54 +++++++++++++++ .github/workflows/sdk-benchmark-report.yml | 65 +++++++++++++++++++ README.md | 4 ++ 5 files changed, 198 insertions(+) create mode 100644 .github/scripts/sdk-benchmark-workflow.test.mjs create mode 100644 .github/workflows/sdk-benchmark-report-tests.yml create mode 100644 .github/workflows/sdk-benchmark-report.md create mode 100644 .github/workflows/sdk-benchmark-report.yml diff --git a/.github/scripts/sdk-benchmark-workflow.test.mjs b/.github/scripts/sdk-benchmark-workflow.test.mjs new file mode 100644 index 0000000..b75ec81 --- /dev/null +++ b/.github/scripts/sdk-benchmark-workflow.test.mjs @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' + +const workflow = readFileSync(new URL('../workflows/sdk-benchmark-report.yml', import.meta.url), 'utf8') + +test('reporting is restricted to completed workflow_run events', () => { + assert.match(workflow, /workflow_call:/) + assert.match(workflow, /if: github\.event_name == 'workflow_run' && github\.event\.action == 'completed'/) + assert.doesNotMatch(workflow, /pull_request_target:/) +}) + +test('publisher uses pinned trusted code and never checks out caller code', () => { + const uses = [...workflow.matchAll(/\buses: (\S+)/g)].map((match) => match[1]) + assert.equal(uses.length, 3) + for (const action of uses) assert.match(action, /^[\w./-]+@[0-9a-f]{40}$/) + assert.ok(uses.some((action) => action.startsWith('PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@'))) + assert.ok(uses[0].startsWith('step-security/harden-runner@')) + assert.doesNotMatch(workflow, /actions\/checkout|actions\/cache|secrets: inherit|script-ref|reporter-ref/) +}) + +test('publishers serialize without cancellation and share a report-specific lock', () => { + assert.match(workflow, /group: sdk-benchmark-report-\$\{\{ github\.repository \}\}-\$\{\{ inputs\.report-key \}\}/) + assert.match(workflow, /cancel-in-progress: false/) +}) + +test('permissions and token are explicit and scoped to the publisher', () => { + assert.match(workflow, /permissions: \{\}/) + assert.match(workflow, /contents: read\s+actions: read\s+pull-requests: write/) + assert.doesNotMatch(workflow, /write-all|contents: write|id-token: write/) + assert.equal((workflow.match(/GH_TOKEN:/g) || []).length, 1) + assert.match(workflow, /GH_TOKEN: \$\{\{ github\.token \}\}/) + assert.match(workflow, /timeout-minutes: 10/) + assert.match(workflow, /egress-policy: audit/) +}) + +test('caller constants reach the publisher as inputs rather than shell interpolation', () => { + for (const input of ['workflow-path', 'artifact-name', 'report-key', 'sdk']) { + assert.match(workflow, new RegExp(`${input}: \\$\\{\\{ inputs\\.${input} \\}\\}`)) + } + assert.match(workflow, /mode: publish/) + assert.doesNotMatch(workflow, /\brun:|\bscript:/) + assert.match(workflow, /if: always\(\)/) + assert.match(workflow, /if-no-files-found: ignore/) +}) diff --git a/.github/workflows/sdk-benchmark-report-tests.yml b/.github/workflows/sdk-benchmark-report-tests.yml new file mode 100644 index 0000000..2143a9c --- /dev/null +++ b/.github/workflows/sdk-benchmark-report-tests.yml @@ -0,0 +1,30 @@ +name: SDK benchmark workflow tests + +on: + pull_request: + paths: + - '.github/workflows/sdk-benchmark-report*.yml' + - '.github/scripts/sdk-benchmark-workflow.test.mjs' + push: + branches: [main] + paths: + - '.github/workflows/sdk-benchmark-report*.yml' + - '.github/scripts/sdk-benchmark-workflow.test.mjs' + +permissions: {} + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24.13.0' + package-manager-cache: false + - run: node --test .github/scripts/sdk-benchmark-workflow.test.mjs diff --git a/.github/workflows/sdk-benchmark-report.md b/.github/workflows/sdk-benchmark-report.md new file mode 100644 index 0000000..818623c --- /dev/null +++ b/.github/workflows/sdk-benchmark-report.md @@ -0,0 +1,54 @@ +# SDK benchmark reporting + +`sdk-benchmark-report.yml` publishes informational performance reports from an SDK repository's completed PR benchmark runs. The SDK owns execution and measurements. `PostHog/posthog-sdk-test-harness` owns the result schema, comparison, renderer, artifact verification, and comment updater; this workflow provides the trusted job boundary and permissions. + +## Caller + +Add a **separate** workflow in the SDK repository, pinning this reusable workflow to a reviewed full commit SHA: + +```yaml +name: Node SDK benchmark report +on: + workflow_run: + workflows: [Node SDK benchmarks] + types: [completed] +permissions: {} +jobs: + report: + permissions: + contents: read + actions: read + pull-requests: write + uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@ + with: + workflow-path: .github/workflows/node-benchmarks.yml + artifact-name: node-benchmark-results + report-key: posthog-node + sdk: posthog-node +``` + +Use repository-owned constants for all four inputs. `report-key` is a stable, SDK-specific comment namespace, not a PR number. `sdk` must match both result documents. The producer artifact contains `base.json`, `head.json`, and `revisions.json` (`target`, `base`, and `head` full commit SHAs). + +The producing `pull_request` workflow remains read-only and can separately invoke the harness action in render-only mode for a job summary. It must not receive the publisher's writable token or service credentials. No manually supplied PR number or artifact-controlled command is accepted by the publisher. + +## Trust and ordering + +- Only completed `workflow_run` events can invoke the publishing job. No caller checkout occurs; reporting code is pinned to a harness commit in this workflow. +- The action verifies the workflow identity, source run and attempt, repository, PR association, head SHA, and merge-base against GitHub metadata before consuming results. Fork associations must be unambiguous. +- Artifacts are bounded, validated data, never executable code. Report text is escaped; result metadata cannot choose the comment's target or URLs. +- The action only updates its exact marker on the expected GitHub Actions bot's comment. It rechecks the current PR head and run attempt and rejects older reports before publication. +- A repository/report-key concurrency group serializes publishers, including fork runs whose event does not identify a PR. Cancellation is disabled so a late older event cannot cancel a newer report. GitHub may replace pending jobs; the source-run guard still prevents stale writes. +- Egress auditing runs first. Audit mode observes traffic; it does not prevent exfiltration or replace least privilege. +- Rendered Markdown and comparison JSON are retained for 14 days, including when publishing cannot complete. The source SDK workflow retains the raw measurement artifacts. + +## Rollout and validation + +The caller's `workflow_run` workflow must be on its default branch before it can service completed benchmark runs. Adding it to an unmerged stacked PR does not enable privileged comment publication. Land the reporting components and caller configuration through normal review; do not work around activation by executing PR code with a writable token. + +Unit tests for schema, comparisons, hostile artifacts, PR identity, forks, and stale comments live with the harness action. This repository's structural tests guard the workflow's pins, event restrictions, permissions, concurrency, token scope, and lack of caller checkout: + +```sh +node --test .github/scripts/sdk-benchmark-workflow.test.mjs +``` + +Also run Actionlint on both `sdk-benchmark-report.yml` and `sdk-benchmark-report-tests.yml`. Performance differences are informational, not merge gates; invalid inputs and operational errors remain visible. Linux runner calibration is separate from correctness of report generation. diff --git a/.github/workflows/sdk-benchmark-report.yml b/.github/workflows/sdk-benchmark-report.yml new file mode 100644 index 0000000..dd4134c --- /dev/null +++ b/.github/workflows/sdk-benchmark-report.yml @@ -0,0 +1,65 @@ +name: SDK benchmark report + +on: + workflow_call: + inputs: + workflow-path: + description: 'Expected producer workflow path in the calling repository.' + required: true + type: string + artifact-name: + description: 'Exact producer artifact name containing base.json, head.json and revisions.json.' + required: true + type: string + report-key: + description: 'Stable comment namespace, unique per SDK within the calling repository.' + required: true + type: string + sdk: + description: 'Expected SDK identifier in both result documents.' + required: true + type: string + +permissions: {} + +jobs: + report: + # A default-branch workflow_run caller is the trust boundary, including for fork PRs. + if: github.event_name == 'workflow_run' && github.event.action == 'completed' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + actions: read + pull-requests: write + # Serialize publishers even when workflow_run has no PR association (common for forks). + # The publisher rechecks head/attempt and source-run ordering immediately before writing. + concurrency: + group: sdk-benchmark-report-${{ github.repository }}-${{ inputs.report-key }} + cancel-in-progress: false + steps: + - name: Audit runner egress + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Verify source run and publish report + uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@a558711336bd3b81be4a820ed10f26372fc568ed # SDK benchmark reporting (harness PR #93) + env: + GH_TOKEN: ${{ github.token }} + with: + mode: publish + workflow-path: ${{ inputs.workflow-path }} + artifact-name: ${{ inputs.artifact-name }} + report-key: ${{ inputs.report-key }} + sdk: ${{ inputs.sdk }} + output-directory: ${{ runner.temp }}/sdk-benchmark-report + + - name: Preserve rendered report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sdk-benchmark-report-${{ inputs.report-key }} + path: ${{ runner.temp }}/sdk-benchmark-report + if-no-files-found: ignore + retention-days: 14 diff --git a/README.md b/README.md index dd3628c..6e4bb93 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ For example, repositories can reference workflows from here using: uses: PostHog/.github/.github/workflows/reusable-workflow.yml@main ``` +### SDK performance reports + +[SDK benchmark reporting](.github/workflows/sdk-benchmark-report.md) consumes read-only PR benchmark artifacts in a separate trusted `workflow_run` job and updates an informational PR comment. + ## 📄 Organization-wide community health files Files like `CODE_OF_CONDUCT.md`, `SECURITY.md`, and other community health files placed here become the default for all repositories in the organization that don't have their own versions. From b03aeaffe0b3c69a93ab366ee89eb42048d985b6 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 18:19:22 -0400 Subject: [PATCH 02/14] fix: queue pending SDK publishers and pin failure reporting fixes --- .github/scripts/sdk-benchmark-workflow.test.mjs | 1 + .github/workflows/sdk-benchmark-report-tests.yml | 5 +++++ .github/workflows/sdk-benchmark-report.md | 4 ++-- .github/workflows/sdk-benchmark-report.yml | 3 ++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/scripts/sdk-benchmark-workflow.test.mjs b/.github/scripts/sdk-benchmark-workflow.test.mjs index b75ec81..28ae9dd 100644 --- a/.github/scripts/sdk-benchmark-workflow.test.mjs +++ b/.github/scripts/sdk-benchmark-workflow.test.mjs @@ -22,6 +22,7 @@ test('publisher uses pinned trusted code and never checks out caller code', () = test('publishers serialize without cancellation and share a report-specific lock', () => { assert.match(workflow, /group: sdk-benchmark-report-\$\{\{ github\.repository \}\}-\$\{\{ inputs\.report-key \}\}/) assert.match(workflow, /cancel-in-progress: false/) + assert.match(workflow, /queue: max/) }) test('permissions and token are explicit and scoped to the publisher', () => { diff --git a/.github/workflows/sdk-benchmark-report-tests.yml b/.github/workflows/sdk-benchmark-report-tests.yml index 2143a9c..415ee20 100644 --- a/.github/workflows/sdk-benchmark-report-tests.yml +++ b/.github/workflows/sdk-benchmark-report-tests.yml @@ -17,6 +17,11 @@ jobs: test: runs-on: ubuntu-24.04 timeout-minutes: 5 + # Exercise the same concurrency syntax on a read-only job before publisher rollout. + concurrency: + group: sdk-benchmark-workflow-test-${{ github.run_id }} + cancel-in-progress: false + queue: max permissions: contents: read steps: diff --git a/.github/workflows/sdk-benchmark-report.md b/.github/workflows/sdk-benchmark-report.md index 818623c..1759061 100644 --- a/.github/workflows/sdk-benchmark-report.md +++ b/.github/workflows/sdk-benchmark-report.md @@ -37,7 +37,7 @@ The producing `pull_request` workflow remains read-only and can separately invok - The action verifies the workflow identity, source run and attempt, repository, PR association, head SHA, and merge-base against GitHub metadata before consuming results. Fork associations must be unambiguous. - Artifacts are bounded, validated data, never executable code. Report text is escaped; result metadata cannot choose the comment's target or URLs. - The action only updates its exact marker on the expected GitHub Actions bot's comment. It rechecks the current PR head and run attempt and rejects older reports before publication. -- A repository/report-key concurrency group serializes publishers, including fork runs whose event does not identify a PR. Cancellation is disabled so a late older event cannot cancel a newer report. GitHub may replace pending jobs; the source-run guard still prevents stale writes. +- A repository/report-key concurrency group serializes publishers, including fork runs whose event does not identify a PR. Cancellation is disabled so a late older event cannot cancel a newer report. `queue: max` preserves up to 100 pending publishers instead of replacing another PR's pending report. GitHub cancels additional arrivals when that queue is full; the source-run guard still prevents stale writes. - Egress auditing runs first. Audit mode observes traffic; it does not prevent exfiltration or replace least privilege. - Rendered Markdown and comparison JSON are retained for 14 days, including when publishing cannot complete. The source SDK workflow retains the raw measurement artifacts. @@ -51,4 +51,4 @@ Unit tests for schema, comparisons, hostile artifacts, PR identity, forks, and s node --test .github/scripts/sdk-benchmark-workflow.test.mjs ``` -Also run Actionlint on both `sdk-benchmark-report.yml` and `sdk-benchmark-report-tests.yml`. Performance differences are informational, not merge gates; invalid inputs and operational errors remain visible. Linux runner calibration is separate from correctness of report generation. +Also run Actionlint on both `sdk-benchmark-report.yml` and `sdk-benchmark-report-tests.yml`. GitHub [supports `queue: max`](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idconcurrency), but Actionlint 1.7.12 does not yet recognize that key; ignore only its `unexpected key "queue" for "concurrency" section` diagnostic until support lands. The read-only test job also uses this setting so GitHub validates the syntax before publisher rollout. Performance differences are informational, not merge gates; invalid inputs and operational errors remain visible. Linux runner calibration is separate from correctness of report generation. diff --git a/.github/workflows/sdk-benchmark-report.yml b/.github/workflows/sdk-benchmark-report.yml index dd4134c..e66f056 100644 --- a/.github/workflows/sdk-benchmark-report.yml +++ b/.github/workflows/sdk-benchmark-report.yml @@ -37,6 +37,7 @@ jobs: concurrency: group: sdk-benchmark-report-${{ github.repository }}-${{ inputs.report-key }} cancel-in-progress: false + queue: max steps: - name: Audit runner egress uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 @@ -44,7 +45,7 @@ jobs: egress-policy: audit - name: Verify source run and publish report - uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@a558711336bd3b81be4a820ed10f26372fc568ed # SDK benchmark reporting (harness PR #93) + uses: PostHog/posthog-sdk-test-harness/.github/actions/benchmark-report@ef2507cce61b81327033aaa6ae5bcc8d96698121 # SDK benchmark reporting (harness PR #93) env: GH_TOKEN: ${{ github.token }} with: From 62bbf249cd14175adfa6eab19477b3c5df08245a Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 27 Oct 2026 19:15:21 -0400 Subject: [PATCH 03/14] feat: own SDK benchmark reporting alongside reusable workflows --- .github/actions/benchmark-report/.gitignore | 4 + .github/actions/benchmark-report/action.yml | 68 + .../benchmark-report/benchmarks/__init__.py | 0 .../benchmarks/report/__init__.py | 0 .../benchmark-report/benchmarks/report/cli.py | 68 + .../benchmarks/report/compare.py | 194 ++ .../benchmarks/report/github.py | 143 ++ .../benchmarks/report/publish.py | 212 +++ .../benchmarks/report/render.py | 213 +++ .../benchmarks/report/validate.py | 114 ++ .../benchmarks/schema/result.schema.json | 537 ++++++ .../actions/benchmark-report/pyproject.toml | 30 + .../benchmark-report/tests/__init__.py | 0 .../tests/fixtures/benchmarks/node-base.json | 667 +++++++ .../tests/fixtures/benchmarks/node-head.json | 667 +++++++ .../fixtures/benchmarks/node-headline.json | 1677 +++++++++++++++++ .../fixtures/benchmarks/python-base.json | 629 +++++++ .../fixtures/benchmarks/python-head.json | 629 +++++++ .../tests/test_benchmark_github.py | 215 +++ .../tests/test_benchmark_publish.py | 377 ++++ .../tests/test_benchmark_report.py | 451 +++++ .github/actions/benchmark-report/uv.lock | 287 +++ .../workflows/sdk-benchmark-report-tests.yml | 28 + .github/workflows/sdk-benchmark-report.md | 101 +- AGENTS.md | 11 +- README.md | 2 +- 26 files changed, 7318 insertions(+), 6 deletions(-) create mode 100644 .github/actions/benchmark-report/.gitignore create mode 100644 .github/actions/benchmark-report/action.yml create mode 100644 .github/actions/benchmark-report/benchmarks/__init__.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/__init__.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/cli.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/compare.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/github.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/publish.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/render.py create mode 100644 .github/actions/benchmark-report/benchmarks/report/validate.py create mode 100644 .github/actions/benchmark-report/benchmarks/schema/result.schema.json create mode 100644 .github/actions/benchmark-report/pyproject.toml create mode 100644 .github/actions/benchmark-report/tests/__init__.py create mode 100644 .github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json create mode 100644 .github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json create mode 100644 .github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json create mode 100644 .github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json create mode 100644 .github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json create mode 100644 .github/actions/benchmark-report/tests/test_benchmark_github.py create mode 100644 .github/actions/benchmark-report/tests/test_benchmark_publish.py create mode 100644 .github/actions/benchmark-report/tests/test_benchmark_report.py create mode 100644 .github/actions/benchmark-report/uv.lock diff --git a/.github/actions/benchmark-report/.gitignore b/.github/actions/benchmark-report/.gitignore new file mode 100644 index 0000000..ae4022a --- /dev/null +++ b/.github/actions/benchmark-report/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ diff --git a/.github/actions/benchmark-report/action.yml b/.github/actions/benchmark-report/action.yml new file mode 100644 index 0000000..b678fa5 --- /dev/null +++ b/.github/actions/benchmark-report/action.yml @@ -0,0 +1,68 @@ +name: SDK benchmark report +description: Validate and render paired SDK results, or publish from a verified workflow_run +inputs: + mode: + description: render (offline) or publish (trusted workflow_run only) + default: render + source-status: + description: Measurement step outcome for render mode; publish mode uses the verified source run conclusion + default: '' + base-results: + description: Baseline result JSON path for render mode + default: '' + head-results: + description: Head result JSON path for render mode + default: '' + output-directory: + description: Directory for report.md and comparison.json + default: benchmark-report + workflow-path: + description: Trusted literal producer workflow path for publish mode + default: '' + artifact-name: + description: Trusted literal producer artifact name for publish mode + default: '' + report-key: + description: Trusted stable comment namespace for publish mode + default: '' + sdk: + description: Trusted literal SDK identity for publish mode + default: '' +outputs: + report-path: + description: Absolute Markdown report path + value: ${{ steps.report.outputs.report-path }} + comparison-path: + description: Absolute comparison JSON path + value: ${{ steps.report.outputs.comparison-path }} +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: '0.13.4' + python-version: '3.12.12' + enable-cache: 'false' + - id: report + shell: bash + env: + REPORT_ACTION_ROOT: ${{ github.action_path }} + REPORT_MODE: ${{ inputs.mode }} + REPORT_SOURCE_STATUS: ${{ inputs.source-status }} + REPORT_BASE: ${{ inputs.base-results }} + REPORT_HEAD: ${{ inputs.head-results }} + REPORT_OUTPUT: ${{ inputs.output-directory }} + REPORT_WORKFLOW: ${{ inputs.workflow-path }} + REPORT_ARTIFACT: ${{ inputs.artifact-name }} + REPORT_KEY: ${{ inputs.report-key }} + REPORT_SDK: ${{ inputs.sdk }} + run: | + set -euo pipefail + root="$(cd "$REPORT_ACTION_ROOT" && pwd)" + uv sync --no-config --project "$root" --frozen --no-dev --no-install-project + uv run --no-config --project "$root" --no-sync \ + python -I "$root/benchmarks/report/cli.py" \ + --mode "$REPORT_MODE" --source-status "$REPORT_SOURCE_STATUS" \ + --base-results "$REPORT_BASE" --head-results "$REPORT_HEAD" \ + --output-directory "$REPORT_OUTPUT" --workflow-path "$REPORT_WORKFLOW" \ + --artifact-name "$REPORT_ARTIFACT" --report-key "$REPORT_KEY" --sdk "$REPORT_SDK" diff --git a/.github/actions/benchmark-report/benchmarks/__init__.py b/.github/actions/benchmark-report/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/benchmarks/report/__init__.py b/.github/actions/benchmark-report/benchmarks/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/benchmarks/report/cli.py b/.github/actions/benchmark-report/benchmarks/report/cli.py new file mode 100644 index 0000000..248266b --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/cli.py @@ -0,0 +1,68 @@ +"""Offline renderer and trusted workflow_run publisher entry point.""" + +import argparse +import os +import sys +from pathlib import Path + +# Support absolute-path invocation from a composite action without installing the project. +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.report.compare import SOURCE_STATUSES, compare +from benchmarks.report.github import GitHub +from benchmarks.report.publish import Publisher +from benchmarks.report.render import render, write_outputs +from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, load_result, require + + +def main(argv=None, environ=None) -> int: + env = os.environ if environ is None else environ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("render", "publish"), default="render") + parser.add_argument("--source-status", choices=sorted(SOURCE_STATUSES | {""}), default="") + parser.add_argument("--base-results", default="") + parser.add_argument("--head-results", default="") + parser.add_argument("--output-directory", default="benchmark-report") + parser.add_argument("--workflow-path", default="") + parser.add_argument("--artifact-name", default="") + parser.add_argument("--report-key", default="") + parser.add_argument("--sdk", default="") + args = parser.parse_args(argv) + try: + if args.mode == "render": + require(args.base_results and args.head_results, "Both result paths are required") + comparison = compare( + load_result(args.base_results), load_result(args.head_results), source_status=args.source_status or None + ) + report = render(comparison) + else: + require(env.get("GITHUB_EVENT_NAME") == "workflow_run", "Publish requires a workflow_run event") + require(bool(env.get("GITHUB_EVENT_PATH")), "Missing GitHub event file") + with Path(env["GITHUB_EVENT_PATH"]).open("rb") as stream: + event = bounded_json(stream.read(MAX_DOCUMENT_BYTES + 1), text_limit=65536) + publisher = Publisher( + GitHub(env.get("GH_TOKEN", "")), + env.get("GITHUB_REPOSITORY", ""), + args.workflow_path, + args.artifact_name, + args.report_key, + args.sdk, + ) + comparison, report = publisher.publish(event) + write_outputs(args.output_directory, comparison, report, env) + return 1 if comparison.get("execution_failed") else 0 + except (InvalidResult, OSError, ValueError) as exc: + # Never print paths, API responses, raw logs or arbitrary exception text. + reason = str(exc) if isinstance(exc, InvalidResult) else "Report input or output could not be processed" + report = f"## SDK performance\n\n**Reporting failed:** {reason}.\n" + try: + write_outputs(args.output_directory, {"comparison_version": 1, "error": reason}, report, env) + except (OSError, ValueError): + pass + print(reason, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/benchmark-report/benchmarks/report/compare.py b/.github/actions/benchmark-report/benchmarks/report/compare.py new file mode 100644 index 0000000..a6964e1 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/compare.py @@ -0,0 +1,194 @@ +"""Descriptive paired comparisons, without a statistical performance gate.""" + +import math +from statistics import median + +from .validate import require, validate + +# These describe the SDK under test, not the measurement environment. +SDK_BUILD_FIELDS = {"dependency_lock_digest", "build_digest", "core_build", "entry", "sdk_version", "packages"} +FAILURES = {"error", "timeout", "oom"} +SOURCE_STATUSES = { + "success", + "failure", + "cancelled", + "timed_out", + "action_required", + "neutral", + "skipped", + "stale", + "startup_failure", +} + + +def without(value: dict, keys: set) -> dict: + return {key: item for key, item in value.items() if key not in keys} + + +def global_mismatches(base: dict, head: dict) -> list[str]: + reasons = [] + if base["sdk"] != head["sdk"]: + reasons.append("SDK differs") + if base["suite"] != head["suite"]: + reasons.append("suite/source/profile/measurement configuration differs") + if without(base["execution"], SDK_BUILD_FIELDS) != without(head["execution"], SDK_BUILD_FIELDS): + reasons.append("paired execution environment differs") + return reasons + + +def run_configuration(workload: dict): + return sorted( + ( + run["pass"], + run["repetition"], + run.get("operations"), + run.get("warmup_operations"), + [point["operations"] for point in run.get("checkpoints", [])], + ) + for run in workload.get("runs", []) + ) + + +def failed_statuses(workload: dict) -> list[str]: + return sorted({s for s in [workload["status"], *(r["status"] for r in workload.get("runs", []))] if s != "ok"}) + + +def _percentage(base: float, head: float): + if base == 0: + return None + value = (head - base) / base * 100 + return value if math.isfinite(value) else None + + +def compare(base: dict, head: dict, *, source_status: str | None = None) -> dict: + require( + source_status is None or (isinstance(source_status, str) and source_status in SOURCE_STATUSES), + "Invalid source execution status", + ) + source_failed = source_status not in (None, "success") + validate(base) + validate(head) + require(base["revision"]["role"] == "base" and head["revision"]["role"] == "head", "Wrong revision roles") + mismatch = global_mismatches(base, head) + rows = [] + workloads = {} + execution_failed = source_failed + bases = {b["id"]: b for b in base["benchmarks"]} + heads = {b["id"]: b for b in head["benchmarks"]} + for workload_id in sorted(bases.keys() | heads.keys()): + left, right = bases.get(workload_id), heads.get(workload_id) + workloads[workload_id] = { + role: { + "parameters": workload["parameters"], + "fixture_digest": workload["fixture_digest"], + "status": workload["status"], + "reason": workload.get("reason"), + "failures": [ + { + key: run[key] + for key in ( + "pass", + "repetition", + "status", + "reason", + "error", + "code", + "signal", + "rss_limit_exceeded", + ) + if key in run + } + for run in workload.get("runs", []) + if run["status"] != "ok" + ], + "checkpoints": [ + {"repetition": r["repetition"], "pass": r["pass"], "points": r["checkpoints"]} + for r in workload.get("runs", []) + if "checkpoints" in r + ], + } + for role, workload in (("base", left), ("head", right)) + if workload + } + metrics = [{m["id"]: m for m in w["metrics"]} if w else {} for w in (left, right)] + # A failed workload with no metrics still needs a prominent row. + for metric_id in sorted(metrics[0].keys() | metrics[1].keys()) or ["(workload)"]: + bm, hm = (m.get(metric_id) for m in metrics) + row = {"workload": workload_id, "metric": metric_id, "base": bm, "head": hm, "status": "inconclusive"} + failures = [] + row["explanations"] = [] + row_failed = False + for role, workload, metric in (("base", left, bm), ("head", right, hm)): + if workload: + for source in (workload, metric): + if source and source["status"] != "ok" and source.get("reason"): + row["explanations"].append(f"{role}: {source['reason']}") + statuses = set(failed_statuses(workload)) + if metric and metric["status"] != "ok": + statuses.add(metric["status"]) + failures.extend(f"{role}: {status}" for status in sorted(statuses)) + row_failed |= bool(statuses & FAILURES) + execution_failed |= row_failed + reasons = list(mismatch) + if source_failed or failures: + row.update( + status="unavailable", + reasons=([f"Source execution: {source_status}"] if source_failed else []) + failures, + execution_failed=source_failed or row_failed, + ) + elif left is None or bm is None: + row.update(status="no baseline", reasons=["baseline workload or metric missing"]) + elif right is None or hm is None: + row.update(status="missing head", reasons=["head workload or metric missing"]) + else: + if left["parameters"] != right["parameters"] or left["fixture_digest"] != right["fixture_digest"]: + reasons.append("workload parameters or fixture differ") + if run_configuration(left) != run_configuration(right): + reasons.append("raw run identities, operations, warmup or checkpoints differ") + if without(bm, {"status", "samples", "reason"}) != without(hm, {"status", "samples", "reason"}): + reasons.append("metric unit, method, quality, direction or configuration differs") + bs = {s["repetition"]: s for s in bm.get("samples", [])} + hs = {s["repetition"]: s for s in hm.get("samples", [])} + if bs.keys() != hs.keys() or any( + bs[k]["operations"] != hs[k]["operations"] for k in bs.keys() & hs.keys() + ): + reasons.append("paired sample identities or operation counts differ") + if reasons: + row.update(status="not comparable", reasons=reasons) + elif bs: + bvalues, hvalues = ([s[k]["value"] for k in sorted(s)] for s in (bs, hs)) + bmedian, hmedian = median(bvalues), median(hvalues) + deltas = [h - b for b, h in zip(bvalues, hvalues)] + percentages = [_percentage(b, h) for b, h in zip(bvalues, hvalues)] + row.update( + base_median=bmedian, + head_median=hmedian, + change_percent=_percentage(bmedian, hmedian), + pairs=len(bs), + paired_delta={"median": median(deltas), "min": min(deltas), "max": max(deltas)}, + paired_change_percent=percentages, + reasons=["Descriptive only; uncertainty is not calibrated"], + ) + if bmedian == 0: + row["reasons"].append("Zero baseline; percentage undefined") + if len(bs) != base["suite"]["repetitions"]: + row["reasons"].append("Incomplete repetitions") + else: + row.update(status="unavailable", reasons=["No completed paired samples"]) + rows.append(row) + rows.sort( + key=lambda r: (not r.get("execution_failed", False), r["status"] == "inconclusive", r["workload"], r["metric"]) + ) + return { + "comparison_version": 1, + "sdk": head["sdk"], + "execution_failed": execution_failed, + "source_execution_status": source_status, + "compatibility": mismatch, + "context": { + role: {k: doc[k] for k in ("revision", "suite", "execution")} + for role, doc in (("base", base), ("head", head)) + }, + "workloads": workloads, + "rows": rows, + } diff --git a/.github/actions/benchmark-report/benchmarks/report/github.py b/.github/actions/benchmark-report/benchmarks/report/github.py new file mode 100644 index 0000000..0124ce5 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/github.py @@ -0,0 +1,143 @@ +"""Small bounded GitHub.com client. Signed downloads never receive API credentials.""" + +import io +import re +import stat +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +import zlib +from http.client import HTTPException + +from .render import json_text +from .validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, require + +API_ORIGIN = "https://api.github.com" +MAX_ARCHIVE_BYTES = 20 * 1024 * 1024 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class GitHub: + def __init__(self, token: str): + require(bool(token) and not any(c in token for c in "\r\n"), "GH_TOKEN is required") + self.token = token + self.opener = urllib.request.build_opener(NoRedirect()) + self.requests = 0 + self.deadline = time.monotonic() + 180 + + def _request(self, url: str, *, authenticated: bool, method="GET", body=None, limit=MAX_DOCUMENT_BYTES): + require(self.requests < 100 and time.monotonic() < self.deadline, "GitHub request budget exceeded") + self.requests += 1 + parsed = urllib.parse.urlsplit(url) + require( + parsed.scheme == "https" + and parsed.hostname + and not parsed.username + and not parsed.password + and not parsed.fragment, + "Unsafe download URL", + ) + headers = {"User-Agent": "posthog-sdk-benchmark-report", "Accept": "application/vnd.github+json"} + if authenticated: + require(f"{parsed.scheme}://{parsed.netloc}" == API_ORIGIN, "Untrusted API origin") + headers.update(Authorization=f"Bearer {self.token}", **{"X-GitHub-Api-Version": "2022-11-28"}) + data = json_text(body).encode() if body is not None else None + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, headers=headers, data=data, method=method) + try: + response = self.opener.open(request, timeout=min(20, max(1, self.deadline - time.monotonic()))) + except urllib.error.HTTPError as exc: + response = exc + except (OSError, ValueError, HTTPException): + raise InvalidResult("GitHub transport failed") from None + try: + with response: + status = response.code + location = response.headers.get("Location") + content = response.read(limit + 1) + require(len(content) <= limit, "GitHub response exceeds limit") + return status, location, content + except (OSError, ValueError, HTTPException) as exc: + if isinstance(exc, InvalidResult): + raise + raise InvalidResult("GitHub response could not be read") from None + + def api(self, path: str, method="GET", body=None): + require(path.startswith("/repos/") and not any(c in path for c in "\r\n#"), "Invalid API path") + status, _, data = self._request(API_ORIGIN + path, authenticated=True, method=method, body=body) + require(200 <= status < 300, f"GitHub API request failed (HTTP {status})") + # Compare responses may contain large patches; only their merge-base metadata is consumed. + comparison = method == "GET" and re.fullmatch( + r"/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/compare/[0-9a-f]{40}\.\.\.[0-9a-f]{40}", path + ) + return bounded_json(data, text_limit=MAX_DOCUMENT_BYTES if comparison else 65536) if data else None + + def pages(self, path: str, key: str | None = None) -> list: + result = [] + for page in range(1, 11): + response = self.api(f"{path}?per_page=100&page={page}") + items = response[key] if key else response + require(isinstance(items, list) and len(items) <= 100, "Invalid paginated API response") + result.extend(items) + if len(items) < 100: + return result + raise InvalidResult("GitHub pagination limit exceeded") + + def download(self, path: str) -> bytes: + require( + re.fullmatch(r"/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/artifacts/[1-9][0-9]*/zip", path), + "Invalid artifact endpoint", + ) + status, location, _ = self._request(API_ORIGIN + path, authenticated=True) + require(status == 302 and location, "Expected signed artifact download redirect") + for _ in range(3): + status, next_location, data = self._request(location, authenticated=False, limit=MAX_ARCHIVE_BYTES) + if status == 200: + return data + require(status in (301, 302, 303, 307, 308) and next_location, "Artifact download failed") + location = urllib.parse.urljoin(location, next_location) + raise InvalidResult("Artifact redirect limit exceeded") + + +def read_archive(data: bytes) -> dict[str, bytes]: + require(len(data) <= MAX_ARCHIVE_BYTES, "Compressed artifact exceeds size limit") + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + entries = archive.infolist() + require(len(entries) == 3, "Artifact must contain exactly three result files") + require( + {entry.filename for entry in entries} == {"base.json", "head.json", "revisions.json"}, + "Unexpected or duplicate artifact filename", + ) + require( + sum(entry.file_size for entry in entries) <= MAX_ARCHIVE_BYTES, + "Uncompressed artifact exceeds size limit", + ) + result = {} + for entry in entries: + require(entry.orig_filename == entry.filename, "Invalid artifact filename") + mode = stat.S_IFMT(entry.external_attr >> 16) + require(mode in (0, stat.S_IFREG) and not entry.is_dir(), "Artifact contains a non-regular file") + require(not entry.flag_bits & 1, "Encrypted artifacts are unsupported") + require( + entry.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), "Unsupported ZIP compression" + ) + limit = 8192 if entry.filename == "revisions.json" else MAX_DOCUMENT_BYTES + require(entry.file_size <= limit, "Artifact member exceeds size limit") + require( + entry.file_size <= max(1, entry.compress_size) * 200, "Artifact compression ratio exceeds limit" + ) + with archive.open(entry) as stream: + content = stream.read(limit + 1) + require(len(content) == entry.file_size and len(content) <= limit, "Invalid ZIP member size") + result[entry.filename] = content + return result + except (zipfile.BadZipFile, RuntimeError, OSError, EOFError, NotImplementedError, UnicodeError, zlib.error): + raise InvalidResult("Malformed ZIP artifact") from None diff --git a/.github/actions/benchmark-report/benchmarks/report/publish.py b/.github/actions/benchmark-report/benchmarks/report/publish.py new file mode 100644 index 0000000..d8f8fad --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/publish.py @@ -0,0 +1,212 @@ +"""Verify GitHub identities before consuming data or updating a bot comment.""" + +import re +from datetime import datetime + +from .compare import compare +from .github import MAX_ARCHIVE_BYTES, GitHub, read_archive +from .render import render +from .validate import InvalidResult, bounded_json, require, validate + +SHA = re.compile(r"[0-9a-f]{40}") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +KEY = re.compile(r"[A-Za-z0-9_.-]{1,100}") + + +class SkipPublication(Exception): + """Verified metadata cannot establish a safe, current publication target.""" + + +def positive_id(value) -> int: + require(type(value) is int and 0 < value < 2**63, "Invalid GitHub numeric identity") + return value + + +def sha(value) -> str: + require(isinstance(value, str) and SHA.fullmatch(value), "Invalid GitHub commit identity") + return value + + +def timestamp(value) -> datetime: + require(isinstance(value, str), "Missing GitHub timestamp") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + require(result.tzinfo is not None, "Missing timestamp timezone") + return result + except ValueError: + raise InvalidResult("Invalid GitHub timestamp") from None + + +def same_repository(left: dict, right: dict) -> bool: + return ( + positive_id(left["id"]) == positive_id(right["id"]) and left["full_name"].lower() == right["full_name"].lower() + ) + + +class Publisher: + def __init__(self, api: GitHub, repository: str, workflow_path: str, artifact_name: str, report_key: str, sdk: str): + require(REPOSITORY.fullmatch(repository), "Invalid caller repository") + require(re.fullmatch(r"\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml", workflow_path), "Invalid workflow path") + require(KEY.fullmatch(report_key), "Invalid report key") + require(0 < len(artifact_name) <= 256 and 0 < len(sdk) <= 256, "Missing artifact name or SDK") + self.api = api + self.repository = repository + self.root = f"/repos/{repository}" + self.workflow_path = workflow_path + self.artifact_name = artifact_name + self.sdk = sdk + self.marker = f"" + self.comparison = None + self.report = None + + def verified_run(self, event: dict) -> dict: + require(event["action"] == "completed", "Expected a completed workflow_run event") + original = event["workflow_run"] + require(event["repository"]["full_name"].lower() == self.repository.lower(), "Event repository differs") + run_id = positive_id(original["id"]) + run = self.api.api(f"{self.root}/actions/runs/{run_id}") + require( + run["id"] == run_id and same_repository(run["repository"], event["repository"]), + "Run repository or ID differs", + ) + require( + run["event"] == "pull_request" and run["status"] == "completed", "Unexpected source run event or status" + ) + require(sha(run["head_sha"]) == sha(original["head_sha"]), "Source run head differs") + if positive_id(run["run_attempt"]) != positive_id(original["run_attempt"]): + raise SkipPublication("Source run attempt has been superseded") + positive_id(run["run_number"]) + workflow = self.api.api(f"{self.root}/actions/workflows/{positive_id(run['workflow_id'])}") + require( + workflow["id"] == run["workflow_id"] and workflow["path"] == self.workflow_path, "Source workflow differs" + ) + return run + + def matches_pr(self, pr: dict, run: dict) -> bool: + return ( + pr["state"] == "open" + and pr["head"]["sha"] == run["head_sha"] + and pr["head"]["repo"] is not None + and same_repository(pr["head"]["repo"], run["head_repository"]) + and same_repository(pr["base"]["repo"], run["repository"]) + ) + + def associate(self, run: dict) -> tuple[dict, str, bool]: + associations = run["pull_requests"] + require(isinstance(associations, list) and len(associations) <= 100, "Invalid PR associations") + fallback = not associations + candidates = associations or self.api.pages(f"{self.root}/commits/{sha(run['head_sha'])}/pulls") + matches = [] + seen = set() + for candidate in candidates: + number = positive_id(candidate["number"]) + require(number not in seen, "Duplicate PR association") + seen.add(number) + pr = self.api.api(f"{self.root}/pulls/{number}") + require(pr["number"] == number, "PR number differs") + if self.matches_pr(pr, run): + target = sha(pr["base"]["sha"] if fallback else candidate["base"]["sha"]) + if not fallback: + require(candidate["head"]["sha"] == run["head_sha"], "Associated PR head differs") + matches.append((pr, target, fallback)) + if len(matches) != 1: + raise SkipPublication("No unique currently open PR matches the source head and repository") + return matches[0] + + def results(self, run: dict, target: str) -> dict: + artifacts = self.api.pages(f"{self.root}/actions/runs/{run['id']}/artifacts", "artifacts") + named = [a for a in artifacts if a["name"] == self.artifact_name and not a["expired"]] + # Artifacts from earlier attempts remain attached to the same run ID. + started = timestamp(run["run_started_at"]) + current = [a for a in named if timestamp(a["created_at"]) >= started] + if not current: + raise SkipPublication("No nonexpired result artifact belongs to this run attempt") + require(len(current) == 1, "Ambiguous result artifacts") + artifact = current[0] + require( + type(artifact["size_in_bytes"]) is int and 0 < artifact["size_in_bytes"] <= MAX_ARCHIVE_BYTES, + "Artifact size exceeds limit", + ) + identity = artifact["workflow_run"] + require( + identity["id"] == run["id"] + and identity["head_sha"] == run["head_sha"] + and identity["head_repository_id"] == run["head_repository"]["id"], + "Artifact source identity differs", + ) + files = read_archive(self.api.download(f"{self.root}/actions/artifacts/{positive_id(artifact['id'])}/zip")) + revisions = bounded_json(files["revisions.json"], limit=8192) + require( + isinstance(revisions, dict) and set(revisions) == {"target", "head", "base"}, "Invalid revisions document" + ) + for value in revisions.values(): + sha(value) + if revisions["target"] != target: + raise SkipPublication("Artifact target revision cannot be verified against trusted PR metadata") + require(revisions["head"] == run["head_sha"], "Artifact head revision differs") + merge = self.api.api(f"{self.root}/compare/{target}...{sha(run['head_sha'])}") + baseline = sha(merge["merge_base_commit"]["sha"]) + require(revisions["base"] == baseline, "Artifact baseline is not the expected merge base") + base, head = (validate(bounded_json(files[f"{role}.json"])) for role in ("base", "head")) + for role, doc, commit in (("base", base, baseline), ("head", head, run["head_sha"])): + require(doc["sdk"] == self.sdk, "Artifact SDK differs") + require(doc["revision"]["role"] == role and doc["revision"]["commit"] == commit, "Result revision differs") + require(not doc["revision"].get("dirty", False), "Dirty SDK result cannot be published") + require(doc["suite"]["source_commit"] == run["head_sha"], "Benchmark source revision differs") + require(isinstance(run.get("conclusion"), str), "Missing source run conclusion") + return compare(base, head, source_status=run["conclusion"]) + + def upsert(self, event: dict, run: dict, pr: dict, target: str, fallback: bool) -> str: + comments = self.api.pages(f"{self.root}/issues/{positive_id(pr['number'])}/comments") + matches = [ + c + for c in comments + if c["user"]["login"] == "github-actions[bot]" + and c["user"]["type"] == "Bot" + and (c.get("body") or "").splitlines()[:1] == [self.marker] + ] + require(len(matches) <= 1, "Multiple reporter bot comments") + order = (run["run_number"], run["id"], run["run_attempt"]) + if matches: + lines = matches[0]["body"].splitlines() + stamp = re.fullmatch( + r"", + lines[1] if len(lines) > 1 else "", + ) + require(stamp is not None, "Existing bot comment lacks a valid source stamp") + if tuple(int(n) for n in stamp.groups()) >= order: + raise SkipPublication("An equal or newer source run has already been published") + current_pr = self.api.api(f"{self.root}/pulls/{pr['number']}") + if not self.matches_pr(current_pr, run) or (fallback and current_pr["base"]["sha"] != target): + raise SkipPublication("PR identity changed before publication") + latest = self.verified_run(event) + require((latest["run_number"], latest["id"], latest["run_attempt"]) == order, "Source run identity changed") + body = self.marker + "\n\n" + self.report + require(len(body.encode("utf-8")) < 60_000, "Comment exceeds size limit") + if matches: + self.api.api(f"{self.root}/issues/comments/{positive_id(matches[0]['id'])}", "PATCH", {"body": body}) + return "updated" + self.api.api(f"{self.root}/issues/{pr['number']}/comments", "POST", {"body": body}) + return "created" + + def publish(self, event: dict) -> tuple[dict, str]: + try: + run = self.verified_run(event) + pr, target, fallback = self.associate(run) + self.comparison = self.results(run, target) + self.report = render(self.comparison) + status = self.upsert(event, run, pr, target, fallback) + self.comparison["publication"] = { + "status": status, + "pull_request": pr["number"], + "run_id": run["id"], + "run_attempt": run["run_attempt"], + } + return self.comparison, self.report + except SkipPublication as exc: + reason = str(exc) # All skip messages are trusted constants. + result = self.comparison or {"comparison_version": 1} + result["publication"] = {"status": "skipped", "reason": reason} + return result, (self.report or "## SDK performance\n\n") + f"\n**Publication skipped:** {reason}.\n" + except (KeyError, TypeError, IndexError, AttributeError): + raise InvalidResult("Malformed GitHub metadata") from None diff --git a/.github/actions/benchmark-report/benchmarks/report/render.py b/.github/actions/benchmark-report/benchmarks/report/render.py new file mode 100644 index 0000000..e20c44f --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/render.py @@ -0,0 +1,213 @@ +"""Render bounded Markdown from data, never artifact-provided markup or URLs.""" + +import json +import unicodedata +from pathlib import Path + +MAX_REPORT_BYTES = 58_000 # Reserve room for the publisher's trusted markers. + + +def safe_text(value, limit: int = 240) -> str: + text = str(value) + text = "".join(" " if unicodedata.category(c).startswith("C") or c.isspace() else c for c in text) + # Bound encoded bytes, including entity expansion, without splitting an entity. + pieces = [] + size = 0 + for c in text: + # Entities stop Markdown/HTML syntax; the separator prevents mentions and URL autolinking. + piece = c if c.isalnum() or c == " " else f"&#{ord(c)};" + ("​" if c in "@:." else "") + length = len(piece.encode("utf-8")) + if size + length > limit - 3: + pieces.append("…") + break + pieces.append(piece) + size += length + return "".join(pieces) + + +def json_text(value) -> str: + # Safe even when a consumer embeds the comparison in HTML; it remains ordinary JSON on disk. + text = json.dumps(value, ensure_ascii=True, allow_nan=False, indent=2) + return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + "\n" + + +def _number(value) -> str: + return "—" if value is None else f"{value:,.4g}" + + +def _row(row: dict, details=False, workload=None) -> str: + metric = row["head"] or row["base"] or {} + label = safe_text(row["metric"], 100) + if metric.get("quality") == "estimated": + label += " (estimated)" + label += " · " + safe_text(metric.get("unit", ""), 30) + change = _number(row.get("change_percent")) + if row.get("change_percent") is not None: + change += "%" + status = safe_text(row["status"]) + if row["status"] != "inconclusive": + status += ": " + safe_text("; ".join(row.get("reasons", [])), 180) + if details: + status += "; " + safe_text("; ".join(row.get("reasons", [])), 220) + if "paired_delta" in row: + delta = row["paired_delta"] + status += ( + f"; {row['pairs']} pairs, paired Δ median {_number(delta['median'])}, " + f"range {_number(delta['min'])}…{_number(delta['max'])}" + ) + for explanation in row.get("explanations", []): + status += "; " + safe_text(explanation, 220) + for role, context in (workload or {}).items(): + failures = context.get("failures", []) + for failure in failures[:3]: + reason = failure.get("reason") or failure.get("error") + if reason: + status += ( + f"; {role} {safe_text(failure['pass'], 32)} repetition {failure['repetition']}: " + + safe_text(reason, 220) + ) + if len(failures) > 3: + status += f"; {len(failures) - 3} more {role} failures in comparison.json" + status += "; method: " + safe_text(metric.get("method", "unavailable"), 160) + return ( + f"| {safe_text(row['workload'], 100)} | {label} | {_number(row.get('base_median'))} | " + f"{_number(row.get('head_median'))} | {change} · {status} |\n" + ) + + +def headline_rows(comparison: dict) -> list[dict]: + """Show representative workload families without letting unsupported rows crowd out measurements.""" + rows = comparison["rows"] + selected = [] + identities = set() + + def add(row): + identity = (row["workload"], row["metric"]) + if identity not in identities: + identities.add(identity) + selected.append(row) + + for row in rows: + if row.get("execution_failed"): + add(row) + + workloads = comparison["workloads"] + + def parameters(workload): + context = workloads[workload] + return (context.get("head") or context["base"])["parameters"] + + def pick(workload, metric): + for row in rows: + if row["workload"] == workload and row["metric"] == metric and "base_median" in row: + add(row) + break + + # These are workload/metric semantics shared by adapters, not profiler-specific behavior. + for mode, metric, all_sizes in ( + ("single", "elapsed_time_per_op", False), + ("all", "elapsed_time_per_op", True), + ("all", "allocated_bytes_per_op", True), + ("all", "observed_workload_peak_rss", False), + ("refresh", "post_settling_process_rss", False), + ("capture", "elapsed_time_per_op", False), + ("drain", "observed_workload_peak_rss", False), + ("single", "process_cpu_time_per_op", False), + ): + candidates = [w for w in workloads if parameters(w).get("mode") == mode] + # Select the boolean single case, ascending scaling sizes, and the largest payload. + # Parameter values are untrusted arbitrary JSON; sorting serialized values is total. + candidates.sort( + key=lambda w: ( + parameters(w).get("flag_index") != 0 if mode == "single" else False, + len(str(parameters(w).get("flag_count", 0))), + str(parameters(w).get("flag_count", 0)), + len(str(parameters(w).get("payload_bytes", 0))), + str(parameters(w).get("payload_bytes", 0)), + w, + ), + reverse=mode in ("capture", "drain"), + ) + for workload in candidates: + before = len(selected) + pick(workload, metric) + if not all_sizes and len(selected) > before: + break + + for row in rows: + if "base_median" in row: + add(row) + for row in rows: + add(row) + return selected[:12] + + +def render(comparison: dict) -> str: + chunks = [ + f"## SDK performance: {safe_text(comparison['sdk'])}\n\n", + "Informational medians of paired samples; all numeric changes are **inconclusive**. " + "Paired ranges describe variability, not confidence intervals. No aggregate score or performance gate.\n\n", + ] + if comparison["execution_failed"]: + chunks.append("**Benchmark execution failed. Partial samples are not evidence of improvement.**\n\n") + for role in ("base", "head"): + context = comparison["context"][role] + execution = context["execution"] + chunks.append(f"**{role.title()}**: {safe_text(context['revision']['commit'])}") + if context["revision"].get("dirty"): + chunks.append(" (dirty working tree)") + chunks.append( + f" · {safe_text(execution['runtime'])} {safe_text(execution['runtime_version'])}" + f" · {safe_text(execution['os'])}/{safe_text(execution['arch'])}" + f" · CPU: {safe_text(execution['cpu'])} · runner: {safe_text(execution['runner_class'])}\n\n" + ) + chunks.append( + "Resource limits: " + safe_text(json.dumps(execution["resource_limits"], sort_keys=True), 1000) + "\n\n" + ) + chunks.append("Full comparison, configuration, samples and checkpoints: `comparison.json` (job artifact).\n\n") + table = "| Workload | Metric · unit | Base | PR | Change · interpretation |\n| --- | --- | --- | --- | --- |\n" + chunks.append(table) + rows = comparison["rows"] + size = len("".join(chunks).encode("utf-8")) + for row in headline_rows(comparison): + line = _row(row) + if size + len(line.encode("utf-8")) > MAX_REPORT_BYTES - 1500: + break + chunks.append(line) + size += len(line.encode("utf-8")) + chunks.append("\n
\nDetailed cases and paired variability\n\n" + table) + size = len("".join(chunks).encode("utf-8")) + shown = 0 + for row in rows: + line = _row(row, details=True, workload=comparison["workloads"][row["workload"]]) + if size + len(line.encode("utf-8")) > MAX_REPORT_BYTES - 1000: + break + chunks.append(line) + size += len(line.encode("utf-8")) + shown += 1 + chunks.append("\n
\n\n") + chunks.append( + f"{len(rows)} metric rows; {shown} detailed rows shown. " + "Omitted or shortened data remain in `comparison.json`.\n" + ) + report = "".join(chunks) + assert len(report.encode("utf-8")) < MAX_REPORT_BYTES + return report + + +def write_outputs(output_directory: str, comparison: dict, report: str, environ: dict) -> tuple[Path, Path]: + directory = Path(output_directory).resolve() + directory.mkdir(parents=True, exist_ok=True) + report_path, comparison_path = directory / "report.md", directory / "comparison.json" + report_path.write_text(report, encoding="utf-8") + comparison_path.write_text(json_text(comparison), encoding="utf-8") + if environ.get("GITHUB_STEP_SUMMARY"): + with Path(environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as stream: + stream.write(report) + if environ.get("GITHUB_OUTPUT"): + # Paths are caller-controlled, but must not inject additional action outputs. + if any(c in str(directory) for c in "\r\n"): + raise ValueError("Output directory contains a newline") + with Path(environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as stream: + stream.write(f"report-path={report_path}\ncomparison-path={comparison_path}\n") + return report_path, comparison_path diff --git a/.github/actions/benchmark-report/benchmarks/report/validate.py b/.github/actions/benchmark-report/benchmarks/report/validate.py new file mode 100644 index 0000000..f12eca6 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/validate.py @@ -0,0 +1,114 @@ +"""Bounded, data-only validation of SDK benchmark result documents.""" + +import json +import math +from pathlib import Path + +from jsonschema import Draft202012Validator + +MAX_DOCUMENT_BYTES = 8 * 1024 * 1024 +SCHEMA_PATH = Path(__file__).resolve().parents[1] / "schema" / "result.schema.json" +VALIDATOR = Draft202012Validator(json.loads(SCHEMA_PATH.read_text())) + + +class InvalidResult(ValueError): + """Untrusted input did not satisfy the result contract.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise InvalidResult(message) + + +def _unique_object(pairs: list) -> dict: + result = {} + for key, value in pairs: + require(key not in result, "Duplicate JSON object key") + result[key] = value + return result + + +def bounded_json(data: bytes, limit: int = MAX_DOCUMENT_BYTES, text_limit: int = 4096): + require(len(data) <= limit, "JSON document exceeds byte limit") + try: + value = json.loads(data, object_pairs_hook=_unique_object) + check_bounds(value, text_limit=text_limit) + return value + except (UnicodeError, RecursionError, ValueError) as exc: + if isinstance(exc, InvalidResult): + raise + raise InvalidResult("Malformed JSON document") from None + + +def check_bounds(value, text_limit: int = 4096) -> None: + nodes = 0 + + def visit(item, depth=0, key=""): + nonlocal nodes + nodes += 1 + require(nodes <= 250_000 and depth <= 20, "JSON structure exceeds limits") + if isinstance(item, dict): + require(len(item) <= 1024, "Too many object fields") + for name, child in item.items(): + require(isinstance(name, str) and len(name) <= 256, "Invalid object key") + visit(child, depth + 1, name) + elif isinstance(item, list): + require(len(item) <= 2000, "Too many array items") + for child in item: + visit(child, depth + 1, key) + elif isinstance(item, str): + require(len(item) <= (65536 if key in ("stdout", "stderr") else text_limit), "Text exceeds limit") + require(not any(0xD800 <= ord(c) <= 0xDFFF for c in item), "Invalid Unicode text") + elif isinstance(item, (int, float)) and not isinstance(item, bool): + require(abs(item) <= 1e100 and math.isfinite(item), "Non-finite or excessive number") + + visit(value) + + +def _unique(items: list, identity, name: str) -> None: + keys = [identity(item) for item in items] + require(len(keys) == len(set(keys)), f"Duplicate {name} identity") + + +def validate(document: dict) -> dict: + check_bounds(document) + # Do not include validator messages: they can contain raw logs or attacker text. + error = next(VALIDATOR.iter_errors(document), None) + require(error is None, "Result does not satisfy schema v1") + _unique(document["benchmarks"], lambda b: b["id"], "workload") + repetitions = document["suite"]["repetitions"] + for workload in document["benchmarks"]: + _unique(workload["metrics"], lambda m: m["id"], "metric") + runs = workload.get("runs", []) + _unique(runs, lambda r: (r["pass"], r["repetition"]), "run") + for run in runs: + require(run["repetition"] < repetitions, "Run repetition outside configured range") + checkpoints = run.get("checkpoints", []) + positions = [c["operations"] for c in checkpoints] + require(positions == sorted(set(positions)), "Invalid checkpoint identities") + if "operations" in run: + require(all(p <= run["operations"] for p in positions), "Checkpoint exceeds completed operations") + _nonnegative_measurements(run) + for metric in workload["metrics"]: + samples = metric.get("samples", []) + _unique(samples, lambda s: s["repetition"], "sample repetition") + require(all(s["repetition"] < repetitions for s in samples), "Sample repetition outside configured range") + return document + + +def _nonnegative_measurements(value, key="") -> None: + if isinstance(value, dict): + for name, child in value.items(): + # Exit codes may legitimately be negative; correctness evidence is not a measurement. + if name not in ("code", "exit_code", "validation"): + _nonnegative_measurements(child, name) + elif isinstance(value, list): + for child in value: + _nonnegative_measurements(child, key) + elif isinstance(value, (float, int)) and not isinstance(value, bool): + require(value >= 0, "Negative raw measurement") + + +def load_result(path: str | Path) -> dict: + with Path(path).open("rb") as stream: + return validate(bounded_json(stream.read(MAX_DOCUMENT_BYTES + 1))) diff --git a/.github/actions/benchmark-report/benchmarks/schema/result.schema.json b/.github/actions/benchmark-report/benchmarks/schema/result.schema.json new file mode 100644 index 0000000..2ed1441 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/schema/result.schema.json @@ -0,0 +1,537 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SDK benchmark result v1", + "type": "object", + "properties": { + "schema_version": { + "const": 1 + }, + "sdk": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "revision": { + "type": "object", + "properties": { + "role": { + "enum": [ + "base", + "head" + ] + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "maxLength": 40 + }, + "dirty": { + "type": "boolean" + } + }, + "required": [ + "role", + "commit" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "version": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "source_commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "maxLength": 40 + }, + "source_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "profile": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "repetitions": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + }, + "required": [ + "id", + "version", + "source_commit", + "source_digest", + "profile", + "repetitions" + ], + "additionalProperties": true + }, + "execution": { + "type": "object", + "properties": { + "pair_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime_version": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime_flags": { + "type": "array", + "items": { + "type": "string", + "maxLength": 4096 + }, + "maxItems": 64, + "minItems": 0 + }, + "os": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "arch": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "kernel": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "cpu": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runner_class": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runner_image": { + "type": [ + "string", + "null" + ], + "maxLength": 256 + }, + "dependency_lock_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "build_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "resource_limits": { + "type": "object", + "minProperties": 1 + }, + "measurement_tools": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "sdk_version": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "entry": { + "type": "string", + "maxLength": 4096 + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "core_build": { + "type": "object", + "properties": { + "entry": { + "type": "string", + "maxLength": 4096 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + } + }, + "required": [ + "entry", + "digest" + ], + "additionalProperties": false + } + }, + "required": [ + "pair_id", + "runtime", + "runtime_version", + "runtime_flags", + "os", + "arch", + "kernel", + "cpu", + "runner_class", + "runner_image", + "dependency_lock_digest", + "resource_limits" + ], + "additionalProperties": true + }, + "benchmarks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "fixture_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "parameters": { + "type": "object" + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "reason": { + "type": "string", + "maxLength": 4096 + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "unit": { + "enum": [ + "ns/op", + "bytes/op", + "count/op", + "bytes", + "ns", + "count", + "ops/s" + ] + }, + "direction": { + "enum": [ + "lower", + "higher" + ] + }, + "method": { + "type": "string", + "maxLength": 4096 + }, + "quality": { + "enum": [ + "measured", + "estimated" + ] + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "reason": { + "type": "string", + "maxLength": 4096 + }, + "samples": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + } + }, + "required": [ + "repetition", + "value", + "operations" + ], + "additionalProperties": false + }, + "maxItems": 100, + "minItems": 0 + }, + "configuration": { + "type": "object" + }, + "scope": { + "type": "string", + "maxLength": 4096 + } + }, + "required": [ + "id", + "unit", + "status" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "ok" + } + } + }, + "then": { + "required": [ + "method", + "quality", + "direction", + "samples" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "samples": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + } + }, + "required": [ + "repetition", + "value", + "operations" + ], + "additionalProperties": false + }, + "maxItems": 100, + "minItems": 1 + } + } + }, + "else": { + "required": [ + "reason" + ] + } + } + ] + }, + "maxItems": 64, + "minItems": 0 + }, + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "pass": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "warmup_operations": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "entry": { + "type": "string", + "maxLength": 4096 + }, + "stdout": { + "type": "string", + "maxLength": 65536 + }, + "stderr": { + "type": "string", + "maxLength": 65536 + }, + "checkpoints": { + "type": "array", + "items": { + "type": "object", + "required": [ + "operations" + ], + "properties": { + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + } + } + }, + "maxItems": 1000, + "minItems": 0 + } + }, + "required": [ + "repetition", + "pass", + "status" + ], + "additionalProperties": true, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "ok" + } + } + }, + "then": { + "required": [ + "operations" + ] + } + } + ] + }, + "maxItems": 400, + "minItems": 0 + } + }, + "required": [ + "id", + "fixture_digest", + "parameters", + "status", + "metrics" + ], + "additionalProperties": false + }, + "maxItems": 128, + "minItems": 1 + } + }, + "required": [ + "schema_version", + "sdk", + "revision", + "suite", + "execution", + "benchmarks" + ], + "additionalProperties": false +} diff --git a/.github/actions/benchmark-report/pyproject.toml b/.github/actions/benchmark-report/pyproject.toml new file mode 100644 index 0000000..8c2866c --- /dev/null +++ b/.github/actions/benchmark-report/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "posthog-sdk-benchmark-report" +version = "0.1.0" +requires-python = "==3.12.12" +dependencies = ["jsonschema==4.26.0"] + +[dependency-groups] +dev = [ + "black==25.12.0", + "pytest==9.0.2", + "pyyaml==6.0.3", + "ruff==0.14.10", +] + +[tool.uv] +package = false + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.black] +line-length = 120 +target-version = ["py312"] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I"] diff --git a/.github/actions/benchmark-report/tests/__init__.py b/.github/actions/benchmark-report/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json new file mode 100644 index 0000000..35d6f2e --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json @@ -0,0 +1,667 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "base", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 23715.815125, + "operations": 16000, + "elapsed_ns": 379453042 + }, + { + "repetition": 1, + "value": 22059.3854375, + "operations": 16000, + "elapsed_ns": 352950167 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27478.125, + "operations": 16000, + "elapsed_ns": 379453042 + }, + { + "repetition": 1, + "value": 26465.5625, + "operations": 16000, + "elapsed_ns": 352950167 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20589.468, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20836.4235, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 379453042, + "elapsed_time_per_op": 23715.815125, + "process_cpu_time_per_op": 27478.125, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 329431488, + "sampled_allocations": 10027, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 352950167, + "elapsed_time_per_op": 22059.3854375, + "process_cpu_time_per_op": 26465.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 333382776, + "sampled_allocations": 10147, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 223641600, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 224296960, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 229195776, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 229982208, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 229195776, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 229982208, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 227065856, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 227852288, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11243712, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 11243096, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 223641600, + "heap_used": 11134400, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 223739904, + "heap_used": 11088536, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 225001472, + "heap_used": 11210032, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 226508800, + "heap_used": 11162808, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 226885632, + "heap_used": 11231512, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 226902016, + "heap_used": 11179480, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 226951168, + "heap_used": 11264696, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 226967552, + "heap_used": 11222656, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227065856, + "heap_used": 11293768, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4316754167, + "after_work": { + "rss": 227065856, + "heap_used": 11294720, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229195776, + "process_lifetime_peak_rss": 229195776, + "after_settling": { + "rss": 227065856, + "heap_used": 11243712, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 224296960, + "heap_used": 11134896, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 227065856, + "heap_used": 11088224, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 227590144, + "heap_used": 11209440, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 227672064, + "heap_used": 11162216, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 227737600, + "heap_used": 11230560, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 227737600, + "heap_used": 11178528, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 227753984, + "heap_used": 11263864, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 227753984, + "heap_used": 11221920, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227852288, + "heap_used": 11293152, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4199492916, + "after_work": { + "rss": 227852288, + "heap_used": 11294104, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229982208, + "process_lifetime_peak_rss": 229982208, + "after_settling": { + "rss": 227852288, + "heap_used": 11243096, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json new file mode 100644 index 0000000..7401057 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json @@ -0,0 +1,667 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24220.0911875, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 23317.8801875, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 28010.5625, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 27506.125, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20723.7, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20533.181, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 387521459, + "elapsed_time_per_op": 24220.0911875, + "process_cpu_time_per_op": 28010.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 331579200, + "sampled_allocations": 10092, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 373086083, + "elapsed_time_per_op": 23317.8801875, + "process_cpu_time_per_op": 27506.125, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 328530896, + "sampled_allocations": 10000, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 225476608, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 224215040, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 228753408, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 227573760, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11239984, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 11242336, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 225476608, + "heap_used": 11130488, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 228278272, + "heap_used": 11085112, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 228392960, + "heap_used": 11206328, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 228556800, + "heap_used": 11159104, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 228605952, + "heap_used": 11227448, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 228638720, + "heap_used": 11175416, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 228671488, + "heap_used": 11260752, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 228671488, + "heap_used": 11218808, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 228753408, + "heap_used": 11290040, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4196690625, + "after_work": { + "rss": 228753408, + "heap_used": 11290992, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 230883328, + "process_lifetime_peak_rss": 230883328, + "after_settling": { + "rss": 228753408, + "heap_used": 11239984, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 224215040, + "heap_used": 11133840, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 224395264, + "heap_used": 11086648, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 227295232, + "heap_used": 11207864, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 227344384, + "heap_used": 11161024, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 227393536, + "heap_used": 11229704, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 227442688, + "heap_used": 11177696, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 227491840, + "heap_used": 11263344, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 227508224, + "heap_used": 11221328, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227573760, + "heap_used": 11292392, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4194177625, + "after_work": { + "rss": 227573760, + "heap_used": 11293344, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229703680, + "process_lifetime_peak_rss": 229703680, + "after_settling": { + "rss": 227573760, + "heap_used": 11242336, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json new file mode 100644 index 0000000..1be00a4 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json @@ -0,0 +1,1677 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24220.0911875, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 23317.8801875, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 28010.5625, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 27506.125, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20723.7, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20533.181, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.rollout", + "parameters": { + "id": "flags.single.rollout", + "mode": "single", + "flag_count": 10, + "flag_index": 1, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "f0aadeaa1e94de0a830969af9d64e308ac5191873d36b6c4e805385552c31446", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17914.6458125, + "operations": 16000, + "elapsed_ns": 286634333 + }, + { + "repetition": 1, + "value": 18454.3203125, + "operations": 16000, + "elapsed_ns": 295269125 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21350.875, + "operations": 16000, + "elapsed_ns": 286634333 + }, + { + "repetition": 1, + "value": 21848.375, + "operations": 16000, + "elapsed_ns": 295269125 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17031.788, + "operations": 16000 + }, + { + "repetition": 1, + "value": 16985.316, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.variant", + "parameters": { + "id": "flags.single.variant", + "mode": "single", + "flag_count": 10, + "flag_index": 2, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "6e8ee590538f89fcb65a47601425959ce480e7730068a3460073823641892bfb", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24070.1640625, + "operations": 16000, + "elapsed_ns": 385122625 + }, + { + "repetition": 1, + "value": 23840.296875, + "operations": 16000, + "elapsed_ns": 381444750 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27900.5625, + "operations": 16000, + "elapsed_ns": 385122625 + }, + { + "repetition": 1, + "value": 27677.9375, + "operations": 16000, + "elapsed_ns": 381444750 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21340.3305, + "operations": 16000 + }, + { + "repetition": 1, + "value": 21302.417, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.property", + "parameters": { + "id": "flags.single.property", + "mode": "single", + "flag_count": 10, + "flag_index": 3, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "fd9712badeda5ddf2d637cde064d87671c10fd7d45e1234ef43a6c41bbaa0e34", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 12840.151, + "operations": 16000, + "elapsed_ns": 205442416 + }, + { + "repetition": 1, + "value": 12862.2864375, + "operations": 16000, + "elapsed_ns": 205796583 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 16501.6875, + "operations": 16000, + "elapsed_ns": 205442416 + }, + { + "repetition": 1, + "value": 16292.875, + "operations": 16000, + "elapsed_ns": 205796583 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14278.814, + "operations": 16000 + }, + { + "repetition": 1, + "value": 14393.112, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.regex", + "parameters": { + "id": "flags.single.regex", + "mode": "single", + "flag_count": 10, + "flag_index": 4, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "14dcea4d4a3b68838929a114a402cc914ce5b4e157789cf6e798dce598918471", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 22357.6015625, + "operations": 16000, + "elapsed_ns": 357721625 + }, + { + "repetition": 1, + "value": 21291.4609375, + "operations": 16000, + "elapsed_ns": 340663375 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27434.5625, + "operations": 16000, + "elapsed_ns": 357721625 + }, + { + "repetition": 1, + "value": 26517.875, + "operations": 16000, + "elapsed_ns": 340663375 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21714.4585, + "operations": 16000 + }, + { + "repetition": 1, + "value": 21738.954, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.cohort", + "parameters": { + "id": "flags.single.cohort", + "mode": "single", + "flag_count": 10, + "flag_index": 5, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "4b1771937bc181170041d2a503f5940e689e0a7c111e97422c1a540b49c54313", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 13332.7291875, + "operations": 16000, + "elapsed_ns": 213323667 + }, + { + "repetition": 1, + "value": 12793.724, + "operations": 16000, + "elapsed_ns": 204699584 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17212.25, + "operations": 16000, + "elapsed_ns": 213323667 + }, + { + "repetition": 1, + "value": 16807.5, + "operations": 16000, + "elapsed_ns": 204699584 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 15794.813, + "operations": 16000 + }, + { + "repetition": 1, + "value": 15655.0615, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.early", + "parameters": { + "id": "flags.single.early", + "mode": "single", + "flag_count": 10, + "flag_index": 6, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "6c6b135eb967c9ec62350f5d675329c3972ff12e1c83fb426d9f28a2e7b027b7", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 13651.08075, + "operations": 16000, + "elapsed_ns": 218417292 + }, + { + "repetition": 1, + "value": 13630.9115, + "operations": 16000, + "elapsed_ns": 218094584 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17075.6875, + "operations": 16000, + "elapsed_ns": 218417292 + }, + { + "repetition": 1, + "value": 17093.75, + "operations": 16000, + "elapsed_ns": 218094584 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 19417.508, + "operations": 16000 + }, + { + "repetition": 1, + "value": 19632.0915, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.late", + "parameters": { + "id": "flags.single.late", + "mode": "single", + "flag_count": 10, + "flag_index": 7, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "11fd86f58bda1ae9b3c66a724ea5f877e3cea7c50731df961194b755f00cf981", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14455.5703125, + "operations": 16000, + "elapsed_ns": 231289125 + }, + { + "repetition": 1, + "value": 14381.328125, + "operations": 16000, + "elapsed_ns": 230101250 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17931.625, + "operations": 16000, + "elapsed_ns": 231289125 + }, + { + "repetition": 1, + "value": 17839.5, + "operations": 16000, + "elapsed_ns": 230101250 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24590.9875, + "operations": 16000 + }, + { + "repetition": 1, + "value": 24576.676, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.no-match", + "parameters": { + "id": "flags.single.no-match", + "mode": "single", + "flag_count": 10, + "flag_index": 8, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "eadae028c525aa88c0b0488eab6ca2cadc67969c9b7e5e74e668d7d862b6d402", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1912.862, + "operations": 16000, + "elapsed_ns": 30605792 + }, + { + "repetition": 1, + "value": 1930.453125, + "operations": 16000, + "elapsed_ns": 30887250 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3635.75, + "operations": 16000, + "elapsed_ns": 30605792 + }, + { + "repetition": 1, + "value": 3659.4375, + "operations": 16000, + "elapsed_ns": 30887250 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17220.415, + "operations": 16000 + }, + { + "repetition": 1, + "value": 17570.8535, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.10", + "parameters": { + "id": "flags.all.10", + "mode": "all", + "flag_count": 10, + "operations": 1600, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "20165e70c1fd37a263143ada015dd08026d90494faad376c254a79d7b46667cb", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 82473.77625, + "operations": 1600, + "elapsed_ns": 131958042 + }, + { + "repetition": 1, + "value": 81382.318125, + "operations": 1600, + "elapsed_ns": 130211709 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 167243.75, + "operations": 1600, + "elapsed_ns": 131958042 + }, + { + "repetition": 1, + "value": 162017.5, + "operations": 1600, + "elapsed_ns": 130211709 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 151583.105, + "operations": 1600 + }, + { + "repetition": 1, + "value": 155848.125, + "operations": 1600 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.100", + "parameters": { + "id": "flags.all.100", + "mode": "all", + "flag_count": 100, + "operations": 160, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "133de92e110e6ee3c2bde38226242aac4302fff953ad423dbeecfc92abc164c7", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 655395.56875, + "operations": 160, + "elapsed_ns": 104863291 + }, + { + "repetition": 1, + "value": 677104.94375, + "operations": 160, + "elapsed_ns": 108336791 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1295731.25, + "operations": 160, + "elapsed_ns": 104863291 + }, + { + "repetition": 1, + "value": 1341325, + "operations": 160, + "elapsed_ns": 108336791 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1474595.55, + "operations": 160 + }, + { + "repetition": 1, + "value": 1461157.9, + "operations": 160 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.1000", + "parameters": { + "id": "flags.all.1000", + "mode": "all", + "flag_count": 1000, + "operations": 160, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "cfaf7acdd2056c29e867677d5280fd52eb28f93459053e1e62dc839103e669b3", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6647329.425, + "operations": 160, + "elapsed_ns": 1063572708 + }, + { + "repetition": 1, + "value": 6680127.6, + "operations": 160, + "elapsed_ns": 1068820416 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11793781.25, + "operations": 160, + "elapsed_ns": 1063572708 + }, + { + "repetition": 1, + "value": 11838550, + "operations": 160, + "elapsed_ns": 1068820416 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14686163.4, + "operations": 160 + }, + { + "repetition": 1, + "value": 14629267.4, + "operations": 160 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.load.1000", + "parameters": { + "id": "flags.load.1000", + "mode": "all", + "flag_count": 1000, + "operations": 1600, + "concurrency": 8, + "passes": [ + "memory" + ] + }, + "fixture_digest": "9c35736c1c79903e10fcc2288b1ccfc3016590a22029795a65a32888b94a6075", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 354271232, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 353189888, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386007040, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380108800, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386023424, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380125184, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386072576, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380174336, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 9127592, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 9116720, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 225476608, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 224215040, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 228753408, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 227573760, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11239984, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 11242336, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + } + ] + }, + { + "id": "capture.enqueue.32", + "parameters": { + "id": "capture.enqueue.32", + "mode": "capture", + "payload_bytes": 32, + "operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "65961f598e5917166a73bc754237304803c5f76536ed23cfe61f6c441ecfaf39", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4105.4375, + "operations": 2000, + "elapsed_ns": 8210875 + }, + { + "repetition": 1, + "value": 4073.3335, + "operations": 2000, + "elapsed_ns": 8146667 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6975, + "operations": 2000, + "elapsed_ns": 8210875 + }, + { + "repetition": 1, + "value": 6932, + "operations": 2000, + "elapsed_ns": 8146667 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 8906.228, + "operations": 2000 + }, + { + "repetition": 1, + "value": 8420.576, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "capture.enqueue.4096", + "parameters": { + "id": "capture.enqueue.4096", + "mode": "capture", + "payload_bytes": 4096, + "operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "06edc0baaf6e302e2cc37659a605f64eb24282f5ca8c87ef2915ae907e376826", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4064.458, + "operations": 2000, + "elapsed_ns": 8128916 + }, + { + "repetition": 1, + "value": 4113.729, + "operations": 2000, + "elapsed_ns": 8227458 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6951.5, + "operations": 2000, + "elapsed_ns": 8128916 + }, + { + "repetition": 1, + "value": 6998, + "operations": 2000, + "elapsed_ns": 8227458 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 8568.984, + "operations": 2000 + }, + { + "repetition": 1, + "value": 8131.808, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "capture.drain.4096", + "parameters": { + "id": "capture.drain.4096", + "mode": "drain", + "payload_bytes": 4096, + "operations": 10000, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "ba591d6aa9238b5f2776ac74287bd19a4f07d0e81bfb1c02af29c60e20c43f0e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 74399744, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 74235904, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 89817088, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 89636864, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 89817088, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 89636864, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 86720512, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 86573056, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 9604016, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 9603952, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json new file mode 100644 index 0000000..64bf0bd --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "sdk": "posthog-python", + "revision": { + "role": "base", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "dirty": false + }, + "suite": { + "id": "python-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "77449378b7ba55b91d978a5383ee32f8bceaab69e83e7d262f4669f4e42ed317", + "profile": "full", + "repetitions": 2, + "allocation_tracking": { + "trace_python_allocators": true, + "file_format": "ALL_ALLOCATIONS", + "scope": "PYMALLOC_MALLOC/CALLOC/REALLOC requests; all worker threads" + }, + "transport": "loopback HTTP receiver in coordinator process; uncompressed v0", + "settling": "two gc.collect calls separated by sleep(0)", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup", + "capture": { + "flush_at": 100000, + "flush_interval_seconds": 3600, + "consumer_threads": 1, + "drain_every": 100 + }, + "flag_poll_interval_seconds": 3600 + }, + "execution": { + "runtime": "cpython", + "runtime_version": "3.12.12", + "implementation": "3.12.12 (main, Oct 31 2025, 23:21:56) [Clang 21.1.4 ]", + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "arm", + "entry": "/synthetic/sdk/entry", + "build_digest": "730edbd69df5111e43ada6fc395e2eb329a2a3a5c86a4ac2731d0d056ea8c718", + "packages": { + "pygments": "2.21.0", + "rich": "15.0.0", + "requests": "2.33.0", + "charset-normalizer": "3.4.2", + "posthog": "7.76.0", + "mdit-py-plugins": "0.6.1", + "urllib3": "2.8.0", + "mdurl": "0.1.2", + "textual": "8.2.8", + "idna": "3.19", + "linkify-it-py": "2.2.0", + "jinja2": "3.1.6", + "markdown-it-py": "4.2.0", + "distro": "1.9.0", + "certifi": "2026.10.19", + "typing-extensions": "4.16.0", + "markupsafe": "3.0.4", + "memray": "1.20.0", + "backoff": "2.2.1", + "psutil": "7.2.2", + "platformdirs": "4.13.5" + }, + "sdk_version": "7.76.0", + "measurement_tools": { + "memray": "1.20.0", + "psutil": "7.2.2" + }, + "pair_id": "example-pair", + "runtime_flags": [ + "-B" + ], + "python_hash_seed": 0, + "bytecode_policy": "uv precompiles both environments; workers do not write bytecode", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "f9c1329b3eb2ef295d5913a328d8e67c92bcb19a6142a2a918ac9b61b8744a6a", + "resource_limits": { + "process_rss_limit_bytes": 536870912, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "allocation_operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "a18a5143bd9edebbe2586a84ab72d2041d90c17b4eeb55423718f68fa4b01131", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "time.perf_counter_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3914.2265625, + "operations": 16000, + "elapsed_ns": 62627625 + }, + { + "repetition": 1, + "value": 3942.0260625, + "operations": 16000, + "elapsed_ns": 63072417 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "time.process_time_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3879.9375, + "operations": 16000, + "elapsed_ns": 62627625 + }, + { + "repetition": 1, + "value": 3937.875, + "operations": 16000, + "elapsed_ns": 63072417 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14805.996, + "operations": 2000 + }, + { + "repetition": 1, + "value": 14805.996, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 160.8935, + "operations": 2000 + }, + { + "repetition": 1, + "value": 160.8935, + "operations": 2000 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 62627625, + "elapsed_time_per_op": 3914.2265625, + "process_cpu_time_per_op": 3879.9375, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 63072417, + "elapsed_time_per_op": 3942.0260625, + "process_cpu_time_per_op": 3937.875, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6c8bab719abd9b61a1073abe748dbc91fd03ff6feadbc6bc59113224046fec39", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 49659904, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 49364992, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51200000, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50364416, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "resource.getrusage.ru_maxrss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51216384, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50380800, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51200000, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50364416, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "status": "unsupported", + "reason": "CPython has no uninstrumented total managed-heap byte counter; tracing would distort this RSS pass." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49659904 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49692672 + }, + { + "operations": 10, + "rss": 51150848 + }, + { + "operations": 15, + "rss": 51167232 + }, + { + "operations": 20, + "rss": 51167232 + }, + { + "operations": 25, + "rss": 51167232 + }, + { + "operations": 30, + "rss": 51200000 + }, + { + "operations": 35, + "rss": 51200000 + }, + { + "operations": 40, + "rss": 51200000 + } + ], + "elapsed_ns": 3435970917, + "after_work": { + "rss": 51200000 + }, + "observed_peak_rss": 51200000, + "process_lifetime_peak_rss": 51216384, + "after_settling": { + "rss": 51200000 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49364992 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49397760 + }, + { + "operations": 10, + "rss": 49430528 + }, + { + "operations": 15, + "rss": 49807360 + }, + { + "operations": 20, + "rss": 50298880 + }, + { + "operations": 25, + "rss": 50331648 + }, + { + "operations": 30, + "rss": 50364416 + }, + { + "operations": 35, + "rss": 50364416 + }, + { + "operations": 40, + "rss": 50364416 + } + ], + "elapsed_ns": 2891613125, + "after_work": { + "rss": 50364416 + }, + "observed_peak_rss": 50364416, + "process_lifetime_peak_rss": 50380800, + "after_settling": { + "rss": 50364416 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json new file mode 100644 index 0000000..6bb12c7 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "sdk": "posthog-python", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "python-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "77449378b7ba55b91d978a5383ee32f8bceaab69e83e7d262f4669f4e42ed317", + "profile": "full", + "repetitions": 2, + "allocation_tracking": { + "trace_python_allocators": true, + "file_format": "ALL_ALLOCATIONS", + "scope": "PYMALLOC_MALLOC/CALLOC/REALLOC requests; all worker threads" + }, + "transport": "loopback HTTP receiver in coordinator process; uncompressed v0", + "settling": "two gc.collect calls separated by sleep(0)", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup", + "capture": { + "flush_at": 100000, + "flush_interval_seconds": 3600, + "consumer_threads": 1, + "drain_every": 100 + }, + "flag_poll_interval_seconds": 3600 + }, + "execution": { + "runtime": "cpython", + "runtime_version": "3.12.12", + "implementation": "3.12.12 (main, Oct 31 2025, 23:21:56) [Clang 21.1.4 ]", + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "arm", + "entry": "/synthetic/sdk/entry", + "build_digest": "730edbd69df5111e43ada6fc395e2eb329a2a3a5c86a4ac2731d0d056ea8c718", + "packages": { + "pygments": "2.21.0", + "rich": "15.0.0", + "requests": "2.33.0", + "charset-normalizer": "3.4.2", + "posthog": "7.76.0", + "mdit-py-plugins": "0.6.1", + "urllib3": "2.8.0", + "mdurl": "0.1.2", + "textual": "8.2.8", + "idna": "3.19", + "linkify-it-py": "2.2.0", + "jinja2": "3.1.6", + "markdown-it-py": "4.2.0", + "distro": "1.9.0", + "certifi": "2026.10.19", + "typing-extensions": "4.16.0", + "markupsafe": "3.0.4", + "memray": "1.20.0", + "backoff": "2.2.1", + "psutil": "7.2.2", + "platformdirs": "4.13.5" + }, + "sdk_version": "7.76.0", + "measurement_tools": { + "memray": "1.20.0", + "psutil": "7.2.2" + }, + "pair_id": "example-pair", + "runtime_flags": [ + "-B" + ], + "python_hash_seed": 0, + "bytecode_policy": "uv precompiles both environments; workers do not write bytecode", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "f9c1329b3eb2ef295d5913a328d8e67c92bcb19a6142a2a918ac9b61b8744a6a", + "resource_limits": { + "process_rss_limit_bytes": 536870912, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "allocation_operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "a18a5143bd9edebbe2586a84ab72d2041d90c17b4eeb55423718f68fa4b01131", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "time.perf_counter_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4009.78125, + "operations": 16000, + "elapsed_ns": 64156500 + }, + { + "repetition": 1, + "value": 3949.6745, + "operations": 16000, + "elapsed_ns": 63194792 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "time.process_time_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4004.0625, + "operations": 16000, + "elapsed_ns": 64156500 + }, + { + "repetition": 1, + "value": 3946.5625, + "operations": 16000, + "elapsed_ns": 63194792 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14805.996, + "operations": 2000 + }, + { + "repetition": 1, + "value": 14805.996, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 160.8935, + "operations": 2000 + }, + { + "repetition": 1, + "value": 160.8935, + "operations": 2000 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 64156500, + "elapsed_time_per_op": 4009.78125, + "process_cpu_time_per_op": 4004.0625, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 63194792, + "elapsed_time_per_op": 3949.6745, + "process_cpu_time_per_op": 3946.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6c8bab719abd9b61a1073abe748dbc91fd03ff6feadbc6bc59113224046fec39", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 49070080, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 49184768, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50167808, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50069504, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "resource.getrusage.ru_maxrss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50184192, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50085888, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50167808, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50069504, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "status": "unsupported", + "reason": "CPython has no uninstrumented total managed-heap byte counter; tracing would distort this RSS pass." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49070080 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49610752 + }, + { + "operations": 10, + "rss": 49627136 + }, + { + "operations": 15, + "rss": 50118656 + }, + { + "operations": 20, + "rss": 50118656 + }, + { + "operations": 25, + "rss": 50135040 + }, + { + "operations": 30, + "rss": 50167808 + }, + { + "operations": 35, + "rss": 50167808 + }, + { + "operations": 40, + "rss": 50167808 + } + ], + "elapsed_ns": 2886622709, + "after_work": { + "rss": 50167808 + }, + "observed_peak_rss": 50167808, + "process_lifetime_peak_rss": 50184192, + "after_settling": { + "rss": 50167808 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49184768 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49217536 + }, + { + "operations": 10, + "rss": 49217536 + }, + { + "operations": 15, + "rss": 49217536 + }, + { + "operations": 20, + "rss": 49217536 + }, + { + "operations": 25, + "rss": 50036736 + }, + { + "operations": 30, + "rss": 50069504 + }, + { + "operations": 35, + "rss": 50069504 + }, + { + "operations": 40, + "rss": 50069504 + } + ], + "elapsed_ns": 2931467333, + "after_work": { + "rss": 50069504 + }, + "observed_peak_rss": 50069504, + "process_lifetime_peak_rss": 50085888, + "after_settling": { + "rss": 50069504 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/test_benchmark_github.py b/.github/actions/benchmark-report/tests/test_benchmark_github.py new file mode 100644 index 0000000..c7d8982 --- /dev/null +++ b/.github/actions/benchmark-report/tests/test_benchmark_github.py @@ -0,0 +1,215 @@ +"""Bounded HTTP and ZIP handling without network access.""" + +import io +import json +import stat +import urllib.error +import zipfile +from http.client import BadStatusLine, IncompleteRead + +import pytest + +from benchmarks.report import cli +from benchmarks.report.github import MAX_ARCHIVE_BYTES, GitHub, NoRedirect, read_archive +from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult + + +def archive(entries=None, compression=zipfile.ZIP_STORED): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w", compression=compression) as z: + for name, data in entries or [("base.json", b"{}"), ("head.json", b"{}"), ("revisions.json", b"{}")]: + z.writestr(name, data) + return stream.getvalue() + + +def test_archive_reads_exact_files_without_extraction(): + assert read_archive(archive()) == {"base.json": b"{}", "head.json": b"{}", "revisions.json": b"{}"} + + +@pytest.mark.parametrize( + "names", + [ + ["../base.json", "head.json", "revisions.json"], + ["/base.json", "head.json", "revisions.json"], + ["base.json", "base.json", "revisions.json"], + ["base.json", "head.json", "folder/"], + ["base.json", "head.json", "revisions.json", "extra"], + ["base.json", "head.json"], + ["base\\.json", "head.json", "revisions.json"], + ], +) +def test_archive_rejects_unexpected_paths_and_duplicates(names): + with pytest.raises(InvalidResult): + read_archive(archive([(n, b"{}") for n in names])) + + +@pytest.mark.parametrize("mode", [stat.S_IFLNK, stat.S_IFDIR, stat.S_IFIFO]) +def test_archive_rejects_non_regular_files(mode): + info = zipfile.ZipInfo("base.json") + info.create_system = 3 + info.external_attr = (mode | 0o644) << 16 + with pytest.raises(InvalidResult): + read_archive(archive([(info, b"{}"), ("head.json", b"{}"), ("revisions.json", b"{}")])) + + +@pytest.mark.parametrize("data", [b"not zip", b"x" * (MAX_ARCHIVE_BYTES + 1), archive()[:-30]]) +def test_archive_rejects_invalid_or_oversized_compressed_input(data): + with pytest.raises(InvalidResult): + read_archive(data) + + +def test_archive_rejects_zip_bombs_and_large_members(): + for size, compression in [(100_000, zipfile.ZIP_DEFLATED), (MAX_DOCUMENT_BYTES + 1, zipfile.ZIP_STORED)]: + with pytest.raises(InvalidResult): + read_archive( + archive([("base.json", b"x" * size), ("head.json", b"{}"), ("revisions.json", b"{}")], compression) + ) + with pytest.raises(InvalidResult): + read_archive(archive([("base.json", b"{}"), ("head.json", b"{}"), ("revisions.json", b"x" * 8193)])) + + +class Response(io.BytesIO): + def __init__(self, code=200, body=b"{}", location=None): + super().__init__(body) + self.code = code + self.headers = {"Location": location} + + +class Opener: + def __init__(self, responses): + self.responses = iter(responses) + self.calls = [] + + def open(self, request, timeout): + self.calls.append((request, timeout)) + response = next(self.responses) + if isinstance(response, Exception): + raise response + return response + + +def client(responses): + api = GitHub("synthetic-token") + api.opener = Opener(responses) + return api + + +def test_signed_download_never_receives_authorization(): + api = client([Response(302, location="https://signed.example/file?signature=example"), Response(body=b"archive")]) + assert api.download("/repos/PostHog/sdk/actions/artifacts/1/zip") == b"archive" + first, second = [req for req, _ in api.opener.calls] + assert first.get_header("Authorization") == "Bearer synthetic-token" + assert second.get_header("Authorization") is None + assert all(0 < timeout <= 20 for _, timeout in api.opener.calls) + assert NoRedirect().redirect_request(None, None, 302, "", {}, "https://evil.example") is None + + +@pytest.mark.parametrize( + "url", + ["http://signed.example/file", "file:///tmp/file", "https://user:pass@host/file", "https://host/file#fragment"], +) +def test_download_rejects_unsafe_signed_urls(url): + api = client([Response(302, location=url)]) + with pytest.raises(InvalidResult): + api.download("/repos/PostHog/sdk/actions/artifacts/1/zip") + assert len(api.opener.calls) == 1 + + +def test_credentials_are_bound_to_fixed_api_origin(): + api = client([]) + for url in ("https://evil.example", "https://api.github.com.evil.example", "https://api.github.com:444"): + with pytest.raises(InvalidResult): + api._request(url, authenticated=True) + assert not api.opener.calls + + +def test_redirect_and_response_limits(): + api = client( + [Response(302, location="https://signed.example/first")] + [Response(302, location="/next") for _ in range(3)] + ) + with pytest.raises(InvalidResult, match="redirect limit"): + api.download("/repos/PostHog/sdk/actions/artifacts/1/zip") + api = client([Response(body=b"x" * 11)]) + with pytest.raises(InvalidResult, match="exceeds limit"): + api._request("https://api.github.com/repos/P/r", authenticated=True, limit=10) + + +@pytest.mark.parametrize( + "response", + [ + Response(403, body=b"SECRET"), + OSError("SECRET"), + urllib.error.HTTPError("url", 500, "SECRET", {}, io.BytesIO(b"SECRET")), + ], +) +def test_api_errors_do_not_disclose_response_or_token(response): + api = client([response]) + with pytest.raises(InvalidResult) as error: + api.api("/repos/PostHog/sdk") + assert "SECRET" not in str(error.value) and "synthetic-token" not in str(error.value) + + +def test_compare_api_accepts_large_patches_without_relaxing_other_endpoints(): + payload = json.dumps( + {"merge_base_commit": {"sha": "a" * 40}, "files": [{"patch": "+changed line\\n" * 9000}]} + ).encode() + path = "/repos/PostHog/sdk/compare/" + "a" * 40 + "..." + "b" * 40 + assert client([Response(body=payload)]).api(path)["merge_base_commit"]["sha"] == "a" * 40 + with pytest.raises(InvalidResult, match="Text exceeds limit"): + client([Response(body=payload)]).api("/repos/PostHog/sdk/pulls/1") + with pytest.raises(InvalidResult, match="response exceeds limit"): + client([Response(body=b"x" * (MAX_DOCUMENT_BYTES + 1))]).api(path) + + +@pytest.mark.parametrize("stage", ["open", "read"]) +def test_protocol_errors_write_safe_cli_failure_artifacts(stage, tmp_path, monkeypatch): + class TruncatedResponse(Response): + def read(self, limit): + raise IncompleteRead(b"PRIVATE_SERVER_DATA") + + api = client([BadStatusLine("PRIVATE_SERVER_DATA") if stage == "open" else TruncatedResponse()]) + monkeypatch.setattr(cli, "GitHub", lambda token: api) + event = tmp_path / "event.json" + event.write_text( + json.dumps({"action": "completed", "repository": {"full_name": "PostHog/sdk"}, "workflow_run": {"id": 1}}) + ) + out = tmp_path / "out" + summary = tmp_path / "summary.md" + args = [ + "--mode", + "publish", + "--workflow-path", + ".github/workflows/benchmarks.yml", + "--artifact-name", + "benchmark-results", + "--report-key", + "node", + "--sdk", + "posthog-node", + "--output-directory", + str(out), + ] + env = { + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_EVENT_PATH": str(event), + "GITHUB_REPOSITORY": "PostHog/sdk", + "GH_TOKEN": "synthetic-token", + "GITHUB_STEP_SUMMARY": str(summary), + } + assert cli.main(args, env) == 1 + for path in (out / "report.md", out / "comparison.json", summary): + assert path.exists() + assert "PRIVATE_SERVER_DATA" not in path.read_text() + assert "synthetic-token" not in path.read_text() + assert "Reporting failed" in summary.read_text() + + +def test_request_and_pagination_budgets(): + api = client([]) + api.requests = 100 + with pytest.raises(InvalidResult, match="budget"): + api.api("/repos/PostHog/sdk") + api = client([Response(body=b"[" + b",".join([b"{}"] * 100) + b"]") for _ in range(10)]) + with pytest.raises(InvalidResult, match="pagination"): + api.pages("/repos/PostHog/sdk/pulls") + assert len(api.opener.calls) == 10 diff --git a/.github/actions/benchmark-report/tests/test_benchmark_publish.py b/.github/actions/benchmark-report/tests/test_benchmark_publish.py new file mode 100644 index 0000000..bd01721 --- /dev/null +++ b/.github/actions/benchmark-report/tests/test_benchmark_publish.py @@ -0,0 +1,377 @@ +"""Trusted publisher identity and stale-result checks; all GitHub calls are mocked.""" + +import copy +import html +import json + +import pytest + +from benchmarks.report import cli +from benchmarks.report.publish import Publisher +from benchmarks.report.validate import InvalidResult +from tests.test_benchmark_github import archive +from tests.test_benchmark_report import pair + +BASE, HEAD, TARGET = "a" * 40, "b" * 40, "c" * 40 +REPO = {"id": 1, "full_name": "PostHog/sdk"} +FORK = {"id": 2, "full_name": "contributor/sdk"} +ROOT = "/repos/PostHog/sdk" +MARKER = "" + + +class FakeGitHub: + def __init__(self): + self.run = { + "id": 10, + "run_number": 20, + "run_attempt": 1, + "head_sha": HEAD, + "repository": REPO, + "head_repository": REPO, + "event": "pull_request", + "status": "completed", + "conclusion": "success", + "workflow_id": 3, + "run_started_at": "2026-10-27T10:00:00Z", + "pull_requests": [{"number": 7, "head": {"sha": HEAD}, "base": {"sha": TARGET}}], + } + self.event = {"action": "completed", "repository": REPO, "workflow_run": copy.deepcopy(self.run)} + self.pr = { + "number": 7, + "state": "open", + "head": {"sha": HEAD, "repo": REPO}, + "base": {"sha": TARGET, "repo": REPO}, + } + self.workflow = {"id": 3, "path": ".github/workflows/benchmarks.yml"} + self.artifacts = [ + { + "id": 40, + "name": "benchmark-results", + "expired": False, + "size_in_bytes": 1000, + "created_at": "2026-10-27T10:01:00Z", + "workflow_run": {"id": 10, "head_sha": HEAD, "head_repository_id": 1}, + } + ] + self.candidates = [{"number": 7}] + self.comments = [] + self.documents = pair() + self.revisions = {"target": TARGET, "head": HEAD, "base": BASE} + self.calls = [] + self.writes = [] + self.run_reads = 0 + self.pr_reads = 0 + self.before_final_pr = None + self.before_final_run = None + self.write_error = False + + def api(self, path, method="GET", body=None): + self.calls.append((path, method)) + if method != "GET": + if self.write_error: + raise InvalidResult("GitHub API request failed (HTTP 403)") + self.writes.append((path, method, body)) + return {"id": 99} + if path == ROOT + "/actions/runs/10": + self.run_reads += 1 + if self.run_reads > 1 and self.before_final_run: + self.before_final_run() + return copy.deepcopy(self.run) + if path == ROOT + "/actions/workflows/3": + return copy.deepcopy(self.workflow) + if path.startswith(ROOT + "/pulls/"): + self.pr_reads += 1 + if self.pr_reads > 1 and self.before_final_pr: + self.before_final_pr() + pr = copy.deepcopy(self.pr) + pr["number"] = int(path.rsplit("/", 1)[-1]) + return pr + if path == ROOT + f"/compare/{TARGET}...{HEAD}": + return {"merge_base_commit": {"sha": BASE}} + raise AssertionError(f"Unexpected API request: {path}") + + def pages(self, path, key=None): + self.calls.append((path, "GET")) + if path == ROOT + "/actions/runs/10/artifacts": + assert key == "artifacts" + return copy.deepcopy(self.artifacts) + if path == ROOT + f"/commits/{HEAD}/pulls": + return copy.deepcopy(self.candidates) + if path == ROOT + "/issues/7/comments": + return copy.deepcopy(self.comments) + raise AssertionError(path) + + def download(self, path): + assert path == ROOT + "/actions/artifacts/40/zip" + self.calls.append((path, "DOWNLOAD")) + return archive( + [(f"{role}.json", json.dumps(doc).encode()) for role, doc in zip(("base", "head"), self.documents)] + + [("revisions.json", json.dumps(self.revisions).encode())] + ) + + +def publisher(api): + return Publisher( + api, "PostHog/sdk", ".github/workflows/benchmarks.yml", "benchmark-results", "node-performance", "posthog-node" + ) + + +def bot_comment(number=19, run_id=9, attempt=1, login="github-actions[bot]", kind="Bot", marker=MARKER): + return { + "id": 50, + "user": {"login": login, "type": kind}, + "body": f"{marker}\n\nold report", + } + + +def test_same_repository_creates_bot_comment_with_trusted_marker(): + api = FakeGitHub() + result, report = publisher(api).publish(api.event) + assert result["publication"]["status"] == "created" + path, method, body = api.writes[0] + assert path == ROOT + "/issues/7/comments" and method == "POST" + assert body["body"].startswith(MARKER + "\n\n") + assert HEAD in report and BASE in report + assert api.run_reads == 2 and api.pr_reads == 2 + + +def test_existing_bot_comment_is_updated(): + api = FakeGitHub() + api.comments = [bot_comment()] + assert publisher(api).publish(api.event)[0]["publication"]["status"] == "updated" + assert api.writes[0][:2] == (ROOT + "/issues/comments/50", "PATCH") + + +@pytest.mark.parametrize( + "change", + [ + {"login": "human"}, + {"kind": "User"}, + {"marker": ""}, + {"marker": "prefix " + MARKER}, + {"login": "different-bot[bot]"}, + ], +) +def test_only_exact_marker_expected_bot_comments_are_updated(change): + api = FakeGitHub() + api.comments = [bot_comment(**change)] + assert publisher(api).publish(api.event)[0]["publication"]["status"] == "created" + assert api.writes[0][1] == "POST" + + +def test_fork_empty_associations_uses_exact_commit_and_repo(): + api = FakeGitHub() + api.run.update(pull_requests=[], head_repository=FORK) + api.pr["head"]["repo"] = FORK + api.artifacts[0]["workflow_run"]["head_repository_id"] = 2 + result, _ = publisher(api).publish(api.event) + assert result["publication"]["status"] == "created" + assert (ROOT + f"/commits/{HEAD}/pulls", "GET") in api.calls + + +@pytest.mark.parametrize( + "mutation", + [ + lambda a: a.run.update(pull_requests=[]), + lambda a: a.pr.update(state="closed"), + lambda a: a.pr["head"].update(sha="d" * 40), + lambda a: a.pr["head"].update(repo=FORK), + lambda a: a.pr["base"].update(repo=FORK), + lambda a: a.pr["head"].update(repo=None), + ], +) +def test_no_verified_current_pr_explicitly_skips(mutation): + api = FakeGitHub() + api.candidates = [] + mutation(api) + result, report = publisher(api).publish(api.event) + assert result["publication"]["status"] == "skipped" + assert "Publication skipped" in report + assert not api.writes + assert not any(method == "DOWNLOAD" for _, method in api.calls) + + +def test_ambiguous_fork_association_skips(): + api = FakeGitHub() + api.run["pull_requests"] = [] + api.candidates = [{"number": 7}, {"number": 8}] + assert publisher(api).publish(api.event)[0]["publication"]["status"] == "skipped" + assert not api.writes + + +@pytest.mark.parametrize( + "mutation", + [ + lambda a: a.event.update(action="requested"), + lambda a: a.event.update(repository=FORK), + lambda a: a.run.update(repository=FORK), + lambda a: a.run.update(event="push"), + lambda a: a.run.update(status="in_progress"), + lambda a: a.run.update(head_sha="d" * 40), + lambda a: a.run.update(id=11), + lambda a: a.workflow.update(path=".github/workflows/other.yml"), + lambda a: a.workflow.update(id=4), + lambda a: a.run["pull_requests"][0]["head"].update(sha="d" * 40), + lambda a: a.artifacts[0]["workflow_run"].update(head_sha="d" * 40), + lambda a: a.artifacts[0]["workflow_run"].update(id=11), + lambda a: a.artifacts[0]["workflow_run"].update(head_repository_id=2), + lambda a: a.revisions.update(head="d" * 40), + lambda a: a.revisions.update(base="d" * 40), + lambda a: a.documents[0]["revision"].update(commit="d" * 40), + lambda a: a.documents[1]["revision"].update(commit="d" * 40), + lambda a: a.documents[1]["revision"].update(dirty=True), + lambda a: a.documents[0]["suite"].update(source_commit=BASE), + lambda a: a.documents[1].update(sdk="other-sdk"), + lambda a: a.revisions.update(pull_request=123), + lambda a: a.artifacts.append(copy.deepcopy(a.artifacts[0])), + lambda a: a.artifacts[0].update(size_in_bytes=30 * 1024 * 1024), + ], +) +def test_mismatched_or_malformed_identities_fail_closed(mutation): + api = FakeGitHub() + mutation(api) + with pytest.raises(InvalidResult): + publisher(api).publish(api.event) + assert not api.writes + + +@pytest.mark.parametrize( + "mutation", + [ + lambda a: a.artifacts.clear(), + lambda a: a.artifacts[0].update(expired=True), + lambda a: a.artifacts[0].update(created_at="2026-10-26T10:00:00Z"), + lambda a: a.artifacts[0].update(name="wrong-name"), + lambda a: a.run.update(run_attempt=2), + lambda a: a.revisions.update(target="d" * 40), + ], +) +def test_missing_artifact_or_unverifiable_attempt_target_skips(mutation): + api = FakeGitHub() + mutation(api) + result, report = publisher(api).publish(api.event) + assert result["publication"]["status"] == "skipped" + assert "Publication skipped" in report and not api.writes + + +@pytest.mark.parametrize("number,run_id,attempt", [(21, 11, 1), (20, 10, 2), (20, 10, 1), (20, 11, 1)]) +def test_equal_or_newer_source_report_never_overwritten(number, run_id, attempt): + api = FakeGitHub() + api.comments = [bot_comment(number, run_id, attempt)] + result, _ = publisher(api).publish(api.event) + assert result["publication"]["status"] == "skipped" and not api.writes + + +@pytest.mark.parametrize("change", ["head", "closed", "attempt", "fork-target"]) +def test_current_pr_and_attempt_rechecked_immediately_before_write(change): + api = FakeGitHub() + if change == "head": + api.before_final_pr = lambda: api.pr["head"].update(sha="d" * 40) + elif change == "closed": + api.before_final_pr = lambda: api.pr.update(state="closed") + elif change == "attempt": + api.before_final_run = lambda: api.run.update(run_attempt=2) + else: + api.run["pull_requests"] = [] + api.before_final_pr = lambda: api.pr["base"].update(sha="d" * 40) + result, report = publisher(api).publish(api.event) + assert result["publication"]["status"] == "skipped" and not api.writes + assert "Informational medians" in report + + +def test_invalid_or_duplicate_bot_markers_fail_closed(): + for comments in [[dict(bot_comment(), body=MARKER + "\nmalformed")], [bot_comment(), bot_comment()]]: + api = FakeGitHub() + api.comments = comments + with pytest.raises(InvalidResult): + publisher(api).publish(api.event) + assert not api.writes + + +def test_api_failure_propagates_not_success(): + api = FakeGitHub() + api.write_error = True + with pytest.raises(InvalidResult, match="403"): + publisher(api).publish(api.event) + assert not api.writes + + +def test_publisher_failed_execution_is_still_reported(): + api = FakeGitHub() + api.documents[1]["benchmarks"][0].update(status="oom") + result, report = publisher(api).publish(api.event) + assert result["execution_failed"] + assert "Benchmark execution failed" in report and api.writes + + +@pytest.mark.parametrize( + "conclusion", + ["failure", "cancelled", "timed_out", "action_required", "neutral", "skipped", "stale", "startup_failure"], +) +def test_failed_source_run_cannot_publish_successful_partial_results(conclusion): + api = FakeGitHub() + api.run["conclusion"] = conclusion + result, report = publisher(api).publish(api.event) + assert result["execution_failed"] + assert result["source_execution_status"] == conclusion + assert all("base_median" not in row and "change_percent" not in row for row in result["rows"]) + assert "Benchmark execution failed" in report and api.writes + assert f"Source execution: {conclusion}" in html.unescape(api.writes[0][2]["body"]).replace("\u200b", "") + + +def test_publish_cli_uses_environment_token_and_writes_skip_outputs(tmp_path, monkeypatch): + api = FakeGitHub() + api.artifacts.clear() + tokens = [] + monkeypatch.setattr(cli, "GitHub", lambda token: tokens.append(token) or api) + event_path = tmp_path / "event.json" + event_path.write_text(json.dumps(api.event)) + env = { + "GH_TOKEN": "synthetic-token", + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_EVENT_PATH": str(event_path), + "GITHUB_REPOSITORY": "PostHog/sdk", + } + args = [ + "--mode", + "publish", + "--workflow-path", + ".github/workflows/benchmarks.yml", + "--artifact-name", + "benchmark-results", + "--report-key", + "node-performance", + "--sdk", + "posthog-node", + "--output-directory", + str(tmp_path / "out"), + ] + assert cli.main(args, env) == 0 + assert tokens == ["synthetic-token"] + assert json.loads((tmp_path / "out/comparison.json").read_text())["publication"]["status"] == "skipped" + env["GITHUB_EVENT_NAME"] = "pull_request_target" + assert cli.main(args, env) == 1 + assert not api.writes + + +def test_newer_attempt_updates_older_same_run_report(): + api = FakeGitHub() + api.run["run_attempt"] = api.event["workflow_run"]["run_attempt"] = 2 + api.comments = [bot_comment(20, 10, 1)] + result, _ = publisher(api).publish(api.event) + assert result["publication"]["status"] == "updated" + assert "sdk-performance-source:20:10:2" in api.writes[0][2]["body"] + + +def test_original_target_is_used_when_same_repo_base_advances(): + api = FakeGitHub() + api.pr["base"]["sha"] = "d" * 40 + assert publisher(api).publish(api.event)[0]["publication"]["status"] == "created" + assert (ROOT + f"/compare/{TARGET}...{HEAD}", "GET") in api.calls + + +def test_previous_attempt_artifact_is_not_selected(): + api = FakeGitHub() + api.artifacts.append(dict(copy.deepcopy(api.artifacts[0]), created_at="2026-10-26T10:00:00Z", id=41)) + assert publisher(api).publish(api.event)[0]["publication"]["status"] == "created" + assert [path for path, method in api.calls if method == "DOWNLOAD"] == [ROOT + "/actions/artifacts/40/zip"] diff --git a/.github/actions/benchmark-report/tests/test_benchmark_report.py b/.github/actions/benchmark-report/tests/test_benchmark_report.py new file mode 100644 index 0000000..ed1d036 --- /dev/null +++ b/.github/actions/benchmark-report/tests/test_benchmark_report.py @@ -0,0 +1,451 @@ +"""Schema, comparison and offline reporting regressions for both producer formats.""" + +import copy +import json +from pathlib import Path + +import pytest +import yaml + +from benchmarks.report import cli +from benchmarks.report.compare import compare +from benchmarks.report.render import MAX_REPORT_BYTES, headline_rows, json_text, render +from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, load_result, validate + +FIXTURES = Path(__file__).parent / "fixtures" / "benchmarks" + + +def pair(sdk="node"): + return [load_result(FIXTURES / f"{sdk}-{role}.json") for role in ("base", "head")] + + +def metric(doc): + return doc["benchmarks"][0]["metrics"][0] + + +@pytest.mark.parametrize("sdk", ["node", "python"]) +def test_producer_fixtures_render_with_checkpoints(sdk): + base, head = pair(sdk) + result = compare(base, head) + assert not result["compatibility"] + assert not result["execution_failed"] + assert any(w["head"]["checkpoints"] for w in result["workloads"].values()) + row = next(r for r in result["rows"] if r.get("pairs")) + assert row["pairs"] == 2 + assert row["paired_delta"]["min"] <= row["paired_delta"]["median"] <= row["paired_delta"]["max"] + report = render(result) + assert "inconclusive" in report and "Resource limits:" in report + assert "confidence intervals" in report + assert head["revision"]["commit"] in report + assert len(report.encode()) < MAX_REPORT_BYTES + + +@pytest.mark.parametrize("status", ["error", "timeout", "oom", "unsupported"]) +@pytest.mark.parametrize("field", ["reason", "error"]) +def test_raw_failure_explanations_retain_pass_and_repetition(status, field): + base, head = pair() + raw = head["benchmarks"][0]["runs"][0] + raw.update(status=status, stdout="PRIVATE_RAW_LOG") + raw[field] = "controlled raw failure