feat: add paired Ruby SDK performance benchmarks

.github/workflows/benchmark-report.yml

@@ -0,0 +1,21 @@
+name: Ruby SDK benchmark report
+
+on:
+    workflow_run:
+        workflows: ['Ruby SDK benchmarks']
+        types: [completed]
+
+permissions: {}
+
+jobs:
+    report:
+        permissions:
+            contents: read
+            actions: read
+            pull-requests: write
+        uses: PostHog/.github/.github/workflows/sdk-benchmark-report.yml@2a6111d08d784c0ca84f7f97f32a13fdd6d31449 # SDK benchmark workflow (PostHog/.github PR #84)
+        with:
+            workflow-path: .github/workflows/benchmarks.yml
+            artifact-name: ruby-benchmark-results
+            report-key: posthog-ruby
+            sdk: posthog-ruby

.github/workflows/benchmarks.yml

@@ -0,0 +1,149 @@
+name: Ruby SDK benchmarks
+
+on: [pull_request]
+
+permissions: {}
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+    cancel-in-progress: true
+
+jobs:
+    benchmark:
+        runs-on: ubuntu-24.04
+        timeout-minutes: 12
+        env:
+            BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle
+        strategy:
+            fail-fast: false
+            matrix:
+                shard: [flags-single, flags-scaling, flags-memory, capture]
+        permissions:
+            contents: read
+        steps:
+            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+              with:
+                  ref: ${{ github.event.pull_request.head.sha }}
+                  fetch-depth: 0
+                  persist-credentials: false
+
+            - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+              with:
+                  python-version: '3.12.12'
+
+            - uses: ruby/setup-ruby@b1da4809b005e0178fe08a1d57317bffc00b7b4c # v1.332.0
+              with:
+                  ruby-version: '3.4.7'
+                  bundler: '4.0.13'
+                  bundler-cache: true
+
+            - name: Prepare pinned baseline
+              env:
+                  TARGET_SHA: ${{ github.event.pull_request.base.sha }}
+                  HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+              run: |
+                  set -euo pipefail
+                  export BASE_WORKTREE="$RUNNER_TEMP/ruby-benchmark-base"
+                  export RESULTS="$RUNNER_TEMP/ruby-benchmark-results"
+                  printf 'BASE_WORKTREE=%s\nRESULTS=%s\n' "$BASE_WORKTREE" "$RESULTS" >> "$GITHUB_ENV"
+                  BASE_SHA=$(git merge-base "$TARGET_SHA" "$HEAD_SHA")
+                  git worktree add --detach "$BASE_WORKTREE" "$BASE_SHA"
+                  mkdir -p "$RESULTS"
+                  python -c 'import json, os, pathlib, sys; pathlib.Path(os.environ["RESULTS"], "revisions.json").write_text(json.dumps({"target":os.environ["TARGET_SHA"], "head":os.environ["HEAD_SHA"], "base":sys.argv[1]}, indent=2))' "$BASE_SHA"
+
+            - name: Build and install isolated gems
+              run: |
+                  python -B benchmarks/prepare.py . "$RUNNER_TEMP/ruby-benchmark-build/head"
+                  python -B benchmarks/prepare.py "$BASE_WORKTREE" "$RUNNER_TEMP/ruby-benchmark-build/base"
+
+            - name: Test measurement controls and fixtures
+              run: |
+                  python -B benchmarks/test_benchmarks.py
+                  python -B benchmarks/check_native.py "$RUNNER_TEMP/ruby-benchmark-build/head"
+
+            - name: Measure paired base and head
+              id: measure
+              timeout-minutes: 10
+              run: |
+                  python -B benchmarks/run.py \
+                    --base "$BASE_WORKTREE" \
+                    --head "$GITHUB_WORKSPACE" \
+                    --build-directory "$RUNNER_TEMP/ruby-benchmark-build" \
+                    --output "$RESULTS" \
+                    --repetitions 6 \
+                    --shard "${{ matrix.shard }}"
+
+            - name: Upload raw shard results
+              if: always()
+              uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+              with:
+                  name: benchmark-shard-${{ matrix.shard }}
+                  path: |
+                      ${{ runner.temp }}/ruby-benchmark-results/base.json
+                      ${{ runner.temp }}/ruby-benchmark-results/head.json
+                      ${{ runner.temp }}/ruby-benchmark-results/revisions.json
+                  if-no-files-found: warn
+                  retention-days: 14
+
+    collect:
+        needs: benchmark
+        if: ${{ !cancelled() }}
+        runs-on: ubuntu-24.04
+        timeout-minutes: 5
+        permissions:
+            contents: read
+        steps:
+            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+              with:
+                  ref: ${{ github.event.pull_request.head.sha }}
+                  persist-credentials: false
+
+            - name: Download shard results
+              uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+              with:
+                  pattern: benchmark-shard-*
+                  path: ${{ runner.temp }}/ruby-benchmark-shards
+                  merge-multiple: false
+
+            - name: Collect and render benchmark comparison
+              uses: PostHog/.github/.github/actions/benchmark-report@04f8421ec4daa7f4066eaeb4c53621ba79ddc3ca # SDK benchmark reporting (PostHog/.github PR #84)
+              with:
+                  mode: collect
+                  source-status: ${{ needs.benchmark.result }}
+                  shard-manifest: benchmarks/shards.json
+                  shard-directory: ${{ runner.temp }}/ruby-benchmark-shards
+                  output-directory: ${{ runner.temp }}/ruby-benchmark-report
+
+            - name: Upload rendered comparison
+              if: always()
+              uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+              with:
+                  name: ruby-benchmark-report
+                  path: |
+                      ${{ runner.temp }}/ruby-benchmark-report/report.md
+                      ${{ runner.temp }}/ruby-benchmark-report/comparison.json
+                  if-no-files-found: ignore
+                  retention-days: 14
+
+            - name: Upload raw benchmark results
+              if: always()
+              id: results
+              uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+              with:
+                  name: ruby-benchmark-results
+                  path: |
+                      ${{ runner.temp }}/ruby-benchmark-report/bundle.json
+                      ${{ runner.temp }}/ruby-benchmark-report/revisions.json
+                  if-no-files-found: warn
+                  retention-days: 14
+
+            - name: Link results
+              if: always()
+              env:
+                  ARTIFACT_URL: ${{ steps.results.outputs.artifact-url }}
+              run: |
+                  printf '## Ruby SDK benchmark data\n\n' >> "$GITHUB_STEP_SUMMARY"
+                  if [[ -n "$ARTIFACT_URL" ]]; then
+                    printf '[Download paired measurements and execution metadata](%s)\n\n' "$ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY"
+                  fi
+                  printf 'Measurements are informational. Check workload statuses before comparing values.\n' >> "$GITHUB_STEP_SUMMARY"

CONTRIBUTING.md

@@ -75,6 +75,37 @@ This section is for external contributors. PostHog maintainers (members of the P
 
 A diff in `public_api_snapshot.txt` (see "CI-aligned checks" above) means your change touches public API.
 
+## SDK performance benchmarks
+
+Every PR runs the 20-workload [`sdk-benchmarks/v1` specification](https://github.com/PostHog/.github/blob/04f8421ec4daa7f4066eaeb4c53621ba79ddc3ca/.github/workflows/sdk-benchmark-specification.md) against its merge base and exact head. The same head-side fixtures run against separately built, installed gems. Four independent workload shards keep all six alternating base/head pairs on one runner, with a fresh Ruby process for each workload, measurement pass and repetition. No performance percentage is a CI gate; build, correctness and execution errors remain failures.
+
+The benchmark preparation step installs each checkout's frozen Bundler dependencies (with the repository's RubyGems cooldown), builds its SDK gem, and installs that gem into a fresh external directory. Worker processes use those installed gems, not source imports:
+
+```bash
+python3 -B benchmarks/prepare.py /path/to/base /path/to/build/base
+python3 -B benchmarks/prepare.py "$PWD" /path/to/build/head
+python3 -B benchmarks/test_benchmarks.py
+python3 -B benchmarks/check_native.py /path/to/build/head
+python3 -B benchmarks/run.py \
+  --base /path/to/base --head "$PWD" --build-directory /path/to/build \
+  --output /path/to/results --smoke --repetitions 1
+```
+
+Omit `--smoke` and use `--repetitions 6` for full measurements. `--shard` selects `flags-single`, `flags-scaling`, `flags-memory` or `capture`; `--filter` selects a workload prefix for debugging and cannot be combined with a shard. Smoke checks retain the complete inventory with smaller operation counts. Full flags allocation passes use the documented smaller counts recorded in `allocation_operations`.
+
+Measurement scope:
+
+- Elapsed time uses `Process.clock_gettime(CLOCK_MONOTONIC)`. CPU uses process time, except enqueue-only workloads use caller-thread time because background serialization has no public enqueue completion signal.
+- Allocation passes count Ruby objects with cumulative `GC.stat(:total_allocated_objects)`, including objects later collected and allocations on SDK background threads. Native malloc is excluded. Cumulative allocated bytes are unsupported, not estimated from retained heap. Enqueue-only allocations are also unsupported; completed-delivery workloads measure the full worker pipeline.
+- Memory passes sample RSS at the specification's operation checkpoints and before/after work. Linux uses `/proc/self/status`; macOS uses `ps`. Linux lifetime peak RSS uses `VmHWM`; it is explicitly unsupported on macOS. `ObjectSpace.memsize_of_all` reports retained live-object bytes after settling, not total allocator arenas or cumulative allocation volume. Snapshot overhead is part of memory observation. GC runs before/after memory measurement, never inside the workload/checkpoints.
+- Eight native threads perform the load scenario under Ruby's GVL; this does not promise eight parallel CPU cores. Load checkpoints join each 32-operation group.
+- A coordinator-owned loopback receiver exercises the normal SDK HTTP, batching, byte-size splitting and background-worker paths. Receiver CPU, allocations and retained fixtures are outside the measured Ruby process. Refresh elapsed/allocations include the worker's loopback control request selecting the next generation. No production service or credentials are used.
+- Public local-only flag getters and reload APIs are used, with flag-called events disabled. The single-flag getter's deprecation warning occurs during warmup. Capture uses public `batch_size: 100000`, `max_queue_size: 100001`, `flush_interval_seconds: 3600` and `compress_request: false`. Native byte-size batching remains active. Warmup deliveries are verified before counters reset; sequences, payloads, users and final delivery are validated without a rescuing flush for delivery workloads.
+
+Workers have a 120-second deadline. After two timeouts a coordinator finishes the current pair, saves partial diagnostics and fails. Each CI shard has a 12-minute job and a 10-minute measurement deadline, reserving time for upload. Collection has five minutes and rejects incomplete shard coverage. Run duration must be assessed from hosted CI, not concurrent local measurements.
+
+The shared reporter validates raw result schema v1 and produces readable tables plus `comparison.json`. Its comment shortlist requires at least a 5% median change and all six pairs agreeing in direction; it is an informational noise filter, not statistical significance. Missing and unsupported counters remain explicit. The read-only producer workflow is separate from the trusted `workflow_run` publisher; automatic PR comments become active only once that caller workflow is on the default branch.
+
 ## Rails package
 
 The `posthog-rails` package has its own package-specific guide in [posthog-rails/CONTRIBUTING.md](posthog-rails/CONTRIBUTING.md).

benchmarks/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+results/

benchmarks/check_native.py

@@ -0,0 +1,97 @@
+"""Run native controls plus regression checks through an installed SDK and loopback receiver."""
+
+import argparse
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+from fixtures import workloads
+from run import HERE, TIMEOUT, WORKER, run_worker, worker_env
+from server import BenchmarkServer
+
+
+def main(installation):
+    with tempfile.TemporaryDirectory(prefix="ruby-controls-") as scratch:
+        env = worker_env(installation, scratch)
+        subprocess.run(
+            ["ruby", str(HERE / "controls.rb")],
+            env=env,
+            cwd=scratch,
+            check=True,
+            timeout=TIMEOUT,
+        )
+        capture = next(
+            s for s in workloads(True) if s["id"] == "capture.drain.steady.32"
+        )
+        code = """require ARGV[0]
+PostHog::Logging.logger = Logger.new(File::NULL)
+w = RubyBenchmark::Workload.new(JSON.parse(ARGV[1]), ARGV[2])
+begin
+  warmup = w.prepare(200)
+  raise 'Warmup size changed' unless warmup == 160
+  raise 'Warmup was not reset' unless w.control('state')['delivered'].zero?
+  # Simulate a flush that returns without delivery at the benchmark seam, not inside the SDK.
+  def w.flush; end
+  w.run(200)
+  begin
+    w.complete
+    raise 'Completed-delivery verification rescued a missing flush'
+  rescue RuntimeError => error
+    raise unless error.message == 'Capture payload, identity or sequence mismatch'
+  end
+ensure
+  w.close
+end
+puts 'Incomplete delivery rejected without a rescuing verification flush'
+"""
+        with BenchmarkServer(capture) as server:
+            subprocess.run(
+                ["ruby", "-e", code, str(WORKER), json.dumps(capture), server.url],
+                env=env,
+                cwd=scratch,
+                check=True,
+                timeout=TIMEOUT,
+            )
+        refresh = next(s for s in workloads(True) if s["mode"] == "refresh")
+        code = """require ARGV[0]
+PostHog::Logging.logger = Logger.new(File::NULL)
+w = RubyBenchmark::Workload.new(JSON.parse(ARGV[1]), ARGV[2])
+begin
+  w.prepare(4)
+  w.reload(1)
+  w.verify
+  w.reload(2)
+  w.verify
+ensure
+  w.close
+end
+puts 'Changing refresh generations verified for all users'
+"""
+        with BenchmarkServer(refresh) as server:
+            subprocess.run(
+                ["ruby", "-e", code, str(WORKER), json.dumps(refresh), server.url],
+                env=env,
+                cwd=scratch,
+                check=True,
+                timeout=TIMEOUT,
+            )
+        # Real child timeout and capped diagnostic output; no SDK state manipulation.
+        sleeper = Path(scratch) / "sleeper.rb"
+        sleeper.write_text("STDERR.write('x' * 100000); STDERR.flush; sleep 60\n")
+        import run
+
+        original = run.WORKER
+        try:
+            run.WORKER = sleeper
+            raw = run_worker(installation, refresh, "memory", timeout=0.5)
+            assert raw["status"] == "timeout" and 0 < len(raw["stderr"]) <= 65536
+        finally:
+            run.WORKER = original
+        print("Real worker timeout classified with capped logs")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("installation", type=Path)
+    main(parser.parse_args().installation.resolve())

benchmarks/controls.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'worker'
+
+# Native counter and fixture controls run against the installed public SDK entry point.
+module RubyBenchmarkControls
+  def self.check(condition, message)
+    raise message unless condition
+  end
+
+  def self.run
+    users = RubyBenchmark::USERS
+    check(users.length == 16, 'Expected 16 users')
+    check(users.map { |u| u[:id] }.uniq.length == 16, 'User identities must be unique')
+    users.each do |user|
+      check(RubyBenchmark.expected(0, user), 'Generation zero boolean')
+      check(!RubyBenchmark.expected(0, user, 1), 'Odd generation boolean')
+      check(!RubyBenchmark.expected(9, user), 'Inactive flag')
+      check(!RubyBenchmark.expected(8, user), 'No-match flag')
+      check(!RubyBenchmark.expected(5, user, 1), 'Odd generation cohort')
+      check(%w[control test].include?(RubyBenchmark.expected(2, user)), 'Variant')
+      check(RubyBenchmark.expected(6, user) == RubyBenchmark.expected(7, user), 'Early and late match equivalence')
+    end
+    before = GC.stat(:total_allocated_objects)
+    objects = Array.new(1000) { Object.new }
+    allocated = GC.stat(:total_allocated_objects) - before
+    check(allocated >= objects.length, 'Cumulative object counter must include live allocations')
+    nil
+    RubyBenchmark.settle
+    check(GC.stat(:total_allocated_objects) >= before + 1000, 'GC must not reset cumulative allocations')
+    snapshot = RubyBenchmark.snapshot
+    check(snapshot[:rss].positive? && snapshot[:heap_used].positive?, 'Native memory counters')
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+    sleep(0.002)
+    check(Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) > start, 'Monotonic clock')
+    check(RubyBenchmark.describe[:entry] == Gem.loaded_specs.fetch('posthog-ruby').full_gem_path,
+          'Installed gem identity')
+    puts 'Native fixture, allocation, memory, clock and installed-entry controls passed'
+  end
+end
+
+RubyBenchmarkControls.run if $PROGRAM_NAME == __FILE__