diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..eecd98490 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,90 @@ +name: Python SDK benchmarks + +on: + pull_request: + paths: + - 'posthog/**' + - 'benchmarks/**' + - 'pyproject.toml' + - 'setup.py' + - 'uv.lock' + - '.github/workflows/benchmarks.yml' + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12.12' + + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: '0.13.4' + enable-cache: true + + - name: Prepare pinned baseline + env: + TARGET_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + export BASE_WORKTREE="$RUNNER_TEMP/python-benchmark-base" + export RESULTS="$RUNNER_TEMP/python-benchmark-results" + printf 'BASE_WORKTREE=%s\nRESULTS=%s\n' "$BASE_WORKTREE" "$RESULTS" >> "$GITHUB_ENV" + BASE_SHA=$(git merge-base "$TARGET_SHA" "$HEAD_SHA") + git worktree add --detach "$BASE_WORKTREE" "$BASE_SHA" + mkdir -p "$RESULTS" + python -c 'import json, os, pathlib, sys; pathlib.Path(os.environ["RESULTS"], "revisions.json").write_text(json.dumps({"target":os.environ["TARGET_SHA"], "head":os.environ["HEAD_SHA"], "base":sys.argv[1]}, indent=2))' "$BASE_SHA" + + - name: Install isolated head and baseline wheels + run: | + python benchmarks/prepare.py . + python benchmarks/prepare.py "$BASE_WORKTREE" + + - name: Test measurement controls and fixtures + run: .venv-benchmark/bin/python -B benchmarks/test_benchmarks.py + + - name: Measure paired base and head + run: | + .venv-benchmark/bin/python -B benchmarks/run.py \ + --base "$BASE_WORKTREE" \ + --head "$GITHUB_WORKSPACE" \ + --output "$RESULTS" \ + --repetitions 6 + + - name: Upload raw benchmark results + if: always() + id: results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-benchmark-results + path: ${{ runner.temp }}/python-benchmark-results + if-no-files-found: warn + retention-days: 14 + + - name: Link results + if: always() + env: + ARTIFACT_URL: ${{ steps.results.outputs.artifact-url }} + run: | + printf '## Python SDK benchmark data\n\n' >> "$GITHUB_STEP_SUMMARY" + if [[ -n "$ARTIFACT_URL" ]]; then + printf '[Download paired measurements and execution metadata](%s)\n\n' "$ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY" + fi + printf 'Measurements are informational. Check workload statuses before comparing values.\n' >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 42fbc82dd..b8e7cb9ac 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ build/ env/ venv/ .venv/ +.venv-benchmark/ flake8.out pylint.out posthog-analytics diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..8ca37a758 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,117 @@ +# Python SDK benchmarks + +PR-only, paired measurements of independently installed Python SDK revisions. Each workload/pass/repetition runs in a fresh process against the target's non-editable wheel. Benchmark code comes from one checkout and emits `base.json` and `head.json` in the Node pilot's result structure. + +## Setup and local execution + +Use **CPython 3.12.12** and **uv 0.13.4**, matching `.github/workflows/benchmarks.yml`. + +From the checkout containing this suite, prepare each revision: + +```sh +python benchmarks/prepare.py /absolute/path/to/head +python benchmarks/prepare.py /absolute/path/to/base +``` + +Each target gets its own `.venv-benchmark`. Preparation installs the SDK and runtime dependencies from that revision's frozen `uv.lock`, builds a non-editable SDK wheel, then installs hash-pinned benchmark instrumentation from this suite's `requirements.txt`. Preparation fails if instrumentation changes an already installed SDK dependency version. Both environments precompile bytecode: importing cached bytecode on one side and compiling source on the other can otherwise create an apparent RSS regression even with identical SDK source. Workers do not write new bytecode. The benchmark environment is disposable and separate from the ordinary development `.venv`. + +Run the controls and benchmarks using the head environment: + +```sh +.venv-benchmark/bin/python -B benchmarks/test_benchmarks.py +.venv-benchmark/bin/python -B benchmarks/run.py \ + --base /absolute/path/to/base \ + --head /absolute/path/to/head \ + --output /tmp/python-benchmark-results \ + --repetitions 6 +``` + +`--smoke --repetitions 1` checks integration quickly; smoke results are not suitable for performance conclusions. `--filter flags.all.` selects workload IDs by prefix. Using the same revision on both sides measures repeatability, not an improvement. + +Rerun preparation after changing SDK source. Preparation forces a `posthog` rebuild/reinstall so source-only changes cannot reuse uv's cached wheel. The worker deliberately does not import the repository source tree. Results fingerprint the installed package's Python files and record its entry path, SDK commit/dirty state, package version, lockfile digest, actual installed dependency/tool versions, benchmark source digest, runtime, fixture digest, and paired-run ID. + +To update benchmark tooling, use a supported uv version and review the regenerated lock: + +```sh +uv pip compile benchmarks/requirements.in --python-version 3.12 --universal \ + --generate-hashes --exclude-newer '7 days' --output-file benchmarks/requirements.txt +``` + +## Workloads + +| Workload | Public path | Measurements | +| --- | --- | --- | +| Nine single-flag cases | `get_feature_flag` | Elapsed time/op, process CPU/op, Python allocator bytes/calls per op | +| All flags: 10, 100, 1,000 | `get_all_flags` | Same metrics per complete all-flags call | +| Sustained 1,000-flag evaluation | Eight threads calling `get_all_flags` | RSS observations and post-settling RSS | +| Forty definition refreshes | `load_feature_flags`, then validated `get_all_flags` | Memory checkpoints every five cycles and at the end | +| Capture: 32-byte and 4,096-byte string properties | `capture` | Foreground elapsed time and caller-thread CPU per event | +| Capture and drain: both payload sizes | `capture`, `flush` every 100 events | Full-pipeline time, process CPU, allocator bytes/calls, and memory | + +The flag mix and sixteen identities match the Node pilot conceptually: boolean, percentage rollout, multivariate assignment, person properties, regex, cohort, early/late/no-match, and inactive flags. Expected values are computed independently of the SDK. Every identity is validated before and after measurement. Refresh generations alter actual boolean/cohort results and regex patterns so a fetch without a state update fails validation. + +The suite covers the supported legacy flag getters, not `evaluate_flags` snapshots. Deprecation warnings are encountered during setup. Eight threads are **GIL-constrained concurrency**, not a claim of eight-way parallel Python execution. + +### Transport and lifecycle + +Python has no public fetch adapter. A bounded loopback HTTP server runs in the coordinator process, outside the measured worker, and serves definitions and accepts uncompressed v0 batches. Its routes are limited to the fixture protocol and a control/inspection interface. It retains counters and per-identity counts, not event history. Remote flag fallback is unexpected and fails verification. + +The SDK uses real requests, JSON parsing/serialization, background consumers, flag loading, flush, and shutdown. No private queue, evaluator, or poller is replaced. Automatic polling and partial-batch deadlines are set to one hour, longer than the worker timeout. Refreshes and draining use public methods. Worker shutdown must stop SDK threads. + +Capture checks require the correct event name, exact payload, identity distribution, and total delivered count. Rejected captures, timeouts that leave undelivered events, malformed payloads, and remote fallback cannot appear as successful low-cost work. + +### Capture measurement boundaries + +`capture()` returns after foreground preparation/enqueue, while a consumer can concurrently prepare its batch. There is no public completion boundary for that background preparation short of flushing. Therefore: + +- `capture.enqueue.*` reports foreground elapsed time and **caller-thread CPU**, not process-wide CPU or allocation totals. +- `capture.drain.*` includes `flush(timeout_seconds=None)` and reports the complete delivery path's process CPU and allocations, including requests/serialization. The external worker timeout bounds this wait. Verification does not flush again: unfinished measured work cannot be rescued outside the measurement. Server CPU and server allocations are excluded; elapsed time includes loopback transport. +- Enqueue-only allocation metrics are explicitly unavailable and direct readers to the drain workload. + +The enqueue workload uses 1,000 events, below queue/batch-count limits; partial batches can still be prepared by the consumer. These metrics describe their named paths rather than pretending Python and Node have identical asynchronous lifecycles. + +## Measurement methods + +Base/head alternate first position between independent repetitions and never benchmark concurrently. The same head-side fixtures and measurement code run against both installations. The runner verifies matching Python, profiler, psutil, OS, and architecture versions. Both workers use a fixed hash seed and a clean Python configuration environment. + +Warmup and correctness checks happen outside timing. Timing, allocation profiling, and memory observation are separate passes. Allocation passes use smaller fixed operation counts where tracing every Python allocation would otherwise produce excessive profiler files; the actual count is included in each result and used as its denominator. These counts are identical for base and head. + +### Time and CPU + +- `elapsed_time_per_op`: `time.perf_counter_ns`, normalized over a batch. +- `process_cpu_time_per_op`: `time.process_time_ns`, including all worker threads. +- `caller_thread_cpu_time_per_op`: `time.thread_time_ns`, for foreground enqueue only. + +The metrics include the small benchmark-loop overhead. Elapsed time is not CPU utilization. + +### Allocations + +Memray runs with `trace_python_allocators=True` and `FileFormat.ALL_ALLOCATIONS`. After tracking stops, the adapter streams all records and sums `PYMALLOC_MALLOC`, `PYMALLOC_CALLOC`, and `PYMALLOC_REALLOC` requests. This includes requests whose allocations were already freed. It excludes free records and native backing allocations, avoiding interpreting backing arenas as individual Python allocations. + +- `allocated_bytes_per_op` is cumulative **requested Python allocator bytes**. A reallocation contributes its requested size, not just its growth. +- `allocations_per_op` is **Python allocator calls, including reallocations**, not a count of distinct Python objects. +- Both are measured counters, unlike Node's estimated V8 allocation bytes. They are still not cross-language-comparable totals. +- Native-library allocations outside Python's allocators are not included. RSS covers the whole process. + +Profiler capture and aggregation do not contribute to reported timing. Profiles are streamed and removed after aggregation so repeated runs do not retain large binary captures. Raw counters and profiler versions remain in the JSON. + +The control suite demonstrates that this configuration records freed small allocations that default Memray tracking does not expose as Python allocator requests. Other controls preserve SDK outputs while adding CPU work and allocation churn; both must be detected. See [Memray's allocator documentation](https://bloomberg.github.io/memray/python_allocators.html) and [API documentation](https://bloomberg.github.io/memray/api.html). + +### Memory + +- Idle RSS: after imports, definitions, validation, warmup, and settling. +- Observed workload peak RSS: operation checkpoints and end-of-work observation; can miss intervening spikes. +- Lifetime peak RSS: `resource.getrusage`, normalized from platform-specific units. Includes setup and warmup, not just measured work. +- Post-settling RSS: after two `gc.collect()` calls separated by a scheduler yield. Python reference counting also frees objects during ordinary execution. + +CPython has no uninstrumented total managed-heap byte counter equivalent to Node's `heapUsed`. `post_settling_heap_used` is explicitly unsupported; enabling tracing in the RSS pass would change the measurement. Retained RSS alone does not prove a leak because allocators can keep arenas resident. A public-entry control deliberately retains 8 MiB on each refresh and verifies that post-GC checkpoints expose the growth. + +Each worker has a two-minute timeout and an externally monitored **512 MiB RSS ceiling**, checked approximately every 20 ms. This is a safety bound, not an exact OS-enforced limit: short spikes can exceed it between observations. A ceiling violation is reported as `error` with `rss_limit_exceeded`, not falsely described as a confirmed OS OOM. An explicit Python `MemoryError` is classified as OOM; a generic kill is not. The suite stops after two worker timeouts, after completing the current base/head pair, and preserves partial results for upload. Worker temporary files, including profiler traces, are owned and removed by the coordinator even when a worker is killed. + +## Results and CI + +The pilot JSON format uses `schema_version: 1`, with the same top-level revision, suite, execution, benchmark, metric, sample, and status structure as Node. Measurement IDs, units, method names, and quality are explicit. Python's intentional differences are allocator-call counts, foreground-thread CPU for enqueue, unavailable uninstrumented heap size, and a separate HTTP receiver. Shared schema finalization and rendering are separate work. + +The PR workflow builds the merge base and exact head into isolated environments, then runs this head-side suite against both. It requests read-only permissions, uses pinned actions, requires no service credentials, preserves partial results after worker failures, and uploads JSON artifacts linked from the job summary. + +Correctness failures and broken execution fail the job. Performance differences do not: this producer does not compute regression verdicts. Shared comparisons, noise classification, and PR comments remain separate. Calibrate on the actual Linux runner before introducing performance gates; local macOS results are not a CI noise floor. diff --git a/benchmarks/fixtures.py b/benchmarks/fixtures.py new file mode 100644 index 000000000..a59599584 --- /dev/null +++ b/benchmarks/fixtures.py @@ -0,0 +1,209 @@ +"""Deterministic fixtures matching the Node pilot's flag mix and identities.""" + +import hashlib +import json +from typing import Any + +CASES = [ + "boolean", + "rollout", + "variant", + "property", + "regex", + "cohort", + "early", + "late", + "no-match", + "inactive", +] +USERS = [ + { + "distinct_id": f"benchmark-user-{i}", + "person_properties": { + "plan": "free" if i % 2 else "pro", + "email": f"user-{i}@example.test", + "score": i, + }, + } + for i in range(16) +] + + +def prop(key, value, operator="exact"): + return {"key": key, "value": value, "operator": operator, "type": "person"} + + +def group(properties=None, rollout=100): + return {"properties": properties or [], "rollout_percentage": rollout} + + +def definitions(count, generation=0): + flags = [] + for i in range(count): + kind = CASES[i % len(CASES)] + match = group([prop("plan", ["pro"])]) + miss = group([prop("plan", ["never"])]) + groups = [group()] + if kind == "rollout": + groups = [group(rollout=50)] + elif kind == "property": + groups = [match] + elif kind == "regex": + groups = [ + group( + [ + prop( + "email", + f"^(user|refresh-{generation})-[0-9]+@example[.]test$", + "regex", + ) + ] + ) + ] + elif kind == "cohort": + groups = [group([{"key": "id", "value": 1, "type": "cohort"}])] + elif kind == "early": + groups = [match] + [miss] * 7 + elif kind == "late": + groups = [miss] * 7 + [match] + elif kind == "no-match": + groups = [miss] * 8 + filters: dict[str, Any] = {"groups": groups} + if kind == "variant": + filters["multivariate"] = { + "variants": [ + {"key": "control", "rollout_percentage": 50}, + {"key": "test", "rollout_percentage": 50}, + ] + } + flags.append( + { + "id": i + 1, + "key": f"{kind}-{i}", + "name": f"{kind}-{i}", + "active": kind != "inactive" + and (kind != "boolean" or generation % 2 == 0), + "filters": filters, + } + ) + return { + "flags": flags, + "group_type_mapping": {}, + "cohorts": { + "1": { + "type": "AND", + "values": [ + prop("plan", ["pro"]), + prop("score", 100 if generation % 2 else -1, "gt"), + ], + }, + }, + } + + +def bucket(key, distinct_id, salt=""): + return int( + hashlib.sha1(f"{key}.{distinct_id}{salt}".encode()).hexdigest()[:15], 16 + ) / (2**60 - 1) + + +def expected_value(index, user, generation=0): + kind = CASES[index % len(CASES)] + key = f"{kind}-{index}" + if kind == "rollout": + return bucket(key, user["distinct_id"]) < 0.5 + if kind == "variant": + return ( + "control" if bucket(key, user["distinct_id"], "variant") < 0.5 else "test" + ) + if kind == "boolean": + return generation % 2 == 0 + if kind == "cohort" and generation % 2: + return False + if kind in ("property", "cohort", "early", "late"): + return user["person_properties"]["plan"] == "pro" + return kind not in ("inactive", "no-match") + + +def expected_flags(count, user, generation=0): + return { + f"{CASES[i % len(CASES)]}-{i}": expected_value(i, user, generation) + for i in range(count) + } + + +def digest(value): + data = ( + value + if isinstance(value, bytes) + else json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ) + return hashlib.sha256(data).hexdigest() + + +def workloads(smoke=False): + return [ + *[ + { + "id": f"flags.single.{kind}", + "mode": "single", + "flag_count": 10, + "flag_index": i, + "operations": 160 if smoke else 16000, + "allocation_operations": 160 if smoke else 2000, + "concurrency": 1, + "passes": ["timing", "allocation"], + } + for i, kind in enumerate(CASES[:9]) + ], + *[ + { + "id": f"flags.all.{count}", + "mode": "all", + "flag_count": count, + "operations": 16 if smoke else max(160, 16000 // count), + "allocation_operations": 16 if smoke else max(16, 1600 // count), + "concurrency": 1, + "passes": ["timing", "allocation"], + } + for count in (10, 100, 1000) + ], + { + "id": "flags.load.1000", + "mode": "all", + "flag_count": 1000, + "operations": 32 if smoke else 1600, + "concurrency": 8, + "passes": ["memory"], + }, + { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 4 if smoke else 40, + "concurrency": 1, + "passes": ["memory"], + }, + *[ + { + "id": f"capture.enqueue.{size}", + "mode": "capture", + "payload_bytes": size, + "operations": 100 if smoke else 1000, + "concurrency": 1, + "passes": ["timing"], + } + for size in (32, 4096) + ], + *[ + { + "id": f"capture.drain.{size}", + "mode": "drain", + "payload_bytes": size, + "operations": 200 if smoke else 1000, + "concurrency": 1, + "passes": ["timing", "allocation", "memory"], + } + for size in (32, 4096) + ], + ] diff --git a/benchmarks/identity.py b/benchmarks/identity.py new file mode 100644 index 000000000..6e77152b8 --- /dev/null +++ b/benchmarks/identity.py @@ -0,0 +1,45 @@ +import importlib.metadata +import platform +import sys +from pathlib import Path + +from fixtures import digest + + +def source_digest(directory): + directory = Path(directory) + return digest( + { + str(path.relative_to(directory)): digest(path.read_bytes()) + for path in sorted(directory.rglob("*")) + if path.is_file() + and path.suffix in (".py", ".txt", ".in") + and "__pycache__" not in path.parts + } + ) + + +def environment(): + import posthog + + entry = Path(posthog.__file__).resolve() + if not entry.is_relative_to(Path(sys.prefix)): + raise RuntimeError(f"Expected installed SDK inside {sys.prefix}, got {entry}") + packages = { + dist.metadata["Name"].lower().replace("_", "-"): dist.version + for dist in importlib.metadata.distributions() + } + return { + "runtime": "cpython", + "runtime_version": platform.python_version(), + "implementation": sys.version, + "os": sys.platform, + "arch": platform.machine(), + "kernel": platform.release(), + "cpu": platform.processor(), + "entry": str(entry), + "build_digest": source_digest(entry.parent), + "packages": packages, + "sdk_version": importlib.metadata.version("posthog"), + "measurement_tools": {name: packages[name] for name in ("memray", "psutil")}, + } diff --git a/benchmarks/measurement.py b/benchmarks/measurement.py new file mode 100644 index 000000000..d79875e17 --- /dev/null +++ b/benchmarks/measurement.py @@ -0,0 +1,85 @@ +"""Process measurements; allocation instrumentation never runs in a timing/RSS pass.""" + +import gc +import resource +import sys +import tempfile +import time +from pathlib import Path + +import psutil + +PROCESS = psutil.Process() + + +def settle(): + gc.collect() + time.sleep(0) + gc.collect() + + +def memory(): + return {"rss": PROCESS.memory_info().rss} + + +def lifetime_peak_rss(): + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return value if sys.platform == "darwin" else value * 1024 + + +def time_operations(run, operations): + cpu_start = time.process_time_ns() + thread_start = time.thread_time_ns() + start = time.perf_counter_ns() + run() + elapsed = time.perf_counter_ns() - start + thread_cpu = time.thread_time_ns() - thread_start + cpu = time.process_time_ns() - cpu_start + return { + "elapsed_ns": elapsed, + "elapsed_time_per_op": elapsed / operations, + "process_cpu_time_per_op": cpu / operations, + "caller_thread_cpu_time_per_op": thread_cpu / operations, + } + + +def allocation_totals(path): + import memray + + allocators = { + memray.AllocatorType.PYMALLOC_MALLOC, + memray.AllocatorType.PYMALLOC_CALLOC, + memray.AllocatorType.PYMALLOC_REALLOC, + } + total_bytes = total_calls = 0 + with memray.FileReader(path) as reader: + for record in reader.get_allocation_records(): + if record.allocator in allocators: + total_bytes += record.size + total_calls += record.n_allocations + return {"allocated_bytes": total_bytes, "allocation_calls": total_calls} + + +def sample_allocations(run): + import memray + + with tempfile.TemporaryDirectory(prefix="posthog-allocations-") as directory: + path = Path(directory) / "allocations.bin" + with memray.Tracker( + path, + trace_python_allocators=True, + file_format=memray.FileFormat.ALL_ALLOCATIONS, + ): + run() + return allocation_totals(path) + + +def classify_exit(*, timed_out=False, rss_limit_exceeded=False, stderr=""): + if timed_out: + return "timeout" + # A parent-enforced RSS ceiling is a measured limit violation, not proof of OS OOM. + if rss_limit_exceeded: + return "error" + if any(line.strip() == "MemoryError" for line in stderr.splitlines()): + return "oom" + return "error" diff --git a/benchmarks/prepare.py b/benchmarks/prepare.py new file mode 100644 index 000000000..c8405ce0b --- /dev/null +++ b/benchmarks/prepare.py @@ -0,0 +1,81 @@ +"""Build an isolated, non-editable SDK installation with fixed benchmark tools.""" + +import argparse +import json +import os +import subprocess +import tempfile +from pathlib import Path + +PACKAGES = "import importlib.metadata as m, json; print(json.dumps({d.metadata['Name'].lower().replace('_','-'): d.version for d in m.distributions()}))" + + +def prepare(target, python_version, uv): + target = target.resolve() + environment = target / ".venv-benchmark" + env = {**os.environ, "UV_PROJECT_ENVIRONMENT": str(environment)} + subprocess.run( + [ + uv, + "sync", + "--project", + str(target), + "--frozen", + "--no-dev", + "--no-editable", + "--compile-bytecode", + "--reinstall-package", + "posthog", + "--python", + python_version, + ], + env=env, + check=True, + ) + python = environment / "bin/python" + # Inspect from outside the source tree; the worker must import the built wheel. + with tempfile.TemporaryDirectory(prefix="posthog-benchmark-prepare-") as cwd: + before = json.loads( + subprocess.check_output([python, "-I", "-c", PACKAGES], cwd=cwd) + ) + subprocess.run( + [ + uv, + "pip", + "install", + "--python", + str(python), + "--no-deps", + "--require-hashes", + "--compile-bytecode", + "-r", + str(Path(__file__).with_name("requirements.txt")), + ], + check=True, + cwd=cwd, + ) + after = json.loads( + subprocess.check_output([python, "-I", "-c", PACKAGES], cwd=cwd) + ) + changed = { + name: (version, after.get(name)) + for name, version in before.items() + if after.get(name) != version + } + if changed: + raise RuntimeError( + f"Benchmark tooling changed locked SDK dependencies: {changed}" + ) + subprocess.run( + [uv, "pip", "check", "--python", str(python)], check=True, cwd=cwd + ) + return python + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("target", type=Path) + parser.add_argument("--python", default="3.12.12") + parser.add_argument("--uv", default="uv") + args = parser.parse_args() + print(prepare(args.target, args.python, args.uv)) diff --git a/benchmarks/requirements.in b/benchmarks/requirements.in new file mode 100644 index 000000000..05ce099b7 --- /dev/null +++ b/benchmarks/requirements.in @@ -0,0 +1,3 @@ +# Benchmark-only instrumentation. Regenerate requirements.txt with uv pip compile. +memray==1.20.0 +psutil==7.2.2 diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 000000000..0d770345f --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,291 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile benchmarks/requirements.in --python-version 3.12 --universal --generate-hashes --exclude-newer 7 days --output-file benchmarks/requirements.txt +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via memray +linkify-it-py==2.2.0 \ + --hash=sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443 \ + --hash=sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee + # via markdown-it-py +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via + # mdit-py-plugins + # rich + # textual +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +mdit-py-plugins==0.6.1 \ + --hash=sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d \ + --hash=sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0 + # via textual +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +memray==1.20.0 \ + --hash=sha256:0601310f1be3fd5012d888fa73354c204d7eb85cd42a89bd91160d3b8a1ada9e \ + --hash=sha256:08b4d1e17219caabc2a01aa17e8e602b9db67a6aad3ba88b134dfc22bace420d \ + --hash=sha256:0c18705b70d20161616d3fc9d0b0c7796211b3ab16499dfb29c8f3ab7ece46c2 \ + --hash=sha256:0f2b2e3640bcff1d2e20f431fb93d17ac03236b28d53087d5e16c87139b4865e \ + --hash=sha256:1a4735275b385d6a962f4123facfe98ab306b39b084ba10ecda4d1df930d1990 \ + --hash=sha256:25e4a802488d54030c30a6bdb969fd79f701228db35e6a834eac75259c69018b \ + --hash=sha256:27dc015d1bfb9e43dc0030b412ba435ccff89cd5b7b28d7da9c8e630e7cfe3ec \ + --hash=sha256:2a6d63a3df7eb96809b82cbd9445033ec1d51574be0f9b3eddf4d64632af7162 \ + --hash=sha256:2f51218744259983ad0ca66366ca2ff80158513132ca9bd835b1874a453199dd \ + --hash=sha256:3270aad45dfa0bc7d32bb86b352aefabee765a8658aace8fb9dc8112a07f2e3c \ + --hash=sha256:359824cc26c9a208e83ab058850e9bf16592d7928307e8c2dc6c7b55e6b6cfa6 \ + --hash=sha256:37bc107afee942f162d30160dfd7f28c2b989251c574570f2f920f5bbbb323e4 \ + --hash=sha256:3c3aec7a4d0b0d0eb55d72e815a492ff060a3a46b014f6ae216ae68a006ea304 \ + --hash=sha256:3d40d606d0f1d248bb5a00701de84d8faab8ec5b6b383305a36f14c1849a7125 \ + --hash=sha256:420bc47528fedf1a92832290c7a7b8a940c2e30bd9337a85811da18a27e07220 \ + --hash=sha256:4d55e9627b6d6a868aa45995a965f3396c56ac7df5bc567b5293068412049557 \ + --hash=sha256:53231e49d55b10ac10ac8739a2592230f0794320892756f957e0026f99f38618 \ + --hash=sha256:546a60027fc9c8a5fbeea62dce45e830d78769a45192098469ee39b2d75c97fc \ + --hash=sha256:5588174a24ac150100faa7ee60b535b0f09eae82bf672133443425a00704b769 \ + --hash=sha256:57981abd8c526d4375c7f640a9d27d5e319b888e79b51688a8fae2f947fe8294 \ + --hash=sha256:59e44042f2495698f6a17fe0a831051ea515242e567f08457ee8546283372993 \ + --hash=sha256:5dd0d4220b607d85aaf6a7275d032915ad43a170b6ae9c73943952d39c224b9b \ + --hash=sha256:5e76806fa8e0fdf0c83285afac2b0a661a48f9ce03d746638fe2e43433a4e67c \ + --hash=sha256:5edcfdd7071628dbcfe5334875cb1d615e26f39a613a2cc2767c193a544b91de \ + --hash=sha256:60f3bf8762adc15a2214f44ad631cddb9e4fa4f4a0dda6aa0ccac6ea71239283 \ + --hash=sha256:610643cfa186f7958a476e5589f6e8a1b0b7b3008ab10250a0f02575a692ef1d \ + --hash=sha256:68dca0caa0b1d1b8474ee54efb98eeb33f548d28bb246113caac45125da7b9c3 \ + --hash=sha256:70010acd3d07dbf41aaca0787de3c81c5d8e22e07a6c0fdf04d66c4a376c0193 \ + --hash=sha256:734a082db18947f8afa609b2b4daca20a8b45100a3e6b6c0e4321d59ed781288 \ + --hash=sha256:7bcd5094cbb9bd1d11a5dfcc523daaa99f743e645a8b0b0805beaa2bbddbf356 \ + --hash=sha256:7e106a6dfd3823194a5ecd0b147ea7cffae900e422402cfac9ff4b2c774b4695 \ + --hash=sha256:8001648aeabd15235337b907a61bc8f26483633057dc987199f50a8977a72d7e \ + --hash=sha256:83cfa504ed4e1061b85af4b3c40228616a852851ebf5968ff1c3d85e363f8d24 \ + --hash=sha256:83eeeadb3a14ecd8180fe53fe7f0287a84d4146f7fdca986291a5fc1f4eeee97 \ + --hash=sha256:90fc0ebfc6a264fe3b77d5c263678eaf4a47a0f4288e1b6a04ae3439743cd4be \ + --hash=sha256:97539fd71c565c55e208df9ea28ac158cd156b8c33564853685688595c5a991b \ + --hash=sha256:996db26e0451301ed935d357854bf84ff1e41bc43580901bf37528a09a8dac3c \ + --hash=sha256:9b21b9cb85699f9b8df664f6c1633e2a9ce9072e8a7ca15700075e4111b001ed \ + --hash=sha256:9b33c58a7bb6b04d9767249924c0d342a9c33ec3526aeb52f95a0185b0f30aca \ + --hash=sha256:9c80ecc9259acd888647a385a2f429f453f7ba5c0b1af363babd1ad8a1210da6 \ + --hash=sha256:9d0c12433fde039a594b6bc254758c236db5532a4ee2ed758b9ec73f8fc78f62 \ + --hash=sha256:a5fc5251b02d99a98eb5874cabfcb74b492ab28e1d5c8b55d481ba0d5e273bf0 \ + --hash=sha256:a6b79c356f7caeed51cbd25dee85485b6210803199f769da8bc3d061bd0e0ada \ + --hash=sha256:a9e3db4952fe56d74036a4e1bcafd6c8c257390021f752d932394fbf3b389a67 \ + --hash=sha256:ab77856a592b04586ffb4a87e26d4de440ee3aba6facbaeabdb20855950fdebc \ + --hash=sha256:ac9a5dd9bef7f44331d98174625167bf0a8f61785ed05440d99f581c93790acf \ + --hash=sha256:b40bf09977f84afe815cca1f34f8a29040a9e757b55467d513ecb9262e6bd809 \ + --hash=sha256:b41289728ec76b48e83a1af9e1e45e997c4e4b692bfbdeb07cfd814bccb24c40 \ + --hash=sha256:b71fe846cbabce7f33017ac94ba2f157b39e22dbc4f723aad0407dd38520f855 \ + --hash=sha256:b9ec82a31783a6e468135a6f8a7a996fbc31530a5b725faffb5a8be7e2431a2b \ + --hash=sha256:bab7e524faeb76c607d364ed341592cadc1e39beb634da5cf58f77f9a0f68001 \ + --hash=sha256:c19934e6b713dbbf35f15d3c84f289e048be613e85039913a500393f10272b9f \ + --hash=sha256:c8c6ee6c3df06abf2d4230d0f2c0440e91e0c8fd51c1fe9b32599485254d1f56 \ + --hash=sha256:cdf08e64a9bcb598ae76ca467273b580b4658de49b73bd78dd8345a736d78816 \ + --hash=sha256:ce1f1d900948d57d7db5b5d8d81f4ebfcb798eff674493503ce5bfaafcea84dc \ + --hash=sha256:d1503ac18928d2493662af95935b9ccaadbf9f2579d86f6757304c23291a077b \ + --hash=sha256:d2f3f681ce8acd713a7216d6c362387e70122527e741685e82f85cfdf6aedf0b \ + --hash=sha256:d92692fb5266aca4322e2001e18810207b170dfc7ce2e8f0570c7ee181574063 \ + --hash=sha256:e073ef4685c7f7c2e9c5dabcff8ff46cae66daf6aeaea4d017239b2940b4ca82 \ + --hash=sha256:e430f8f8533d17bbf45ad0e0c2f16c0e0b767729234771b06745670b61ea5f26 \ + --hash=sha256:e64121d1b3ca248e0af39df09593bd2b04779ff8ee5261ea9b46ae5e47bc1c7e \ + --hash=sha256:f5d6980770a2d5d1a4f3e8d04d6f92309119671c1f830959a8bea868fab0b01f \ + --hash=sha256:f6653ac2fbd49e66e2883575fa8c5dde127f4699205415d1170a8cf7d1db1c86 \ + --hash=sha256:ffd3c7c53419465922be92654c17ebff577708fa1f74168dbef9eb5efa6bcb02 + # via -r benchmarks/requirements.in +platformdirs==4.13.5 \ + --hash=sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0 \ + --hash=sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e + # via textual +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 + # via -r benchmarks/requirements.in +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # rich + # textual +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via + # memray + # textual +textual==8.2.8 \ + --hash=sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a \ + --hash=sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b + # via memray +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via textual diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 000000000..d7fb14f91 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,395 @@ +"""Run the head-side suite against two independently installed SDK revisions.""" + +import argparse +import json +import math +import os +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from pathlib import Path + +import psutil +from fixtures import USERS, definitions, digest, workloads +from identity import source_digest +from measurement import classify_exit +from server import BenchmarkServer + +HERE = Path(__file__).resolve().parent +WORKER = HERE / "worker.py" +TIMEOUT = 120 +MAX_WORKER_TIMEOUTS = 2 +RSS_LIMIT = 512 * 1024 * 1024 + + +def worker_env(): + env = { + key: value for key, value in os.environ.items() if not key.startswith("PYTHON") + } + env.update( + PYTHONHASHSEED="0", + PYTHONNOUSERSITE="1", + PYTHONDONTWRITEBYTECODE="1", + POSTHOG_CAPTURE_MODE="v0", + ) + return env + + +def run_worker(python, spec, measurement_pass, timeout=TIMEOUT, rss_limit=RSS_LIMIT): + with ( + tempfile.TemporaryDirectory(prefix="posthog-benchmark-worker-") as directory, + BenchmarkServer(spec) as server, + ): + output = Path(directory) / "result.json" + command = [ + str(python), + "-B", + str(WORKER), + json.dumps({**spec, "pass": measurement_pass}), + server.url, + str(output), + ] + with subprocess.Popen( + command, + cwd=directory, + env={**worker_env(), "TMPDIR": directory}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as process: + logs = {"stdout": bytearray(), "stderr": bytearray()} + + def read_log(stream, target): + while chunk := stream.read(4096): + target.extend(chunk) + del target[:-65536] + + readers = [ + threading.Thread( + target=read_log, + args=(getattr(process, name), logs[name]), + daemon=True, + ) + for name in logs + ] + for reader in readers: + reader.start() + observed = psutil.Process(process.pid) + start = time.monotonic() + timed_out = limit_exceeded = False + while process.poll() is None: + try: + limit_exceeded = observed.memory_info().rss > rss_limit + except psutil.NoSuchProcess: + pass + timed_out = time.monotonic() - start > timeout + if timed_out or limit_exceeded: + process.kill() + break + time.sleep(0.02) + process.wait() + for reader in readers: + reader.join() + diagnostic = { + name: data.decode("utf-8", errors="replace") + for name, data in logs.items() + } + if ( + process.returncode == 0 + and output.exists() + and not timed_out + and not limit_exceeded + ): + if output.stat().st_size > 2 * 1024 * 1024: + raise ValueError("Worker result exceeds size limit") + return {"status": "ok", **json.loads(output.read_text()), **diagnostic} + return { + "status": classify_exit( + timed_out=timed_out, + rss_limit_exceeded=limit_exceeded, + stderr=diagnostic["stderr"], + ), + "code": process.returncode, + "pass": measurement_pass, + "rss_limit_exceeded": limit_exceeded, + "error": "Worker exceeded RSS limit" + if limit_exceeded + else "Worker did not complete", + **diagnostic, + } + + +def git(directory, *args): + return subprocess.check_output( + ["git", "-C", str(directory), *args], text=True + ).strip() + + +def document_for(directory, role, pair_id, suite): + python = directory / ".venv-benchmark/bin/python" + with tempfile.TemporaryDirectory(prefix="posthog-benchmark-identity-") as cwd: + execution = json.loads( + subprocess.check_output( + [python, "-B", WORKER, "--describe"], + cwd=cwd, + env=worker_env(), + text=True, + ) + ) + execution.update( + { + "pair_id": pair_id, + "runtime_flags": ["-B"], + "python_hash_seed": 0, + "bytecode_policy": "uv precompiles both environments; workers do not write bytecode", + "runner_class": "github-hosted-or-self-hosted" + if os.getenv("RUNNER_NAME") + else "local", + "runner_image": os.getenv("ImageVersion"), + "dependency_lock_digest": digest((directory / "uv.lock").read_bytes()), + "resource_limits": { + "process_rss_limit_bytes": RSS_LIMIT, + "worker_timeout_ms": TIMEOUT * 1000, + "max_worker_timeouts": MAX_WORKER_TIMEOUTS, + }, + } + ) + return { + "schema_version": 1, + "sdk": "posthog-python", + "revision": { + "role": role, + "commit": git(directory, "rev-parse", "HEAD"), + "dirty": bool( + git( + directory, + "status", + "--porcelain", + "--", + ".", + ":(exclude).venv-benchmark", + ) + ), + }, + "suite": suite, + "execution": execution, + "benchmarks": [], + } + + +def append_sample(benchmark, raw, repetition): + benchmark["runs"].append({"repetition": repetition, **raw}) + if raw["status"] != "ok": + benchmark["status"] = raw["status"] + return + + def add(name, unit, method, value): + if not math.isfinite(value) or value < 0: + raise ValueError(f"Invalid {name}: {value}") + metric = next((m for m in benchmark["metrics"] if m["id"] == name), None) + if metric is None: + metric = { + "id": name, + "unit": unit, + "method": method, + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [], + } + benchmark["metrics"].append(metric) + metric["samples"].append( + { + "repetition": repetition, + "value": value, + "operations": raw["operations"], + **({"elapsed_ns": raw["elapsed_ns"]} if "elapsed_ns" in raw else {}), + } + ) + + def unavailable(name, unit, reason): + if not any(m["id"] == name for m in benchmark["metrics"]): + benchmark["metrics"].append( + {"id": name, "unit": unit, "status": "unsupported", "reason": reason} + ) + + if raw["pass"] == "timing": + add( + "elapsed_time_per_op", + "ns/op", + "time.perf_counter_ns", + raw["elapsed_time_per_op"], + ) + if "process_cpu_time_per_op" in raw: + add( + "process_cpu_time_per_op", + "ns/op", + "time.process_time_ns", + raw["process_cpu_time_per_op"], + ) + else: + add( + "caller_thread_cpu_time_per_op", + "ns/op", + "time.thread_time_ns", + raw["caller_thread_cpu_time_per_op"], + ) + for name, unit in ( + ("allocated_bytes_per_op", "bytes/op"), + ("allocations_per_op", "count/op"), + ): + unavailable( + name, + unit, + "Background preparation has no public completion boundary before flush; see capture.drain for full-pipeline allocations.", + ) + elif raw["pass"] == "allocation": + add( + "allocated_bytes_per_op", + "bytes/op", + "memray.python_allocator_requests", + raw["allocated_bytes"] / raw["operations"], + ) + add( + "allocations_per_op", + "count/op", + "memray.python_allocator_requests", + raw["allocation_calls"] / raw["operations"], + ) + elif raw["pass"] == "memory": + add( + "idle_process_rss", + "bytes", + "psutil.Process.memory_info.rss", + raw["before"]["rss"], + ) + add( + "observed_workload_peak_rss", + "bytes", + "psutil.Process.memory_info.rss.operation_checkpoints", + raw["observed_peak_rss"], + ) + add( + "process_lifetime_peak_rss", + "bytes", + "resource.getrusage.ru_maxrss", + raw["process_lifetime_peak_rss"], + ) + add( + "post_settling_process_rss", + "bytes", + "psutil.Process.memory_info.rss", + raw["after_settling"]["rss"], + ) + unavailable( + "post_settling_heap_used", + "bytes", + "CPython has no uninstrumented total managed-heap byte counter; tracing would distort this RSS pass.", + ) + + +def main(args=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--head", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repetitions", type=int, default=6, choices=range(1, 31)) + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--filter", default="") + options = parser.parse_args(args) + specs = [ + spec + for spec in workloads(options.smoke) + if spec["id"].startswith(options.filter) + ] + if not specs: + parser.error("No matching workloads") + pair_id = str(uuid.uuid4()) + suite = { + "id": "python-sdk-performance", + "version": 1, + "source_commit": git(HERE, "rev-parse", "HEAD"), + "source_digest": source_digest(HERE), + "profile": "smoke" if options.smoke else "full", + "repetitions": options.repetitions, + "allocation_tracking": { + "trace_python_allocators": True, + "file_format": "ALL_ALLOCATIONS", + "scope": "PYMALLOC_MALLOC/CALLOC/REALLOC requests; all worker threads", + }, + "transport": "loopback HTTP receiver in coordinator process; uncompressed v0", + "settling": "two gc.collect calls separated by sleep(0)", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup", + "capture": { + "flush_at": 100000, + "flush_interval_seconds": 3600, + "consumer_threads": 1, + "drain_every": 100, + }, + "flag_poll_interval_seconds": 3600, + } + directories = {"base": options.base.resolve(), "head": options.head.resolve()} + documents = { + role: document_for(directory, role, pair_id, suite) + for role, directory in directories.items() + } + for key in ("runtime_version", "measurement_tools", "os", "arch"): + if documents["base"]["execution"][key] != documents["head"]["execution"][key]: + raise ValueError(f"Base/head {key} differ") + options.output.mkdir(parents=True, exist_ok=True) + + def save(): + for role, document in documents.items(): + temporary = options.output / f"{role}.json.tmp" + temporary.write_text(json.dumps(document, indent=2, allow_nan=False) + "\n") + temporary.replace(options.output / f"{role}.json") + + failures = 0 + timeouts = 0 + for spec in specs: + records = {} + for role in directories: + records[role] = { + "id": spec["id"], + "parameters": spec, + "fixture_digest": digest( + { + "spec": spec, + "definitions": definitions(spec["flag_count"]) + if "flag_count" in spec + else None, + "users": USERS, + } + ), + "status": "ok", + "metrics": [], + "runs": [], + } + documents[role]["benchmarks"].append(records[role]) + for repetition in range(options.repetitions): + for measurement_pass in spec["passes"]: + for role in ["head", "base"] if repetition % 2 else ["base", "head"]: + raw = run_worker( + directories[role] / ".venv-benchmark/bin/python", + spec, + measurement_pass, + ) + append_sample(records[role], raw, repetition) + failures += raw["status"] != "ok" + timeouts += raw["status"] == "timeout" + print( + f"{spec['id']} {measurement_pass} {repetition + 1}/{options.repetitions} {role}: {raw['status']}", + file=sys.stderr, + ) + save() + if timeouts >= MAX_WORKER_TIMEOUTS: + return 1 + if any(record["status"] != "ok" for record in records.values()): + break + return int(failures > 0) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/server.py b/benchmarks/server.py new file mode 100644 index 000000000..b513791bf --- /dev/null +++ b/benchmarks/server.py @@ -0,0 +1,119 @@ +"""Bounded loopback receiver owned by the coordinator, never the measured worker. + +The SDK has no public transport injection option. HTTP preserves its normal +loading, serialization, background-consumer and flush paths. +""" + +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import urlsplit + +from fixtures import USERS, definitions + + +class BenchmarkServer: + def __init__(self, spec, batch_delay=0): + self.spec = spec + self.lock = threading.Lock() + self.payload = "x" * spec.get("payload_bytes", 0) + self.body = json.dumps(definitions(spec.get("flag_count", 0))).encode() + self.state: dict[str, Any] = { + "definitions": 0, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": {user["distinct_id"]: 0 for user in USERS}, + "unexpected": [], + } + owner = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def reply(self, status, body): + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + with owner.lock: + path = urlsplit(self.path).path.rstrip("/") + if path == "/flags/definitions": + owner.state["definitions"] += 1 + self.reply(200, owner.body) + elif path == "/__benchmark/state": + self.reply(200, json.dumps(owner.state).encode()) + else: + self.unexpected(path) + + def unexpected(self, path): + if len(owner.state["unexpected"]) < 20: + owner.state["unexpected"].append(path) + self.reply(404, b'{"error":"unexpected benchmark request"}') + + def do_POST(self): + with owner.lock: + path = urlsplit(self.path).path.rstrip("/") + try: + size = int(self.headers.get("Content-Length", "0")) + if not 0 < size <= 20 * 1024 * 1024: + raise ValueError("Invalid body length") + body = json.loads(self.rfile.read(size)) + if path == "/__benchmark/definitions": + generation = body["generation"] + if ( + not isinstance(generation, int) + or not 0 <= generation <= 1000 + ): + raise ValueError("Invalid generation") + owner.body = json.dumps( + definitions(spec["flag_count"], generation) + ).encode() + self.reply(200, b"{}") + elif path == "/batch": + if batch_delay: + time.sleep(batch_delay) + batch = body["batch"] + if not isinstance(batch, list): + raise ValueError("Invalid batch") + owner.state["requests"] += 1 + owner.state["delivered"] += len(batch) + for event in batch: + properties = event.get("properties", {}) + user = event.get("distinct_id") + if ( + event.get("event") != "benchmark" + or user not in owner.state["delivered_by_user"] + or properties.get("payload") != owner.payload + or properties.get("plan") != "pro" + ): + owner.state["invalid_events"] += 1 + else: + owner.state["delivered_by_user"][user] += 1 + self.reply(200, b'{"status":1}') + else: + self.unexpected(path) + except (ValueError, KeyError, TypeError, AttributeError): + owner.state["invalid_events"] += 1 + self.reply(400, b'{"error":"invalid benchmark body"}') + + self.http = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread( + target=self.http.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + self.url = f"http://127.0.0.1:{self.http.server_port}" + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *_): + self.http.shutdown() + self.http.server_close() + self.thread.join() diff --git a/benchmarks/test_benchmarks.py b/benchmarks/test_benchmarks.py new file mode 100644 index 000000000..b5dd19983 --- /dev/null +++ b/benchmarks/test_benchmarks.py @@ -0,0 +1,387 @@ +import gc +import json +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import patch +from urllib.error import HTTPError + +import memray +from fixtures import USERS, definitions, digest, expected_flags, workloads +from identity import environment, source_digest +from measurement import ( + allocation_totals, + classify_exit, + memory, + sample_allocations, + settle, + time_operations, +) +from prepare import prepare +from run import append_sample, main, run_worker +from server import BenchmarkServer +from workload import Workload, control + +from posthog import Posthog + +ROOT = Path(__file__).resolve().parents[1] +PYTHON = ROOT / ".venv-benchmark/bin/python" + + +class BenchmarksTest(unittest.TestCase): + def test_deterministic_and_changing_fixtures(self): + self.assertEqual(digest(definitions(1000)), digest(definitions(1000))) + self.assertNotEqual(digest(definitions(1000)), digest(definitions(1000, 1))) + self.assertTrue(expected_flags(10, USERS[0])["boolean-0"]) + self.assertFalse(expected_flags(10, USERS[0], 1)["boolean-0"]) + self.assertFalse(expected_flags(10, USERS[0], 1)["cohort-5"]) + + def test_public_entry_fixtures_and_lifecycle(self): + for spec in workloads(smoke=True): + with self.subTest(workload=spec["id"]), BenchmarkServer(spec) as server: + workload = Workload(spec, server.url) + try: + workload.run() + counts = workload.verify() + self.assertEqual(counts["unexpected"], []) + if spec["mode"] == "refresh": + self.assertEqual(counts["definitions"], spec["operations"] + 1) + if spec["mode"] in ("capture", "drain"): + self.assertEqual(counts["delivered"], spec["operations"]) + finally: + workload.close() + self.assertFalse( + any(thread.is_alive() for thread in workload.client.consumers) + ) + if workload.client.poller: + self.assertFalse(workload.client.poller.is_alive()) + + def test_remote_fallback_is_not_a_successful_fake_evaluation(self): + with BenchmarkServer(workloads(True)[0]) as server: + with self.assertRaises(HTTPError): + control(server.url, "/flags/?v=2", {}) + self.assertEqual( + control(server.url, "/__benchmark/state")["unexpected"], ["/flags"] + ) + + def test_capture_checks_payload_and_identity_distribution(self): + for mutation in ("payload", "identity"): + + class Corrupted(Posthog): + def capture(self, event, _mutation=mutation, **kwargs): + if _mutation == "payload": + kwargs["properties"] = {**kwargs["properties"], "payload": ""} + else: + kwargs["distinct_id"] = USERS[0]["distinct_id"] + return super().capture(event, **kwargs) + + spec = next(s for s in workloads(True) if s["id"] == "capture.enqueue.4096") + with self.subTest(mutation=mutation), BenchmarkServer(spec) as server: + workload = Workload(spec, server.url, client_class=Corrupted) + try: + workload.run() + with self.assertRaises(AssertionError): + workload.verify() + self.assertEqual(workload.state()["delivered"], spec["operations"]) + finally: + workload.close() + + def test_delayed_draining_finishes_inside_the_measurement(self): + calls = [] + + class ShortDeadline(Posthog): + def flush(self, timeout_seconds=10): + calls.append(timeout_seconds) + # Make finite-deadline misuse observable without a five-second test. + return super().flush( + timeout_seconds=None if timeout_seconds is None else 0.001 + ) + + spec = { + **next(s for s in workloads(True) if s["id"] == "capture.drain.32"), + "operations": 100, + } + with BenchmarkServer(spec, batch_delay=0.05) as server: + workload = Workload(spec, server.url, client_class=ShortDeadline) + try: + measured = time_operations(workload.run, spec["operations"]) + self.assertGreaterEqual(measured["elapsed_ns"], 50_000_000) + self.assertEqual(calls, [None]) + self.assertEqual(workload.verify()["delivered"], 100) + self.assertEqual( + calls, + [None], + "Verification must not rescue an unfinished measured drain", + ) + finally: + workload.close() + + def test_preparation_forces_a_source_rebuild_and_preserves_runtime_versions(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("prepare.subprocess.run") as run, + patch( + "prepare.subprocess.check_output", + return_value=b'{"posthog":"1.0","requests":"2.0"}', + ), + ): + prepare(Path(directory), "3.12.12", "uv") + command = run.call_args_list[0].args[0] + self.assertEqual( + command[command.index("--reinstall-package") + 1], "posthog" + ) + self.assertIn("--frozen", command) + self.assertIn("--no-editable", command) + self.assertIn("--compile-bytecode", command) + self.assertIn("--compile-bytecode", run.call_args_list[1].args[0]) + with ( + tempfile.TemporaryDirectory() as directory, + patch("prepare.subprocess.run"), + patch( + "prepare.subprocess.check_output", + side_effect=[b'{"requests":"2.0"}', b'{"requests":"3.0"}'], + ), + self.assertRaisesRegex(RuntimeError, "changed locked SDK dependencies"), + ): + prepare(Path(directory), "3.12.12", "uv") + + def test_allocation_churn_includes_freed_small_python_allocations(self): + def churn(): + for _ in range(20000): + value = bytearray(200) + del value + gc.collect() + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "default.bin" + with memray.Tracker(path): + churn() + self.assertEqual(allocation_totals(path)["allocation_calls"], 0) + measured = sample_allocations(churn) + # Includes allocator requests even when objects have already been released. + self.assertGreater(measured["allocated_bytes"], 20000 * 200) + self.assertGreater(measured["allocation_calls"], 40000) + self.assertLess(measured["allocation_calls"], 200000) + + def test_cpu_and_memory_controls(self): + def busy(): + start = time.process_time_ns() + while time.process_time_ns() - start < 25_000_000: + pass + + measured = time_operations(busy, 1) + self.assertGreaterEqual(measured["process_cpu_time_per_op"], 25_000_000) + settle() + before = memory()["rss"] + retained = bytearray(64 * 1024 * 1024) + for i in range(0, len(retained), 4096): + retained[i] = 1 + settle() + self.assertGreater(memory()["rss"] - before, 32 * 1024 * 1024) + del retained + settle() + + def test_refresh_checkpoints_detect_retained_state(self): + class Retaining(Posthog): + def __init__(self, *args, **kwargs): + self.retained = [] + super().__init__(*args, **kwargs) + + def load_feature_flags(self): + result = super().load_feature_flags() + block = bytearray(8 * 1024 * 1024) + for i in range(0, len(block), 4096): + block[i] = 1 + self.retained.append(block) + return result + + spec = { + **next(s for s in workloads(True) if s["mode"] == "refresh"), + "operations": 4, + } + with BenchmarkServer(spec) as server: + workload = Workload(spec, server.url, client_class=Retaining) + try: + settle() + before = memory()["rss"] + checkpoints = [] + + def checkpoint(completed): + settle() + checkpoints.append((completed, memory()["rss"])) + + workload.run(checkpoint=checkpoint) + workload.verify() + self.assertEqual([point[0] for point in checkpoints], [1, 2, 3, 4]) + self.assertGreater(checkpoints[-1][1] - before, 24 * 1024 * 1024) + self.assertGreater( + checkpoints[-1][1] - checkpoints[0][1], 16 * 1024 * 1024 + ) + finally: + workload.close() + + def test_public_entry_cpu_and_allocation_regressions(self): + class Expensive(Posthog): + def get_feature_flag(self, *args, **kwargs): + result = super().get_feature_flag(*args, **kwargs) + start = time.thread_time_ns() + while time.thread_time_ns() - start < 500_000: + pass + allocation = bytearray(256 * 1024) + allocation[0] = 1 + return result + + spec = workloads(True)[0] + results = [] + for cls in (Posthog, Expensive): + with BenchmarkServer(spec) as server: + workload = Workload(spec, server.url, client_class=cls) + try: + workload.run() + timing = time_operations(workload.run, spec["operations"]) + allocation = sample_allocations(workload.run) + workload.verify() + results.append((timing, allocation)) + finally: + workload.close() + self.assertGreater( + results[1][0]["process_cpu_time_per_op"], + results[0][0]["process_cpu_time_per_op"] * 3, + ) + self.assertGreater( + results[1][1]["allocated_bytes"], results[0][1]["allocated_bytes"] * 3 + ) + + def test_worker_pass_isolation_and_installed_package(self): + raw = run_worker(PYTHON, workloads(True)[0], "allocation") + self.assertEqual(raw["status"], "ok", raw["stderr"]) + self.assertNotIn("elapsed_time_per_op", raw) + self.assertTrue(Path(raw["entry"]).is_relative_to(ROOT / ".venv-benchmark")) + result: dict[str, Any] = {"status": "ok", "metrics": [], "runs": []} + append_sample(result, raw, 0) + self.assertEqual(result["metrics"][1]["id"], "allocations_per_op") + self.assertEqual(result["metrics"][1]["quality"], "measured") + + def test_worker_failures_are_not_zero_cost_samples(self): + raw = run_worker(PYTHON, {**workloads(True)[0], "flag_index": 100}, "timing") + self.assertEqual(raw["status"], "error") + result: dict[str, Any] = {"status": "ok", "metrics": [], "runs": []} + append_sample(result, raw, 0) + self.assertEqual(result["status"], "error") + self.assertEqual(result["metrics"], []) + self.assertEqual( + run_worker(PYTHON, workloads(True)[0], "timing", timeout=0.01)["status"], + "timeout", + ) + limited = run_worker(PYTHON, workloads(True)[0], "timing", rss_limit=1) + self.assertTrue(limited["rss_limit_exceeded"]) + self.assertEqual(limited["status"], "error") + + def test_killed_worker_temporary_files_are_removed_by_the_coordinator(self): + with tempfile.TemporaryDirectory() as directory: + worker = Path(directory) / "worker.py" + worker.write_text( + "import tempfile, time\nprint(tempfile.mkdtemp(prefix='profile-'), flush=True)\ntime.sleep(10)\n" + ) + with patch("run.WORKER", worker): + raw = run_worker(PYTHON, workloads(True)[0], "allocation", timeout=0.5) + self.assertEqual(raw["status"], "timeout") + profile = Path(raw["stdout"].strip()) + self.assertTrue(profile.name.startswith("profile-")) + self.assertTrue(profile.parent.name.startswith("posthog-benchmark-worker-")) + self.assertFalse(profile.parent.exists()) + + def test_repeated_timeouts_stop_the_suite_with_partial_results(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch( + "run.run_worker", return_value={"status": "timeout", "pass": "timing"} + ) as worker, + ): + code = main( + [ + "--base", + str(ROOT), + "--head", + str(ROOT), + "--output", + directory, + "--smoke", + ] + ) + self.assertEqual(code, 1) + self.assertEqual(worker.call_count, 2) + for role in ("base", "head"): + document = json.loads((Path(directory) / f"{role}.json").read_text()) + self.assertEqual(len(document["benchmarks"]), 1) + self.assertEqual(document["benchmarks"][0]["status"], "timeout") + self.assertEqual(document["benchmarks"][0]["metrics"], []) + self.assertEqual(len(document["benchmarks"][0]["runs"]), 1) + + def test_oom_classification_requires_evidence(self): + self.assertEqual(classify_exit(stderr="Killed"), "error") + self.assertEqual(classify_exit(stderr="Traceback...\nMemoryError\n"), "oom") + self.assertEqual(classify_exit(rss_limit_exceeded=True), "error") + self.assertEqual(classify_exit(timed_out=True), "timeout") + + def test_package_identity_and_source_fingerprints(self): + self.assertTrue(Path(environment()["entry"]).is_relative_to(Path(sys.prefix))) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "module.py" + path.write_text("VALUE = 1") + before = source_digest(directory) + path.write_text("VALUE = 2") + self.assertNotEqual(source_digest(directory), before) + + def test_paired_documents(self): + with tempfile.TemporaryDirectory() as directory: + self.assertEqual( + main( + [ + "--base", + str(ROOT), + "--head", + str(ROOT), + "--output", + directory, + "--smoke", + "--repetitions", + "2", + "--filter", + "flags.single.boolean", + ] + ), + 0, + ) + base = json.loads((Path(directory) / "base.json").read_text()) + head = json.loads((Path(directory) / "head.json").read_text()) + self.assertEqual(base["execution"]["pair_id"], head["execution"]["pair_id"]) + self.assertEqual( + base["execution"]["build_digest"], head["execution"]["build_digest"] + ) + self.assertEqual( + base["benchmarks"][0]["fixture_digest"], + head["benchmarks"][0]["fixture_digest"], + ) + self.assertEqual(base["revision"]["role"], "base") + self.assertEqual(head["revision"]["role"], "head") + for metric in head["benchmarks"][0]["metrics"]: + self.assertEqual(len(metric["samples"]), 2) + + def test_worker_describes_the_installed_distribution(self): + # Validate the actual recorded installation, not source-tree import resolution. + output = subprocess.check_output( + [PYTHON, "-B", str(ROOT / "benchmarks/worker.py"), "--describe"], + cwd=tempfile.gettempdir(), + ) + self.assertEqual( + json.loads(output)["build_digest"], environment()["build_digest"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/worker.py b/benchmarks/worker.py new file mode 100644 index 000000000..e8b80f235 --- /dev/null +++ b/benchmarks/worker.py @@ -0,0 +1,96 @@ +"""One isolated revision/workload/measurement-pass execution.""" + +import json +import sys +import time +from pathlib import Path + +from measurement import ( + lifetime_peak_rss, + memory, + sample_allocations, + settle, + time_operations, +) +from workload import Workload + + +def execute(spec, host): + assert sys.flags.optimize == 0, "Benchmark correctness checks require assertions" + import posthog + + with_installed_sdk = Path(posthog.__file__).resolve() + assert with_installed_sdk.is_relative_to(Path(sys.prefix)), ( + "Expected the target's installed wheel" + ) + workload = Workload(spec, host) + try: + warmup = ( + 2 + if spec["mode"] == "refresh" + else min(spec["operations"], 32 if spec.get("flag_count") == 1000 else 160) + ) + workload.run(warmup) + workload.verify() + settle() + operations = ( + spec.get("allocation_operations", spec["operations"]) + if spec["pass"] == "allocation" + else spec["operations"] + ) + result = { + "pass": spec["pass"], + "operations": operations, + "warmup_operations": warmup, + "entry": str(with_installed_sdk), + } + if spec["pass"] == "timing": + result.update(time_operations(workload.run, spec["operations"])) + if spec["mode"] == "capture": + # Foreground enqueue does not have a public background-preparation completion signal. + # Process-wide CPU/allocations for full delivery belong to the drain workload. + del result["process_cpu_time_per_op"] + else: + del result["caller_thread_cpu_time_per_op"] + elif spec["pass"] == "allocation": + result.update(sample_allocations(lambda: workload.run(operations))) + elif spec["pass"] == "memory": + result["before"] = memory() + result["checkpoints"] = [] + observed = result["before"]["rss"] + + def checkpoint(completed): + nonlocal observed + observed = max(observed, memory()["rss"]) + if spec["mode"] == "refresh" and ( + completed % 5 == 0 or completed == spec["operations"] + ): + settle() + result["checkpoints"].append({"operations": completed, **memory()}) + + start = time.perf_counter_ns() + workload.run(checkpoint=checkpoint) + result["elapsed_ns"] = time.perf_counter_ns() - start + result["after_work"] = memory() + result["observed_peak_rss"] = max(observed, result["after_work"]["rss"]) + result["process_lifetime_peak_rss"] = lifetime_peak_rss() + settle() + result["after_settling"] = memory() + else: + raise ValueError(f"Unknown pass {spec['pass']}") + result["validation"] = workload.verify() + return result + finally: + workload.close() + + +if __name__ == "__main__": + if sys.argv[1:] == ["--describe"]: + from identity import environment + + print(json.dumps(environment())) + else: + spec, host, output = sys.argv[1:] + Path(output).write_text( + json.dumps(execute(json.loads(spec), host)), encoding="utf-8" + ) diff --git a/benchmarks/workload.py b/benchmarks/workload.py new file mode 100644 index 000000000..069824c77 --- /dev/null +++ b/benchmarks/workload.py @@ -0,0 +1,171 @@ +import json +from concurrent.futures import ThreadPoolExecutor +from urllib.request import Request, urlopen + +from fixtures import CASES, USERS, expected_flags, expected_value + + +def control(host, path, data=None): + body = None if data is None else json.dumps(data).encode() + request = Request( + host + path, data=body, headers={"Content-Type": "application/json"} + ) + with urlopen(request, timeout=5) as response: + return json.load(response) + + +class Workload: + def __init__(self, spec, host, client_class=None): + if client_class is None: + from posthog import Posthog + + client_class = Posthog + self.spec = spec + self.host = host + self.generation = 0 + self.captured = 0 + self.captured_by_user = {user["distinct_id"]: 0 for user in USERS} + self.errors: list[str] = [] + self.checks = 0 + self.flags = spec["mode"] in ("single", "all", "refresh") + self.client = client_class( + "benchmark-project-key", + host=host, + max_queue_size=100001, + flush_at=100000, + flush_interval=3600, + poll_interval=3600, + timeout=5, + max_retries=0, + feature_flags_request_max_retries=0, + gzip=False, + capture_mode="v0", + thread=1, + on_error=lambda error, _batch: self.errors.append(str(error)), + **({"personal_api_key": "benchmark-local-only"} if self.flags else {}), + ) + self.pool = ( + ThreadPoolExecutor(max_workers=spec["concurrency"]) + if spec["concurrency"] > 1 + else None + ) + self.events = [ + { + "event": "benchmark", + "distinct_id": user["distinct_id"], + "properties": { + "payload": "x" * spec.get("payload_bytes", 0), + "plan": "pro", + }, + } + for user in USERS + ] + try: + if self.flags: + self.check_flags() + assert self.state()["definitions"] > 0 + self.verify() + except BaseException: + self.close() + raise + + def state(self): + return control(self.host, "/__benchmark/state") + + def evaluate(self, index): + user = USERS[index % len(USERS)] + options = { + "person_properties": user["person_properties"], + "only_evaluate_locally": True, + } + if self.spec["mode"] == "single": + i = self.spec["flag_index"] + return self.client.get_feature_flag( + f"{CASES[i]}-{i}", + user["distinct_id"], + send_feature_flag_events=False, + **options, + ) + return self.client.get_all_flags(user["distinct_id"], **options) + + def check_flags(self): + for i, user in enumerate(USERS): + expected = ( + expected_value(self.spec["flag_index"], user, self.generation) + if self.spec["mode"] == "single" + else expected_flags(self.spec["flag_count"], user, self.generation) + ) + actual = self.evaluate(i) + assert actual == expected, ( + f"Wrong local evaluation for {self.spec['id']}, user {i}, generation {self.generation}" + ) + self.checks += 1 + + def run(self, operations=None, checkpoint=None): + operations = self.spec["operations"] if operations is None else operations + mode = self.spec["mode"] + if mode == "refresh": + for i in range(operations): + self.generation += 1 + control( + self.host, + "/__benchmark/definitions", + {"generation": self.generation}, + ) + before = self.state()["definitions"] + self.client.load_feature_flags() + assert self.state()["definitions"] == before + 1 + self.check_flags() + if checkpoint: + checkpoint(i + 1) + elif mode in ("capture", "drain"): + for i in range(operations): + event = self.events[i % len(self.events)] + assert self.client.capture(**event) is not None, ( + "Capture rejected an event" + ) + self.captured += 1 + self.captured_by_user[USERS[i % len(USERS)]["distinct_id"]] += 1 + if (i + 1) % 100 == 0 or i + 1 == operations: + if mode == "drain": + self.client.flush(timeout_seconds=None) + if checkpoint: + checkpoint(i + 1) + else: + for start in range(0, operations, self.spec["concurrency"]): + if self.pool: + # Bound pending work and discard completed results; this is GIL-constrained threading. + futures = [ + self.pool.submit(self.evaluate, i) + for i in range( + start, min(start + self.spec["concurrency"], operations) + ) + ] + for future in futures: + future.result() + else: + self.evaluate(start) + if checkpoint and start % 32 == 0: + checkpoint(min(start + self.spec["concurrency"], operations)) + + def verify(self): + if self.flags: + self.check_flags() + elif self.spec["mode"] == "capture": + self.client.flush(timeout_seconds=5) + state = self.state() + assert not self.errors, self.errors + assert not state["unexpected"], state["unexpected"] + assert state["delivered"] == self.captured, ( + "Delivered counts must match captures" + ) + assert state["invalid_events"] == 0, "Capture payload or identity changed" + assert state["delivered_by_user"] == self.captured_by_user, ( + "Capture identity distribution changed" + ) + return {**state, "captured": self.captured, "evaluations": self.checks} + + def close(self): + if self.pool: + self.pool.shutdown(wait=True) + self.client.shutdown()