diff --git a/.github/actions/benchmark-report/.gitignore b/.github/actions/benchmark-report/.gitignore new file mode 100644 index 0000000..ae4022a --- /dev/null +++ b/.github/actions/benchmark-report/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ diff --git a/.github/actions/benchmark-report/action.yml b/.github/actions/benchmark-report/action.yml new file mode 100644 index 0000000..9d6e881 --- /dev/null +++ b/.github/actions/benchmark-report/action.yml @@ -0,0 +1,82 @@ +name: SDK benchmark report +description: Validate and render paired SDK results, or publish from a verified workflow_run +inputs: + mode: + description: render or collect (offline), or publish (trusted workflow_run only) + default: render + source-status: + description: Measurement or matrix outcome for render/collect mode; publish mode uses the verified source run conclusion + default: '' + base-results: + description: Baseline result JSON path for render mode + default: '' + head-results: + description: Head result JSON path for render mode + default: '' + bundle-results: + description: Saved bundle JSON path for render mode (instead of base/head) + default: '' + shard-manifest: + description: Producer JSON manifest path for collect mode + default: '' + shard-directory: + description: Directory containing benchmark-shard-ID downloads for collect mode + default: '' + output-directory: + description: Directory for report.md and comparison.json; collect also writes bundle.json and revisions.json + default: benchmark-report + workflow-path: + description: Trusted literal producer workflow path for publish mode + default: '' + artifact-name: + description: Trusted literal producer artifact name for publish mode + default: '' + report-key: + description: Trusted stable comment namespace for publish mode + default: '' + sdk: + description: Trusted literal SDK identity for publish mode + default: '' +outputs: + report-path: + description: Absolute Markdown report path + value: ${{ steps.report.outputs.report-path }} + comparison-path: + description: Absolute comparison JSON path + value: ${{ steps.report.outputs.comparison-path }} +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: '0.13.4' + python-version: '3.12.12' + enable-cache: 'false' + - id: report + shell: bash + env: + REPORT_ACTION_ROOT: ${{ github.action_path }} + REPORT_MODE: ${{ inputs.mode }} + REPORT_SOURCE_STATUS: ${{ inputs.source-status }} + REPORT_BASE: ${{ inputs.base-results }} + REPORT_HEAD: ${{ inputs.head-results }} + REPORT_BUNDLE: ${{ inputs.bundle-results }} + REPORT_MANIFEST: ${{ inputs.shard-manifest }} + REPORT_SHARDS: ${{ inputs.shard-directory }} + REPORT_OUTPUT: ${{ inputs.output-directory }} + REPORT_WORKFLOW: ${{ inputs.workflow-path }} + REPORT_ARTIFACT: ${{ inputs.artifact-name }} + REPORT_KEY: ${{ inputs.report-key }} + REPORT_SDK: ${{ inputs.sdk }} + run: | + set -euo pipefail + root="$(cd "$REPORT_ACTION_ROOT" && pwd)" + uv sync --no-config --project "$root" --frozen --no-dev --no-install-project + uv run --no-config --project "$root" --no-sync \ + python -I "$root/benchmarks/report/cli.py" \ + --mode "$REPORT_MODE" --source-status "$REPORT_SOURCE_STATUS" \ + --base-results "$REPORT_BASE" --head-results "$REPORT_HEAD" \ + --bundle-results "$REPORT_BUNDLE" --shard-manifest "$REPORT_MANIFEST" \ + --shard-directory "$REPORT_SHARDS" \ + --output-directory "$REPORT_OUTPUT" --workflow-path "$REPORT_WORKFLOW" \ + --artifact-name "$REPORT_ARTIFACT" --report-key "$REPORT_KEY" --sdk "$REPORT_SDK" diff --git a/.github/actions/benchmark-report/benchmarks/__init__.py b/.github/actions/benchmark-report/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/benchmarks/report/__init__.py b/.github/actions/benchmark-report/benchmarks/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/benchmarks/report/bundle.py b/.github/actions/benchmark-report/benchmarks/report/bundle.py new file mode 100644 index 0000000..ca900d2 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/bundle.py @@ -0,0 +1,195 @@ +"""Bounded shard collection and independent paired comparisons; inputs are data only.""" + +import json +import os +import re +import stat +from pathlib import Path + +from .compare import compare +from .validate import MAX_DOCUMENT_BYTES, bounded_json, check_bounds, require, validate + +MAX_SHARDS = 16 +SHARD_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,63}") + + +def validate_manifest(manifest: dict) -> dict: + check_bounds(manifest) + require( + isinstance(manifest, dict) and set(manifest) == {"manifest_version", "sdk", "shards"}, + "Invalid shard manifest fields", + ) + require(type(manifest["manifest_version"]) is int and manifest["manifest_version"] == 1, "Invalid manifest version") + require(isinstance(manifest["sdk"], str) and 0 < len(manifest["sdk"]) <= 256, "Invalid manifest SDK") + shards = manifest["shards"] + require(isinstance(shards, list) and 0 < len(shards) <= MAX_SHARDS, "Invalid manifest shard count") + ids, workloads = set(), set() + for shard in shards: + require(isinstance(shard, dict) and set(shard) == {"id", "workloads"}, "Invalid manifest shard fields") + identity = shard["id"] + require(isinstance(identity, str) and SHARD_ID.fullmatch(identity), "Unsafe shard ID") + require(identity not in ids, "Duplicate manifest shard") + ids.add(identity) + members = shard["workloads"] + require(isinstance(members, list) and 0 < len(members) <= 128, "Invalid shard workload count") + for member in members: + require(isinstance(member, str) and 0 < len(member) <= 256, "Invalid manifest workload ID") + require(member not in workloads, "Duplicate manifest workload") + workloads.add(member) + return manifest + + +def validate_revisions(revisions: dict) -> dict: + require(isinstance(revisions, dict) and set(revisions) == {"target", "head", "base"}, "Invalid revisions document") + require( + all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value) for value in revisions.values()), + "Invalid revision SHA", + ) + return revisions + + +def validate_bundle(bundle: dict) -> dict: + check_bounds(bundle) + require( + isinstance(bundle, dict) and set(bundle) == {"bundle_version", "manifest", "shards"}, "Invalid bundle fields" + ) + require(type(bundle["bundle_version"]) is int and bundle["bundle_version"] == 1, "Invalid bundle version") + manifest = validate_manifest(bundle["manifest"]) + expected = {shard["id"]: set(shard["workloads"]) for shard in manifest["shards"]} + shards = bundle["shards"] + require(isinstance(shards, list) and len(shards) == len(expected), "Bundle shard count differs") + seen, pairs, revisions = set(), set(), {} + suite = None + for shard in shards: + require(isinstance(shard, dict) and set(shard) == {"id", "base", "head"}, "Invalid bundle shard fields") + identity = shard["id"] + require( + isinstance(identity, str) and identity in expected and identity not in seen, + "Unexpected or duplicate bundle shard", + ) + seen.add(identity) + for role in ("base", "head"): + doc = validate(shard[role]) + require(doc["sdk"] == manifest["sdk"], "Shard SDK differs") + require(doc["revision"]["role"] == role, "Wrong shard revision role") + require(doc["execution"].get("shard_id") == identity, "Result shard ID differs") + require({w["id"] for w in doc["benchmarks"]} == expected[identity], "Shard workload coverage differs") + if suite is None: + suite = doc["suite"] + require(doc["suite"] == suite, "Shard suite/source/profile differs") + revisions.setdefault(role, doc["revision"]) + require(doc["revision"] == revisions[role], "Shard revision identity differs") + pair_id = shard["base"]["execution"]["pair_id"] + require(pair_id == shard["head"]["execution"]["pair_id"], "Shard pair IDs differ") + require(pair_id not in pairs, "Pair ID reused across shards") + pairs.add(pair_id) + return bundle + + +def compare_bundle(bundle: dict, *, source_status: str | None = None) -> dict: + validate_bundle(bundle) + shards = {} + rows, workloads, workload_shards = [], {}, {} + for shard in bundle["shards"]: + identity = shard["id"] + comparison = compare(shard["base"], shard["head"], source_status=source_status) + shards[identity] = {key: comparison[key] for key in ("context", "compatibility", "execution_failed")} + rows.extend(dict(row, shard_id=identity) for row in comparison["rows"]) + workloads.update(comparison["workloads"]) + workload_shards.update({workload: identity for workload in comparison["workloads"]}) + rows.sort( + key=lambda r: (not r.get("execution_failed", False), r["status"] == "inconclusive", r["workload"], r["metric"]) + ) + return { + "comparison_version": 2, + "sdk": bundle["manifest"]["sdk"], + "manifest": bundle["manifest"], + "source_execution_status": source_status, + "execution_failed": any(shard["execution_failed"] for shard in shards.values()), + "shards": shards, + "workload_shards": workload_shards, + "workloads": workloads, + "rows": rows, + } + + +def read_regular(path: Path, limit: int = MAX_DOCUMENT_BYTES) -> bytes: + require(stat.S_ISREG(path.lstat().st_mode), "Input must be a regular file") + with path.open("rb") as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), "Input must be a regular file") + require(os.fstat(stream.fileno()).st_size <= limit, "Input exceeds byte limit") + data = stream.read(limit + 1) + require(len(data) <= limit, "Input exceeds byte limit") + return data + + +def load_bundle(path: str | Path) -> dict: + return validate_bundle(bounded_json(read_regular(Path(path)))) + + +def directory_members(path: Path, maximum: int) -> set[str]: + require(stat.S_ISDIR(path.lstat().st_mode), "Shard input must be a regular directory") + names = set() + with os.scandir(path) as entries: + for entry in entries: + names.add(entry.name) + require(len(names) <= maximum, "Too many shard directory members") + return names + + +def encode_bundle(bundle: dict) -> bytes: + # Compact encoding keeps the canonical document within the same input byte budget. + data = json.dumps(bundle, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode() + b"\n" + require(len(data) <= MAX_DOCUMENT_BYTES, "Bundle exceeds byte limit") + return data + + +def collect(manifest_path: str, shard_directory: str) -> tuple[dict, dict]: + manifest_bytes = read_regular(Path(manifest_path)) + manifest = validate_manifest(bounded_json(manifest_bytes)) + root = Path(shard_directory) + expected = {f"benchmark-shard-{shard['id']}" for shard in manifest["shards"]} + require(directory_members(root, MAX_SHARDS) == expected, "Missing or unexpected shard directory") + paths = [] + total = len(manifest_bytes) + # Preflight all sizes before retaining any result collection. + for shard in manifest["shards"]: + directory = root / f"benchmark-shard-{shard['id']}" + require( + directory_members(directory, 3) == {"base.json", "head.json", "revisions.json"}, + "Unexpected or missing shard member", + ) + for name in ("base.json", "head.json", "revisions.json"): + path = directory / name + info = path.lstat() + require(stat.S_ISREG(info.st_mode), "Shard member must be a regular file") + limit = 8192 if name == "revisions.json" else MAX_DOCUMENT_BYTES + require(info.st_size <= limit, "Shard member exceeds byte limit") + total += info.st_size + require(total <= MAX_DOCUMENT_BYTES, "Aggregate shard inputs exceed byte limit") + paths.append((shard["id"], name, path, limit)) + shards = {shard["id"]: {"id": shard["id"]} for shard in manifest["shards"]} + revisions = None + consumed = len(manifest_bytes) + for identity, name, path, limit in paths: + data = read_regular(path, min(limit, MAX_DOCUMENT_BYTES - consumed)) + consumed += len(data) + doc = bounded_json(data, limit=limit) + if name == "revisions.json": + validate_revisions(doc) + if revisions is None: + revisions = doc + require(doc == revisions, "Shard revisions documents differ") + else: + shards[identity][name.removesuffix(".json")] = doc + bundle = validate_bundle({"bundle_version": 1, "manifest": manifest, "shards": list(shards.values())}) + for shard in bundle["shards"]: + for role in ("base", "head"): + require( + shard[role]["revision"]["commit"] == revisions[role], "Shard revision differs from revisions document" + ) + require( + shard[role]["suite"]["source_commit"] == revisions["head"], "Shard source differs from head revision" + ) + encode_bundle(bundle) + return bundle, revisions diff --git a/.github/actions/benchmark-report/benchmarks/report/cli.py b/.github/actions/benchmark-report/benchmarks/report/cli.py new file mode 100644 index 0000000..b1b9a5a --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/cli.py @@ -0,0 +1,99 @@ +"""Offline renderer and trusted workflow_run publisher entry point.""" + +import argparse +import os +import sys +from pathlib import Path + +# Support absolute-path invocation from a composite action without installing the project. +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from benchmarks.report.bundle import collect, compare_bundle, encode_bundle, load_bundle +from benchmarks.report.compare import SOURCE_STATUSES, compare +from benchmarks.report.github import GitHub +from benchmarks.report.publish import Publisher +from benchmarks.report.render import json_text, render, write_outputs +from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, load_result, require + + +def main(argv=None, environ=None) -> int: + env = os.environ if environ is None else environ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("render", "collect", "publish"), default="render") + parser.add_argument("--source-status", choices=sorted(SOURCE_STATUSES | {""}), default="") + parser.add_argument("--base-results", default="") + parser.add_argument("--head-results", default="") + parser.add_argument("--bundle-results", default="") + parser.add_argument("--shard-manifest", default="") + parser.add_argument("--shard-directory", default="") + parser.add_argument("--output-directory", default="benchmark-report") + parser.add_argument("--workflow-path", default="") + parser.add_argument("--artifact-name", default="") + parser.add_argument("--report-key", default="") + parser.add_argument("--sdk", default="") + args = parser.parse_args(argv) + try: + if args.mode == "collect": + require(not (args.base_results or args.head_results or args.bundle_results), "Ambiguous collection inputs") + require(args.shard_manifest and args.shard_directory, "Manifest and shard directory are required") + bundle, revisions = collect(args.shard_manifest, args.shard_directory) + comparison = compare_bundle(bundle, source_status=args.source_status or None) + report = render(comparison, repository=env.get("GITHUB_REPOSITORY"), run_id=env.get("GITHUB_RUN_ID")) + directory = Path(args.output_directory) + directory.mkdir(parents=True, exist_ok=True) + (directory / "bundle.json").write_bytes(encode_bundle(bundle)) + (directory / "revisions.json").write_text(json_text(revisions), encoding="utf-8") + elif args.mode == "render": + require(not (args.shard_manifest or args.shard_directory), "Shard inputs require collect mode") + if args.bundle_results: + require(not (args.base_results or args.head_results), "Ambiguous rendering inputs") + comparison = compare_bundle(load_bundle(args.bundle_results), source_status=args.source_status or None) + else: + require(args.base_results and args.head_results, "Both result paths are required") + comparison = compare( + load_result(args.base_results), + load_result(args.head_results), + source_status=args.source_status or None, + ) + report = render(comparison, repository=env.get("GITHUB_REPOSITORY"), run_id=env.get("GITHUB_RUN_ID")) + else: + require( + not ( + args.base_results + or args.head_results + or args.bundle_results + or args.shard_manifest + or args.shard_directory + ), + "Publish accepts only verified artifact inputs", + ) + require(env.get("GITHUB_EVENT_NAME") == "workflow_run", "Publish requires a workflow_run event") + require(bool(env.get("GITHUB_EVENT_PATH")), "Missing GitHub event file") + with Path(env["GITHUB_EVENT_PATH"]).open("rb") as stream: + event = bounded_json(stream.read(MAX_DOCUMENT_BYTES + 1), text_limit=65536) + publisher = Publisher( + GitHub(env.get("GH_TOKEN", "")), + env.get("GITHUB_REPOSITORY", ""), + args.workflow_path, + args.artifact_name, + args.report_key, + args.sdk, + ) + comparison, report = publisher.publish(event, report_run_id=env.get("GITHUB_RUN_ID")) + write_outputs(args.output_directory, comparison, report, env) + return 1 if comparison.get("execution_failed") else 0 + except (InvalidResult, OSError, ValueError) as exc: + # Never print paths, API responses, raw logs or arbitrary exception text. + reason = str(exc) if isinstance(exc, InvalidResult) else "Report input or output could not be processed" + report = f"## SDK performance\n\n**Reporting failed:** {reason}.\n" + try: + write_outputs(args.output_directory, {"comparison_version": 1, "error": reason}, report, env) + except (OSError, ValueError): + pass + print(reason, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/benchmark-report/benchmarks/report/compare.py b/.github/actions/benchmark-report/benchmarks/report/compare.py new file mode 100644 index 0000000..a6964e1 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/compare.py @@ -0,0 +1,194 @@ +"""Descriptive paired comparisons, without a statistical performance gate.""" + +import math +from statistics import median + +from .validate import require, validate + +# These describe the SDK under test, not the measurement environment. +SDK_BUILD_FIELDS = {"dependency_lock_digest", "build_digest", "core_build", "entry", "sdk_version", "packages"} +FAILURES = {"error", "timeout", "oom"} +SOURCE_STATUSES = { + "success", + "failure", + "cancelled", + "timed_out", + "action_required", + "neutral", + "skipped", + "stale", + "startup_failure", +} + + +def without(value: dict, keys: set) -> dict: + return {key: item for key, item in value.items() if key not in keys} + + +def global_mismatches(base: dict, head: dict) -> list[str]: + reasons = [] + if base["sdk"] != head["sdk"]: + reasons.append("SDK differs") + if base["suite"] != head["suite"]: + reasons.append("suite/source/profile/measurement configuration differs") + if without(base["execution"], SDK_BUILD_FIELDS) != without(head["execution"], SDK_BUILD_FIELDS): + reasons.append("paired execution environment differs") + return reasons + + +def run_configuration(workload: dict): + return sorted( + ( + run["pass"], + run["repetition"], + run.get("operations"), + run.get("warmup_operations"), + [point["operations"] for point in run.get("checkpoints", [])], + ) + for run in workload.get("runs", []) + ) + + +def failed_statuses(workload: dict) -> list[str]: + return sorted({s for s in [workload["status"], *(r["status"] for r in workload.get("runs", []))] if s != "ok"}) + + +def _percentage(base: float, head: float): + if base == 0: + return None + value = (head - base) / base * 100 + return value if math.isfinite(value) else None + + +def compare(base: dict, head: dict, *, source_status: str | None = None) -> dict: + require( + source_status is None or (isinstance(source_status, str) and source_status in SOURCE_STATUSES), + "Invalid source execution status", + ) + source_failed = source_status not in (None, "success") + validate(base) + validate(head) + require(base["revision"]["role"] == "base" and head["revision"]["role"] == "head", "Wrong revision roles") + mismatch = global_mismatches(base, head) + rows = [] + workloads = {} + execution_failed = source_failed + bases = {b["id"]: b for b in base["benchmarks"]} + heads = {b["id"]: b for b in head["benchmarks"]} + for workload_id in sorted(bases.keys() | heads.keys()): + left, right = bases.get(workload_id), heads.get(workload_id) + workloads[workload_id] = { + role: { + "parameters": workload["parameters"], + "fixture_digest": workload["fixture_digest"], + "status": workload["status"], + "reason": workload.get("reason"), + "failures": [ + { + key: run[key] + for key in ( + "pass", + "repetition", + "status", + "reason", + "error", + "code", + "signal", + "rss_limit_exceeded", + ) + if key in run + } + for run in workload.get("runs", []) + if run["status"] != "ok" + ], + "checkpoints": [ + {"repetition": r["repetition"], "pass": r["pass"], "points": r["checkpoints"]} + for r in workload.get("runs", []) + if "checkpoints" in r + ], + } + for role, workload in (("base", left), ("head", right)) + if workload + } + metrics = [{m["id"]: m for m in w["metrics"]} if w else {} for w in (left, right)] + # A failed workload with no metrics still needs a prominent row. + for metric_id in sorted(metrics[0].keys() | metrics[1].keys()) or ["(workload)"]: + bm, hm = (m.get(metric_id) for m in metrics) + row = {"workload": workload_id, "metric": metric_id, "base": bm, "head": hm, "status": "inconclusive"} + failures = [] + row["explanations"] = [] + row_failed = False + for role, workload, metric in (("base", left, bm), ("head", right, hm)): + if workload: + for source in (workload, metric): + if source and source["status"] != "ok" and source.get("reason"): + row["explanations"].append(f"{role}: {source['reason']}") + statuses = set(failed_statuses(workload)) + if metric and metric["status"] != "ok": + statuses.add(metric["status"]) + failures.extend(f"{role}: {status}" for status in sorted(statuses)) + row_failed |= bool(statuses & FAILURES) + execution_failed |= row_failed + reasons = list(mismatch) + if source_failed or failures: + row.update( + status="unavailable", + reasons=([f"Source execution: {source_status}"] if source_failed else []) + failures, + execution_failed=source_failed or row_failed, + ) + elif left is None or bm is None: + row.update(status="no baseline", reasons=["baseline workload or metric missing"]) + elif right is None or hm is None: + row.update(status="missing head", reasons=["head workload or metric missing"]) + else: + if left["parameters"] != right["parameters"] or left["fixture_digest"] != right["fixture_digest"]: + reasons.append("workload parameters or fixture differ") + if run_configuration(left) != run_configuration(right): + reasons.append("raw run identities, operations, warmup or checkpoints differ") + if without(bm, {"status", "samples", "reason"}) != without(hm, {"status", "samples", "reason"}): + reasons.append("metric unit, method, quality, direction or configuration differs") + bs = {s["repetition"]: s for s in bm.get("samples", [])} + hs = {s["repetition"]: s for s in hm.get("samples", [])} + if bs.keys() != hs.keys() or any( + bs[k]["operations"] != hs[k]["operations"] for k in bs.keys() & hs.keys() + ): + reasons.append("paired sample identities or operation counts differ") + if reasons: + row.update(status="not comparable", reasons=reasons) + elif bs: + bvalues, hvalues = ([s[k]["value"] for k in sorted(s)] for s in (bs, hs)) + bmedian, hmedian = median(bvalues), median(hvalues) + deltas = [h - b for b, h in zip(bvalues, hvalues)] + percentages = [_percentage(b, h) for b, h in zip(bvalues, hvalues)] + row.update( + base_median=bmedian, + head_median=hmedian, + change_percent=_percentage(bmedian, hmedian), + pairs=len(bs), + paired_delta={"median": median(deltas), "min": min(deltas), "max": max(deltas)}, + paired_change_percent=percentages, + reasons=["Descriptive only; uncertainty is not calibrated"], + ) + if bmedian == 0: + row["reasons"].append("Zero baseline; percentage undefined") + if len(bs) != base["suite"]["repetitions"]: + row["reasons"].append("Incomplete repetitions") + else: + row.update(status="unavailable", reasons=["No completed paired samples"]) + rows.append(row) + rows.sort( + key=lambda r: (not r.get("execution_failed", False), r["status"] == "inconclusive", r["workload"], r["metric"]) + ) + return { + "comparison_version": 1, + "sdk": head["sdk"], + "execution_failed": execution_failed, + "source_execution_status": source_status, + "compatibility": mismatch, + "context": { + role: {k: doc[k] for k in ("revision", "suite", "execution")} + for role, doc in (("base", base), ("head", head)) + }, + "workloads": workloads, + "rows": rows, + } diff --git a/.github/actions/benchmark-report/benchmarks/report/github.py b/.github/actions/benchmark-report/benchmarks/report/github.py new file mode 100644 index 0000000..0d3be69 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/github.py @@ -0,0 +1,145 @@ +"""Small bounded GitHub.com client. Signed downloads never receive API credentials.""" + +import io +import re +import stat +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +import zlib +from http.client import HTTPException + +from .render import json_text +from .validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, require + +API_ORIGIN = "https://api.github.com" +MAX_ARCHIVE_BYTES = 20 * 1024 * 1024 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class GitHub: + def __init__(self, token: str): + require(bool(token) and not any(c in token for c in "\r\n"), "GH_TOKEN is required") + self.token = token + self.opener = urllib.request.build_opener(NoRedirect()) + self.requests = 0 + self.deadline = time.monotonic() + 180 + + def _request(self, url: str, *, authenticated: bool, method="GET", body=None, limit=MAX_DOCUMENT_BYTES): + require(self.requests < 100 and time.monotonic() < self.deadline, "GitHub request budget exceeded") + self.requests += 1 + parsed = urllib.parse.urlsplit(url) + require( + parsed.scheme == "https" + and parsed.hostname + and not parsed.username + and not parsed.password + and not parsed.fragment, + "Unsafe download URL", + ) + headers = {"User-Agent": "posthog-sdk-benchmark-report", "Accept": "application/vnd.github+json"} + if authenticated: + require(f"{parsed.scheme}://{parsed.netloc}" == API_ORIGIN, "Untrusted API origin") + headers.update(Authorization=f"Bearer {self.token}", **{"X-GitHub-Api-Version": "2022-11-28"}) + data = json_text(body).encode() if body is not None else None + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, headers=headers, data=data, method=method) + try: + response = self.opener.open(request, timeout=min(20, max(1, self.deadline - time.monotonic()))) + except urllib.error.HTTPError as exc: + response = exc + except (OSError, ValueError, HTTPException): + raise InvalidResult("GitHub transport failed") from None + try: + with response: + status = response.code + location = response.headers.get("Location") + content = response.read(limit + 1) + require(len(content) <= limit, "GitHub response exceeds limit") + return status, location, content + except (OSError, ValueError, HTTPException) as exc: + if isinstance(exc, InvalidResult): + raise + raise InvalidResult("GitHub response could not be read") from None + + def api(self, path: str, method="GET", body=None): + require(path.startswith("/repos/") and not any(c in path for c in "\r\n#"), "Invalid API path") + status, _, data = self._request(API_ORIGIN + path, authenticated=True, method=method, body=body) + require(200 <= status < 300, f"GitHub API request failed (HTTP {status})") + # Compare responses may contain large patches; only their merge-base metadata is consumed. + comparison = method == "GET" and re.fullmatch( + r"/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/compare/[0-9a-f]{40}\.\.\.[0-9a-f]{40}", path + ) + return bounded_json(data, text_limit=MAX_DOCUMENT_BYTES if comparison else 65536) if data else None + + def pages(self, path: str, key: str | None = None) -> list: + result = [] + separator = "&" if "?" in path else "?" + for page in range(1, 11): + response = self.api(f"{path}{separator}per_page=100&page={page}") + items = response[key] if key else response + require(isinstance(items, list) and len(items) <= 100, "Invalid paginated API response") + result.extend(items) + if len(items) < 100: + return result + raise InvalidResult("GitHub pagination limit exceeded") + + def download(self, path: str) -> bytes: + require( + re.fullmatch(r"/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/artifacts/[1-9][0-9]*/zip", path), + "Invalid artifact endpoint", + ) + status, location, _ = self._request(API_ORIGIN + path, authenticated=True) + require(status == 302 and location, "Expected signed artifact download redirect") + for _ in range(3): + status, next_location, data = self._request(location, authenticated=False, limit=MAX_ARCHIVE_BYTES) + if status == 200: + return data + require(status in (301, 302, 303, 307, 308) and next_location, "Artifact download failed") + location = urllib.parse.urljoin(location, next_location) + raise InvalidResult("Artifact redirect limit exceeded") + + +def read_archive(data: bytes) -> dict[str, bytes]: + require(len(data) <= MAX_ARCHIVE_BYTES, "Compressed artifact exceeds size limit") + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + entries = archive.infolist() + names = {entry.filename for entry in entries} + require( + len(entries) == len(names) + and names in ({"base.json", "head.json", "revisions.json"}, {"bundle.json", "revisions.json"}), + "Unexpected or duplicate artifact filename", + ) + require( + sum(entry.file_size for entry in entries) <= MAX_ARCHIVE_BYTES, + "Uncompressed artifact exceeds size limit", + ) + result = {} + for entry in entries: + require(entry.orig_filename == entry.filename, "Invalid artifact filename") + mode = stat.S_IFMT(entry.external_attr >> 16) + require(mode in (0, stat.S_IFREG) and not entry.is_dir(), "Artifact contains a non-regular file") + require(not entry.flag_bits & 1, "Encrypted artifacts are unsupported") + require( + entry.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), "Unsupported ZIP compression" + ) + limit = 8192 if entry.filename == "revisions.json" else MAX_DOCUMENT_BYTES + require(entry.file_size <= limit, "Artifact member exceeds size limit") + require( + entry.file_size <= max(1, entry.compress_size) * 200, "Artifact compression ratio exceeds limit" + ) + with archive.open(entry) as stream: + content = stream.read(limit + 1) + require(len(content) == entry.file_size and len(content) <= limit, "Invalid ZIP member size") + result[entry.filename] = content + return result + except (zipfile.BadZipFile, RuntimeError, OSError, EOFError, NotImplementedError, UnicodeError, zlib.error): + raise InvalidResult("Malformed ZIP artifact") from None diff --git a/.github/actions/benchmark-report/benchmarks/report/publish.py b/.github/actions/benchmark-report/benchmarks/report/publish.py new file mode 100644 index 0000000..3a19d7b --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/publish.py @@ -0,0 +1,229 @@ +"""Verify GitHub identities before consuming data or updating a bot comment.""" + +import re +from datetime import datetime +from urllib.parse import urlencode + +from .bundle import compare_bundle, validate_bundle, validate_revisions +from .compare import compare +from .github import MAX_ARCHIVE_BYTES, GitHub, read_archive +from .render import render +from .validate import InvalidResult, bounded_json, require, validate + +SHA = re.compile(r"[0-9a-f]{40}") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +KEY = re.compile(r"[A-Za-z0-9_.-]{1,100}") + + +class SkipPublication(Exception): + """Verified metadata cannot establish a safe, current publication target.""" + + +def positive_id(value) -> int: + require(type(value) is int and 0 < value < 2**63, "Invalid GitHub numeric identity") + return value + + +def sha(value) -> str: + require(isinstance(value, str) and SHA.fullmatch(value), "Invalid GitHub commit identity") + return value + + +def timestamp(value) -> datetime: + require(isinstance(value, str), "Missing GitHub timestamp") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + require(result.tzinfo is not None, "Missing timestamp timezone") + return result + except ValueError: + raise InvalidResult("Invalid GitHub timestamp") from None + + +def same_repository(left: dict, right: dict) -> bool: + return ( + positive_id(left["id"]) == positive_id(right["id"]) and left["full_name"].lower() == right["full_name"].lower() + ) + + +class Publisher: + def __init__(self, api: GitHub, repository: str, workflow_path: str, artifact_name: str, report_key: str, sdk: str): + require(REPOSITORY.fullmatch(repository), "Invalid caller repository") + require(re.fullmatch(r"\.github/workflows/[A-Za-z0-9_.-]+\.ya?ml", workflow_path), "Invalid workflow path") + require(KEY.fullmatch(report_key), "Invalid report key") + require(0 < len(artifact_name) <= 256 and 0 < len(sdk) <= 256, "Missing artifact name or SDK") + self.api = api + self.repository = repository + self.root = f"/repos/{repository}" + self.workflow_path = workflow_path + self.artifact_name = artifact_name + self.sdk = sdk + self.marker = f"" + self.comparison = None + self.report = None + + def verified_run(self, event: dict) -> dict: + require(event["action"] == "completed", "Expected a completed workflow_run event") + original = event["workflow_run"] + require(event["repository"]["full_name"].lower() == self.repository.lower(), "Event repository differs") + run_id = positive_id(original["id"]) + run = self.api.api(f"{self.root}/actions/runs/{run_id}") + require( + run["id"] == run_id and same_repository(run["repository"], event["repository"]), + "Run repository or ID differs", + ) + require( + run["event"] == "pull_request" and run["status"] == "completed", "Unexpected source run event or status" + ) + require(sha(run["head_sha"]) == sha(original["head_sha"]), "Source run head differs") + if positive_id(run["run_attempt"]) != positive_id(original["run_attempt"]): + raise SkipPublication("Source run attempt has been superseded") + positive_id(run["run_number"]) + workflow = self.api.api(f"{self.root}/actions/workflows/{positive_id(run['workflow_id'])}") + require( + workflow["id"] == run["workflow_id"] and workflow["path"] == self.workflow_path, "Source workflow differs" + ) + return run + + def matches_pr(self, pr: dict, run: dict) -> bool: + return ( + pr["state"] == "open" + and pr["head"]["sha"] == run["head_sha"] + and pr["head"]["repo"] is not None + and same_repository(pr["head"]["repo"], run["head_repository"]) + and same_repository(pr["base"]["repo"], run["repository"]) + ) + + def associate(self, run: dict) -> tuple[dict, str, bool]: + associations = run["pull_requests"] + require(isinstance(associations, list) and len(associations) <= 100, "Invalid PR associations") + fallback = not associations + if fallback: + repository = run["head_repository"]["full_name"] + branch = run["head_branch"] + require(isinstance(repository, str) and REPOSITORY.fullmatch(repository), "Invalid head repository") + require(isinstance(branch, str) and branch, "Missing source head branch") + owner = repository.split("/", 1)[0] + query = urlencode({"state": "open", "head": f"{owner}:{branch}"}) + candidates = self.api.pages(f"{self.root}/pulls?{query}") + else: + candidates = associations + matches = [] + seen = set() + for candidate in candidates: + number = positive_id(candidate["number"]) + require(number not in seen, "Duplicate PR association") + seen.add(number) + pr = self.api.api(f"{self.root}/pulls/{number}") + require(pr["number"] == number, "PR number differs") + if self.matches_pr(pr, run): + target = sha(pr["base"]["sha"] if fallback else candidate["base"]["sha"]) + if not fallback: + require(candidate["head"]["sha"] == run["head_sha"], "Associated PR head differs") + matches.append((pr, target, fallback)) + if len(matches) != 1: + raise SkipPublication("No unique currently open PR matches the source head and repository") + return matches[0] + + def results(self, run: dict, target: str) -> dict: + artifacts = self.api.pages(f"{self.root}/actions/runs/{run['id']}/artifacts", "artifacts") + named = [a for a in artifacts if a["name"] == self.artifact_name and not a["expired"]] + # Artifacts from earlier attempts remain attached to the same run ID. + started = timestamp(run["run_started_at"]) + current = [a for a in named if timestamp(a["created_at"]) >= started] + if not current: + raise SkipPublication("No nonexpired result artifact belongs to this run attempt") + require(len(current) == 1, "Ambiguous result artifacts") + artifact = current[0] + require( + type(artifact["size_in_bytes"]) is int and 0 < artifact["size_in_bytes"] <= MAX_ARCHIVE_BYTES, + "Artifact size exceeds limit", + ) + identity = artifact["workflow_run"] + require( + identity["id"] == run["id"] + and identity["head_sha"] == run["head_sha"] + and identity["head_repository_id"] == run["head_repository"]["id"], + "Artifact source identity differs", + ) + files = read_archive(self.api.download(f"{self.root}/actions/artifacts/{positive_id(artifact['id'])}/zip")) + revisions = bounded_json(files["revisions.json"], limit=8192) + validate_revisions(revisions) + if revisions["target"] != target: + raise SkipPublication("Artifact target revision cannot be verified against trusted PR metadata") + require(revisions["head"] == run["head_sha"], "Artifact head revision differs") + merge = self.api.api(f"{self.root}/compare/{target}...{sha(run['head_sha'])}") + baseline = sha(merge["merge_base_commit"]["sha"]) + require(revisions["base"] == baseline, "Artifact baseline is not the expected merge base") + bundle = validate_bundle(bounded_json(files["bundle.json"])) if "bundle.json" in files else None + pairs = ( + [(shard["base"], shard["head"]) for shard in bundle["shards"]] + if bundle is not None + else [tuple(validate(bounded_json(files[f"{role}.json"])) for role in ("base", "head"))] + ) + for base, head in pairs: + for role, doc, commit in (("base", base, baseline), ("head", head, run["head_sha"])): + require(doc["sdk"] == self.sdk, "Artifact SDK differs") + require( + doc["revision"]["role"] == role and doc["revision"]["commit"] == commit, "Result revision differs" + ) + require(not doc["revision"].get("dirty", False), "Dirty SDK result cannot be published") + require(doc["suite"]["source_commit"] == run["head_sha"], "Benchmark source revision differs") + require(isinstance(run.get("conclusion"), str), "Missing source run conclusion") + if bundle is not None: + return compare_bundle(bundle, source_status=run["conclusion"]) + return compare(*pairs[0], source_status=run["conclusion"]) + + def upsert(self, event: dict, run: dict, pr: dict, target: str, fallback: bool) -> str: + comments = self.api.pages(f"{self.root}/issues/{positive_id(pr['number'])}/comments") + matches = [ + c + for c in comments + if c["user"]["login"] == "github-actions[bot]" + and c["user"]["type"] == "Bot" + and (c.get("body") or "").splitlines()[:1] == [self.marker] + ] + require(len(matches) <= 1, "Multiple reporter bot comments") + order = (run["run_number"], run["id"], run["run_attempt"]) + if matches: + lines = matches[0]["body"].splitlines() + stamp = re.fullmatch( + r"", + lines[1] if len(lines) > 1 else "", + ) + require(stamp is not None, "Existing bot comment lacks a valid source stamp") + if tuple(int(n) for n in stamp.groups()) >= order: + raise SkipPublication("An equal or newer source run has already been published") + current_pr = self.api.api(f"{self.root}/pulls/{pr['number']}") + if not self.matches_pr(current_pr, run) or (fallback and current_pr["base"]["sha"] != target): + raise SkipPublication("PR identity changed before publication") + latest = self.verified_run(event) + require((latest["run_number"], latest["id"], latest["run_attempt"]) == order, "Source run identity changed") + body = self.marker + "\n\n" + self.report + require(len(body.encode("utf-8")) < 60_000, "Comment exceeds size limit") + if matches: + self.api.api(f"{self.root}/issues/comments/{positive_id(matches[0]['id'])}", "PATCH", {"body": body}) + return "updated" + self.api.api(f"{self.root}/issues/{pr['number']}/comments", "POST", {"body": body}) + return "created" + + def publish(self, event: dict, *, report_run_id: str | None = None) -> tuple[dict, str]: + try: + run = self.verified_run(event) + pr, target, fallback = self.associate(run) + self.comparison = self.results(run, target) + self.report = render(self.comparison, repository=self.repository, run_id=report_run_id) + status = self.upsert(event, run, pr, target, fallback) + self.comparison["publication"] = { + "status": status, + "pull_request": pr["number"], + "run_id": run["id"], + "run_attempt": run["run_attempt"], + } + return self.comparison, self.report + except SkipPublication as exc: + reason = str(exc) # All skip messages are trusted constants. + result = self.comparison or {"comparison_version": 1} + result["publication"] = {"status": "skipped", "reason": reason} + return result, (self.report or "## SDK performance\n\n") + f"\n**Publication skipped:** {reason}.\n" + except (KeyError, TypeError, IndexError, AttributeError): + raise InvalidResult("Malformed GitHub metadata") from None diff --git a/.github/actions/benchmark-report/benchmarks/report/render.py b/.github/actions/benchmark-report/benchmarks/report/render.py new file mode 100644 index 0000000..40ab38d --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/render.py @@ -0,0 +1,293 @@ +"""Render bounded Markdown from data, never artifact-provided markup or URLs.""" + +import json +import re +import unicodedata +from pathlib import Path + +MAX_REPORT_BYTES = 58_000 # Reserve room for the publisher's trusted markers. +REVIEW_THRESHOLD_PERCENT = 5 +REVIEW_PAIRS = 6 + +SINGLE_FLAG_NAMES = { + "boolean": "Evaluate one boolean flag", + "rollout": "Evaluate a rollout flag", + "variant": "Evaluate a multivariate flag", + "property": "Evaluate a property-filtered flag", + "regex": "Evaluate a regex-filtered flag", + "cohort": "Evaluate a cohort-filtered flag", + "early": "Evaluate a flag (early match)", + "late": "Evaluate a flag (late match)", + "no-match": "Evaluate a flag (no match)", +} +METRIC_NAMES = { + "elapsed_time_per_op": "Time per operation", + "process_cpu_time_per_op": "Process CPU time per operation", + "caller_thread_cpu_time_per_op": "Caller-thread CPU time per operation", + "allocated_bytes_per_op": "Allocated memory per operation", + "allocations_per_op": "Allocations per operation", + "idle_process_rss": "Idle process memory (RSS)", + "observed_workload_peak_rss": "Observed peak process memory (RSS)", + "process_lifetime_peak_rss": "Lifetime peak process memory (RSS)", + "post_settling_process_rss": "Process memory after settling (RSS)", + "post_settling_heap_used": "Managed heap after settling", +} + + +def safe_text(value, limit: int = 240) -> str: + text = str(value) + text = "".join(" " if unicodedata.category(c).startswith("C") or c.isspace() else c for c in text) + # Bound encoded bytes, including entity expansion, without splitting an entity. + pieces = [] + size = 0 + for c in text: + # Entities stop Markdown/HTML syntax; the separator prevents mentions and URL autolinking. + piece = c if c.isalnum() or c == " " else f"&#{ord(c)};" + ("​" if c in "@:." else "") + length = len(piece.encode("utf-8")) + if size + length > limit - 3: + pieces.append("…") + break + pieces.append(piece) + size += length + return "".join(pieces) + + +def json_text(value) -> str: + # Safe even when a consumer embeds the comparison in HTML; it remains ordinary JSON on disk. + text = json.dumps(value, ensure_ascii=True, allow_nan=False, indent=2) + return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + "\n" + + +def _quantity(value, unit: str) -> str: + if value is None: + return "—" + suffix = "/op" if unit.endswith("/op") else "" + if unit in ("ns", "ns/op"): + scales = ((1e9, "s"), (1e6, "ms"), (1e3, "µs"), (1, "ns")) + elif unit in ("bytes", "bytes/op"): + scales = ((1024**3, "GiB"), (1024**2, "MiB"), (1024, "KiB"), (1, "B")) + else: + scales = ((1, unit.removesuffix("/op")),) + scale, label = next(((scale, label) for scale, label in scales if abs(value) >= scale), scales[-1]) + scaled = value / scale + number = "<0.01" if 0 < scaled < 0.01 else f"{scaled:,.2f}".rstrip("0").rstrip(".") + return f"{number} {safe_text(label, 40)}{suffix}" + + +def _workload_name(identity: str) -> str: + if identity.startswith("flags.single."): + return SINGLE_FLAG_NAMES.get(identity.removeprefix("flags.single."), identity) + match = re.fullmatch(r"flags\.(all|load|refresh)\.([0-9]{1,7})", identity) + if match: + mode, count = match.groups() + count = f"{int(count):,}" + return { + "all": f"Evaluate all {count} flags", + "load": f"Evaluate {count} flags under concurrent load", + "refresh": f"Refresh {count} flag definitions", + }[mode] + match = re.fullmatch(r"capture\.(enqueue|drain)(?:\.(steady|burst))?\.([0-9]{1,7})", identity) + if match: + mode, strategy, size = match.groups() + operation = "Queue an event" if mode == "enqueue" else "Capture and deliver events" + schedule = f", {strategy}" if strategy else "" + return f"{operation} ({int(size):,}-byte payload{schedule})" + return identity + + +def review_change(row: dict) -> str | None: + """Descriptive review filter, not a significance test or an execution gate.""" + change = row.get("change_percent") + paired = row.get("paired_change_percent", []) + if ( + row["status"] != "inconclusive" + or row.get("execution_failed") + or change is None + or abs(change) < REVIEW_THRESHOLD_PERCENT + or row.get("pairs") != REVIEW_PAIRS + or len(paired) != REVIEW_PAIRS + or "Incomplete repetitions" in row.get("reasons", []) + or not all(value is not None and (value > 0 if change > 0 else value < 0) for value in paired) + ): + return None + direction = (row["head"] or row["base"])["direction"] + improved = change < 0 if direction == "lower" else change > 0 + return "improvement" if improved else "regression" + + +def _change(row: dict, unit: str) -> str: + change = row.get("change_percent") + if change is None: + return "—" + if change == 0: + return "No observed change" + amount = "<0.1" if abs(change) < 0.1 else f"{abs(change):.1f}".rstrip("0").rstrip(".") + if unit in ("ns", "ns/op"): + description = "less time" if change < 0 else "more time" + elif unit in ("bytes", "bytes/op"): + description = "less memory" if change < 0 else "more memory" + elif unit in ("count", "count/op"): + description = "fewer" if change < 0 else "more" + else: + description = "lower" if change < 0 else "higher" + return f"{amount}% {description}" + + +def _row(row: dict, *, details=False, workload=None) -> str: + metric = row["head"] or row["base"] or {} + unit = metric.get("unit", "") + label = safe_text(METRIC_NAMES.get(row["metric"], row["metric"]), 160) + if metric.get("quality") == "estimated": + label += " (estimated)" + workload_label = safe_text(_workload_name(row["workload"]), 240) + if details and "shard_id" in row: + workload_label += "
Shard: " + safe_text(row["shard_id"], 100) + line = ( + f"| {workload_label} | {label} | {_quantity(row.get('base_median'), unit)} | " + f"{_quantity(row.get('head_median'), unit)} | {_change(row, unit)} |" + ) + if details: + flagged = review_change(row) + assessment = f"Worth reviewing: observed {flagged}" if flagged else "Not flagged" + if row["status"] != "inconclusive": + assessment = safe_text(row["status"]) + reasons = [r for r in row.get("reasons", []) if r != "Descriptive only; uncertainty is not calibrated"] + reasons.extend(row.get("explanations", [])) + for reason in dict.fromkeys(reasons): + assessment += "; " + safe_text(reason, 220) + for role, context in (workload or {}).items(): + failures = context.get("failures", []) + for failure in failures[:3]: + reason = failure.get("reason") or failure.get("error") + if reason: + assessment += ( + f"; {role} {safe_text(failure['pass'], 32)} repetition {failure['repetition']}: " + + safe_text(reason, 220) + ) + if len(failures) > 3: + assessment += f"; {len(failures) - 3} more {role} failures in comparison.json" + line += f" {assessment} |" + return line + "\n" + + +def _incomplete(comparison: dict) -> bool: + return not any("base_median" in row for row in comparison["rows"]) or any( + row["status"] in ("not comparable", "no baseline", "missing head") + or "Incomplete repetitions" in row.get("reasons", []) + for row in comparison["rows"] + ) + + +def headline_rows(comparison: dict) -> list[dict]: + if comparison["execution_failed"] or _incomplete(comparison): + return [] + return sorted( + (row for row in comparison["rows"] if review_change(row)), + key=lambda row: (-abs(row["change_percent"]), row["workload"], row["metric"]), + ) + + +def render(comparison: dict, *, repository=None, run_id=None) -> str: + flagged = headline_rows(comparison) + chunks = [f"## SDK performance: {safe_text(comparison['sdk'])}\n\n"] + if comparison["execution_failed"]: + chunks.append("**Benchmark execution failed. No overall performance conclusion is available.**\n\n") + elif _incomplete(comparison): + chunks.append( + "**No overall performance conclusion is available. " + "Measurements are missing, incomplete or incompatible.**\n\n" + ) + elif flagged: + improvements = sum(review_change(row) == "improvement" for row in flagged) + regressions = len(flagged) - improvements + counts = [] + for count, name in ((improvements, "improvement"), (regressions, "regression")): + if count: + counts.append(f"{count} observed {name}{'s' if count != 1 else ''}") + chunks.append( + f"**{len(flagged)} change{'s' if len(flagged) != 1 else ''} worth reviewing: {', '.join(counts)}.**\n\n" + ) + else: + chunks.append("**Benchmarks completed. No changes flagged for review.**\n\n") + chunks.append( + "Flags require at least a 5% change in the median and all six pairs agreeing on direction. " + "This is a noise filter, not proof of a performance change or a CI gate.\n\n" + ) + table = "| Test | Measurement | Before PR | With PR | Change |\n| --- | --- | ---: | ---: | --- |\n" + if flagged: + chunks.append(table) + chunks.extend(_row(row) for row in flagged[:12]) + if len(flagged) > 12: + chunks.append(f"\nShowing the 12 largest flagged changes; {len(flagged) - 12} more are in all results.\n") + chunks.append( + "\n
\nView all results\n\n" + "| Test | Measurement | Before PR | With PR | Change | Assessment |\n" + "| --- | --- | ---: | ---: | --- | --- |\n" + ) + # Reserve the footer and explanations before adding bounded rows. + size = len("".join(chunks).encode("utf-8")) + shown = 0 + for row in comparison["rows"]: + line = _row(row, details=True, workload=comparison["workloads"][row["workload"]]) + if size + len(line.encode("utf-8")) > MAX_REPORT_BYTES - 3000: + break + chunks.append(line) + size += len(line.encode("utf-8")) + shown += 1 + chunks.append( + "\nValues are medians of paired measurements. Not flagged does not mean unchanged. " + "Unavailable means a measurement could not be provided, not zero. " + "Estimated allocation measurements retain their instrumentation uncertainty. " + "RSS measures the whole process, not just SDK allocations; observed peaks are sampled checkpoints. " + "Paired ranges in the diagnostics describe variability, not confidence intervals.\n\n" + ) + if comparison["comparison_version"] == 2: + chunks.append( + "Each shard uses its own runner; compare Before PR against With PR within a row, not across shards.\n\n" + ) + context = next(iter(comparison["shards"].values()))["context"] + else: + context = comparison["context"] + chunks.append("
\n\n") + revisions = [] + for role in ("base", "head"): + revision = context[role]["revision"] + revisions.append( + f"`{safe_text(revision['commit'])}`" + (" (dirty working tree)" if revision.get("dirty") else "") + ) + chunks.append(f"Compared {revisions[0]} → {revisions[1]}.\n\n") + # Link only to a GitHub run built from caller metadata, never a result-supplied URL. + if ( + isinstance(repository, str) + and len(repository) <= 201 + and re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) + and all(part not in (".", "..") for part in repository.split("/")) + and re.fullmatch(r"[1-9][0-9]{0,18}", str(run_id)) + ): + chunks.append(f"[Full report and diagnostics](https://github.com/{repository}/actions/runs/{run_id})\n\n") + chunks.append( + f"{len(comparison['rows'])} metric rows; {shown} rows shown in all results. " + "Omitted or shortened data remain in `comparison.json`, including per-runner environments, " + "resource limits, measurement methods, samples and checkpoints.\n" + ) + report = "".join(chunks) + assert len(report.encode("utf-8")) < MAX_REPORT_BYTES + return report + + +def write_outputs(output_directory: str, comparison: dict, report: str, environ: dict) -> tuple[Path, Path]: + directory = Path(output_directory).resolve() + directory.mkdir(parents=True, exist_ok=True) + report_path, comparison_path = directory / "report.md", directory / "comparison.json" + report_path.write_text(report, encoding="utf-8") + comparison_path.write_text(json_text(comparison), encoding="utf-8") + if environ.get("GITHUB_STEP_SUMMARY"): + with Path(environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as stream: + stream.write(report) + if environ.get("GITHUB_OUTPUT"): + # Paths are caller-controlled, but must not inject additional action outputs. + if any(c in str(directory) for c in "\r\n"): + raise ValueError("Output directory contains a newline") + with Path(environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as stream: + stream.write(f"report-path={report_path}\ncomparison-path={comparison_path}\n") + return report_path, comparison_path diff --git a/.github/actions/benchmark-report/benchmarks/report/validate.py b/.github/actions/benchmark-report/benchmarks/report/validate.py new file mode 100644 index 0000000..f12eca6 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/report/validate.py @@ -0,0 +1,114 @@ +"""Bounded, data-only validation of SDK benchmark result documents.""" + +import json +import math +from pathlib import Path + +from jsonschema import Draft202012Validator + +MAX_DOCUMENT_BYTES = 8 * 1024 * 1024 +SCHEMA_PATH = Path(__file__).resolve().parents[1] / "schema" / "result.schema.json" +VALIDATOR = Draft202012Validator(json.loads(SCHEMA_PATH.read_text())) + + +class InvalidResult(ValueError): + """Untrusted input did not satisfy the result contract.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise InvalidResult(message) + + +def _unique_object(pairs: list) -> dict: + result = {} + for key, value in pairs: + require(key not in result, "Duplicate JSON object key") + result[key] = value + return result + + +def bounded_json(data: bytes, limit: int = MAX_DOCUMENT_BYTES, text_limit: int = 4096): + require(len(data) <= limit, "JSON document exceeds byte limit") + try: + value = json.loads(data, object_pairs_hook=_unique_object) + check_bounds(value, text_limit=text_limit) + return value + except (UnicodeError, RecursionError, ValueError) as exc: + if isinstance(exc, InvalidResult): + raise + raise InvalidResult("Malformed JSON document") from None + + +def check_bounds(value, text_limit: int = 4096) -> None: + nodes = 0 + + def visit(item, depth=0, key=""): + nonlocal nodes + nodes += 1 + require(nodes <= 250_000 and depth <= 20, "JSON structure exceeds limits") + if isinstance(item, dict): + require(len(item) <= 1024, "Too many object fields") + for name, child in item.items(): + require(isinstance(name, str) and len(name) <= 256, "Invalid object key") + visit(child, depth + 1, name) + elif isinstance(item, list): + require(len(item) <= 2000, "Too many array items") + for child in item: + visit(child, depth + 1, key) + elif isinstance(item, str): + require(len(item) <= (65536 if key in ("stdout", "stderr") else text_limit), "Text exceeds limit") + require(not any(0xD800 <= ord(c) <= 0xDFFF for c in item), "Invalid Unicode text") + elif isinstance(item, (int, float)) and not isinstance(item, bool): + require(abs(item) <= 1e100 and math.isfinite(item), "Non-finite or excessive number") + + visit(value) + + +def _unique(items: list, identity, name: str) -> None: + keys = [identity(item) for item in items] + require(len(keys) == len(set(keys)), f"Duplicate {name} identity") + + +def validate(document: dict) -> dict: + check_bounds(document) + # Do not include validator messages: they can contain raw logs or attacker text. + error = next(VALIDATOR.iter_errors(document), None) + require(error is None, "Result does not satisfy schema v1") + _unique(document["benchmarks"], lambda b: b["id"], "workload") + repetitions = document["suite"]["repetitions"] + for workload in document["benchmarks"]: + _unique(workload["metrics"], lambda m: m["id"], "metric") + runs = workload.get("runs", []) + _unique(runs, lambda r: (r["pass"], r["repetition"]), "run") + for run in runs: + require(run["repetition"] < repetitions, "Run repetition outside configured range") + checkpoints = run.get("checkpoints", []) + positions = [c["operations"] for c in checkpoints] + require(positions == sorted(set(positions)), "Invalid checkpoint identities") + if "operations" in run: + require(all(p <= run["operations"] for p in positions), "Checkpoint exceeds completed operations") + _nonnegative_measurements(run) + for metric in workload["metrics"]: + samples = metric.get("samples", []) + _unique(samples, lambda s: s["repetition"], "sample repetition") + require(all(s["repetition"] < repetitions for s in samples), "Sample repetition outside configured range") + return document + + +def _nonnegative_measurements(value, key="") -> None: + if isinstance(value, dict): + for name, child in value.items(): + # Exit codes may legitimately be negative; correctness evidence is not a measurement. + if name not in ("code", "exit_code", "validation"): + _nonnegative_measurements(child, name) + elif isinstance(value, list): + for child in value: + _nonnegative_measurements(child, key) + elif isinstance(value, (float, int)) and not isinstance(value, bool): + require(value >= 0, "Negative raw measurement") + + +def load_result(path: str | Path) -> dict: + with Path(path).open("rb") as stream: + return validate(bounded_json(stream.read(MAX_DOCUMENT_BYTES + 1))) diff --git a/.github/actions/benchmark-report/benchmarks/schema/result.schema.json b/.github/actions/benchmark-report/benchmarks/schema/result.schema.json new file mode 100644 index 0000000..2ed1441 --- /dev/null +++ b/.github/actions/benchmark-report/benchmarks/schema/result.schema.json @@ -0,0 +1,537 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SDK benchmark result v1", + "type": "object", + "properties": { + "schema_version": { + "const": 1 + }, + "sdk": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "revision": { + "type": "object", + "properties": { + "role": { + "enum": [ + "base", + "head" + ] + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "maxLength": 40 + }, + "dirty": { + "type": "boolean" + } + }, + "required": [ + "role", + "commit" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "version": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "source_commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "maxLength": 40 + }, + "source_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "profile": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "repetitions": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + }, + "required": [ + "id", + "version", + "source_commit", + "source_digest", + "profile", + "repetitions" + ], + "additionalProperties": true + }, + "execution": { + "type": "object", + "properties": { + "pair_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime_version": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runtime_flags": { + "type": "array", + "items": { + "type": "string", + "maxLength": 4096 + }, + "maxItems": 64, + "minItems": 0 + }, + "os": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "arch": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "kernel": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "cpu": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runner_class": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "runner_image": { + "type": [ + "string", + "null" + ], + "maxLength": 256 + }, + "dependency_lock_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "build_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "resource_limits": { + "type": "object", + "minProperties": 1 + }, + "measurement_tools": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "sdk_version": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "entry": { + "type": "string", + "maxLength": 4096 + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "core_build": { + "type": "object", + "properties": { + "entry": { + "type": "string", + "maxLength": 4096 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + } + }, + "required": [ + "entry", + "digest" + ], + "additionalProperties": false + } + }, + "required": [ + "pair_id", + "runtime", + "runtime_version", + "runtime_flags", + "os", + "arch", + "kernel", + "cpu", + "runner_class", + "runner_image", + "dependency_lock_digest", + "resource_limits" + ], + "additionalProperties": true + }, + "benchmarks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "fixture_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "maxLength": 64 + }, + "parameters": { + "type": "object" + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "reason": { + "type": "string", + "maxLength": 4096 + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "unit": { + "enum": [ + "ns/op", + "bytes/op", + "count/op", + "bytes", + "ns", + "count", + "ops/s" + ] + }, + "direction": { + "enum": [ + "lower", + "higher" + ] + }, + "method": { + "type": "string", + "maxLength": 4096 + }, + "quality": { + "enum": [ + "measured", + "estimated" + ] + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "reason": { + "type": "string", + "maxLength": 4096 + }, + "samples": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + } + }, + "required": [ + "repetition", + "value", + "operations" + ], + "additionalProperties": false + }, + "maxItems": 100, + "minItems": 0 + }, + "configuration": { + "type": "object" + }, + "scope": { + "type": "string", + "maxLength": 4096 + } + }, + "required": [ + "id", + "unit", + "status" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "ok" + } + } + }, + "then": { + "required": [ + "method", + "quality", + "direction", + "samples" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "samples": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + } + }, + "required": [ + "repetition", + "value", + "operations" + ], + "additionalProperties": false + }, + "maxItems": 100, + "minItems": 1 + } + } + }, + "else": { + "required": [ + "reason" + ] + } + } + ] + }, + "maxItems": 64, + "minItems": 0 + }, + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "repetition": { + "type": "integer", + "minimum": 0, + "maximum": 99 + }, + "pass": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "status": { + "enum": [ + "ok", + "error", + "timeout", + "oom", + "unsupported" + ] + }, + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "warmup_operations": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "elapsed_ns": { + "type": "number", + "minimum": 0, + "maximum": 1e+100 + }, + "entry": { + "type": "string", + "maxLength": 4096 + }, + "stdout": { + "type": "string", + "maxLength": 65536 + }, + "stderr": { + "type": "string", + "maxLength": 65536 + }, + "checkpoints": { + "type": "array", + "items": { + "type": "object", + "required": [ + "operations" + ], + "properties": { + "operations": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + } + } + }, + "maxItems": 1000, + "minItems": 0 + } + }, + "required": [ + "repetition", + "pass", + "status" + ], + "additionalProperties": true, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "ok" + } + } + }, + "then": { + "required": [ + "operations" + ] + } + } + ] + }, + "maxItems": 400, + "minItems": 0 + } + }, + "required": [ + "id", + "fixture_digest", + "parameters", + "status", + "metrics" + ], + "additionalProperties": false + }, + "maxItems": 128, + "minItems": 1 + } + }, + "required": [ + "schema_version", + "sdk", + "revision", + "suite", + "execution", + "benchmarks" + ], + "additionalProperties": false +} diff --git a/.github/actions/benchmark-report/pyproject.toml b/.github/actions/benchmark-report/pyproject.toml new file mode 100644 index 0000000..de6548f --- /dev/null +++ b/.github/actions/benchmark-report/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "posthog-sdk-benchmark-report" +version = "0.1.0" +requires-python = "==3.12.12" +dependencies = ["jsonschema==4.26.0"] + +[dependency-groups] +dev = [ + "black==25.12.0", + "pytest==9.0.2", + "pyyaml==6.0.3", + "ruff==0.14.10", +] + +[tool.uv] +package = false +exclude-newer = "7 days" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.black] +line-length = 120 +target-version = ["py312"] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I"] diff --git a/.github/actions/benchmark-report/tests/__init__.py b/.github/actions/benchmark-report/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json new file mode 100644 index 0000000..35d6f2e --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-base.json @@ -0,0 +1,667 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "base", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 23715.815125, + "operations": 16000, + "elapsed_ns": 379453042 + }, + { + "repetition": 1, + "value": 22059.3854375, + "operations": 16000, + "elapsed_ns": 352950167 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27478.125, + "operations": 16000, + "elapsed_ns": 379453042 + }, + { + "repetition": 1, + "value": 26465.5625, + "operations": 16000, + "elapsed_ns": 352950167 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20589.468, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20836.4235, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 379453042, + "elapsed_time_per_op": 23715.815125, + "process_cpu_time_per_op": 27478.125, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 329431488, + "sampled_allocations": 10027, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 352950167, + "elapsed_time_per_op": 22059.3854375, + "process_cpu_time_per_op": 26465.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 333382776, + "sampled_allocations": 10147, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 223641600, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 224296960, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 229195776, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 229982208, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 229195776, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 229982208, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 227065856, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 227852288, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11243712, + "operations": 40, + "elapsed_ns": 4316754167 + }, + { + "repetition": 1, + "value": 11243096, + "operations": 40, + "elapsed_ns": 4199492916 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 223641600, + "heap_used": 11134400, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 223739904, + "heap_used": 11088536, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 225001472, + "heap_used": 11210032, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 226508800, + "heap_used": 11162808, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 226885632, + "heap_used": 11231512, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 226902016, + "heap_used": 11179480, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 226951168, + "heap_used": 11264696, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 226967552, + "heap_used": 11222656, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227065856, + "heap_used": 11293768, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4316754167, + "after_work": { + "rss": 227065856, + "heap_used": 11294720, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229195776, + "process_lifetime_peak_rss": 229195776, + "after_settling": { + "rss": 227065856, + "heap_used": 11243712, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 224296960, + "heap_used": 11134896, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 227065856, + "heap_used": 11088224, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 227590144, + "heap_used": 11209440, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 227672064, + "heap_used": 11162216, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 227737600, + "heap_used": 11230560, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 227737600, + "heap_used": 11178528, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 227753984, + "heap_used": 11263864, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 227753984, + "heap_used": 11221920, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227852288, + "heap_used": 11293152, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4199492916, + "after_work": { + "rss": 227852288, + "heap_used": 11294104, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229982208, + "process_lifetime_peak_rss": 229982208, + "after_settling": { + "rss": 227852288, + "heap_used": 11243096, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json new file mode 100644 index 0000000..7401057 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-head.json @@ -0,0 +1,667 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24220.0911875, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 23317.8801875, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 28010.5625, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 27506.125, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20723.7, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20533.181, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 387521459, + "elapsed_time_per_op": 24220.0911875, + "process_cpu_time_per_op": 28010.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 331579200, + "sampled_allocations": 10092, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 373086083, + "elapsed_time_per_op": 23317.8801875, + "process_cpu_time_per_op": 27506.125, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "estimated_bytes": 328530896, + "sampled_allocations": 10000, + "sampling_interval_bytes": 32768, + "validation": { + "definitions": 1, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 48, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 225476608, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 224215040, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 228753408, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 227573760, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11239984, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 11242336, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 225476608, + "heap_used": 11130488, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 228278272, + "heap_used": 11085112, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 228392960, + "heap_used": 11206328, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 228556800, + "heap_used": 11159104, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 228605952, + "heap_used": 11227448, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 228638720, + "heap_used": 11175416, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 228671488, + "heap_used": 11260752, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 228671488, + "heap_used": 11218808, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 228753408, + "heap_used": 11290040, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4196690625, + "after_work": { + "rss": 228753408, + "heap_used": 11290992, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 230883328, + "process_lifetime_peak_rss": 230883328, + "after_settling": { + "rss": 228753408, + "heap_used": 11239984, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 224215040, + "heap_used": 11133840, + "external": 3708754, + "array_buffers": 19743 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 224395264, + "heap_used": 11086648, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 10, + "rss": 227295232, + "heap_used": 11207864, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 15, + "rss": 227344384, + "heap_used": 11161024, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 20, + "rss": 227393536, + "heap_used": 11229704, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 25, + "rss": 227442688, + "heap_used": 11177696, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 30, + "rss": 227491840, + "heap_used": 11263344, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 35, + "rss": 227508224, + "heap_used": 11221328, + "external": 3708794, + "array_buffers": 19743 + }, + { + "operations": 40, + "rss": 227573760, + "heap_used": 11292392, + "external": 3708794, + "array_buffers": 19743 + } + ], + "elapsed_ns": 4194177625, + "after_work": { + "rss": 227573760, + "heap_used": 11293344, + "external": 3708794, + "array_buffers": 19743 + }, + "observed_peak_rss": 229703680, + "process_lifetime_peak_rss": 229703680, + "after_settling": { + "rss": 227573760, + "heap_used": 11242336, + "external": 3708794, + "array_buffers": 19743 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "deliveredByUser": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "requests": 0, + "invalidEvents": 0, + "unexpected": [], + "evaluations": 720, + "captured": 0 + }, + "stderr": "", + "stdout": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json new file mode 100644 index 0000000..1be00a4 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/node-headline.json @@ -0,0 +1,1677 @@ +{ + "schema_version": 1, + "sdk": "posthog-node", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "node-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "032d1d844bc9a8a4e954f6e8d700b494729582f69c93316ec89826759503457f", + "profile": "full", + "repetitions": 2, + "allocation_sampling": { + "samplingInterval": 32768, + "includeObjectsCollectedByMajorGC": true, + "includeObjectsCollectedByMinorGC": true + }, + "transport": "public fetch adapter; in-process JSON; no network", + "settling": "two explicit GC requests separated by event-loop turns", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup" + }, + "execution": { + "pair_id": "example-pair", + "runtime": "node", + "runtime_version": "v24.13.0", + "v8_version": "13.6.233.17-node.37", + "runtime_flags": [ + "--expose-gc", + "--max-old-space-size=512" + ], + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "Apple M4 Pro", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "c3657194c11a07914083cb9087263cdc3d50712c6c9acc995d3ba9b605298b72", + "build_digest": "fd96537a5f482554b4907b68c04a9338d35b514404adb7bc82a06f558693afbc", + "core_build": { + "entry": "/synthetic/sdk/entry", + "digest": "1bfad3d21a775d5c83cfc0f766ecaeb35ffd5c832e5a2fb14b0dc8211c322bc1" + }, + "sdk_version": "5.57.2", + "resource_limits": { + "v8_old_space_mb": 512, + "process_rss_limit_bytes": null, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "3622ecb7382395d5cd358d53085fefa07e80c6abb4e2b90c13b8f49c59a6d80f", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24220.0911875, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 23317.8801875, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 28010.5625, + "operations": 16000, + "elapsed_ns": 387521459 + }, + { + "repetition": 1, + "value": 27506.125, + "operations": 16000, + "elapsed_ns": 373086083 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 20723.7, + "operations": 16000 + }, + { + "repetition": 1, + "value": 20533.181, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.rollout", + "parameters": { + "id": "flags.single.rollout", + "mode": "single", + "flag_count": 10, + "flag_index": 1, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "f0aadeaa1e94de0a830969af9d64e308ac5191873d36b6c4e805385552c31446", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17914.6458125, + "operations": 16000, + "elapsed_ns": 286634333 + }, + { + "repetition": 1, + "value": 18454.3203125, + "operations": 16000, + "elapsed_ns": 295269125 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21350.875, + "operations": 16000, + "elapsed_ns": 286634333 + }, + { + "repetition": 1, + "value": 21848.375, + "operations": 16000, + "elapsed_ns": 295269125 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17031.788, + "operations": 16000 + }, + { + "repetition": 1, + "value": 16985.316, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.variant", + "parameters": { + "id": "flags.single.variant", + "mode": "single", + "flag_count": 10, + "flag_index": 2, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "6e8ee590538f89fcb65a47601425959ce480e7730068a3460073823641892bfb", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24070.1640625, + "operations": 16000, + "elapsed_ns": 385122625 + }, + { + "repetition": 1, + "value": 23840.296875, + "operations": 16000, + "elapsed_ns": 381444750 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27900.5625, + "operations": 16000, + "elapsed_ns": 385122625 + }, + { + "repetition": 1, + "value": 27677.9375, + "operations": 16000, + "elapsed_ns": 381444750 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21340.3305, + "operations": 16000 + }, + { + "repetition": 1, + "value": 21302.417, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.property", + "parameters": { + "id": "flags.single.property", + "mode": "single", + "flag_count": 10, + "flag_index": 3, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "fd9712badeda5ddf2d637cde064d87671c10fd7d45e1234ef43a6c41bbaa0e34", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 12840.151, + "operations": 16000, + "elapsed_ns": 205442416 + }, + { + "repetition": 1, + "value": 12862.2864375, + "operations": 16000, + "elapsed_ns": 205796583 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 16501.6875, + "operations": 16000, + "elapsed_ns": 205442416 + }, + { + "repetition": 1, + "value": 16292.875, + "operations": 16000, + "elapsed_ns": 205796583 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14278.814, + "operations": 16000 + }, + { + "repetition": 1, + "value": 14393.112, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.regex", + "parameters": { + "id": "flags.single.regex", + "mode": "single", + "flag_count": 10, + "flag_index": 4, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "14dcea4d4a3b68838929a114a402cc914ce5b4e157789cf6e798dce598918471", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 22357.6015625, + "operations": 16000, + "elapsed_ns": 357721625 + }, + { + "repetition": 1, + "value": 21291.4609375, + "operations": 16000, + "elapsed_ns": 340663375 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 27434.5625, + "operations": 16000, + "elapsed_ns": 357721625 + }, + { + "repetition": 1, + "value": 26517.875, + "operations": 16000, + "elapsed_ns": 340663375 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 21714.4585, + "operations": 16000 + }, + { + "repetition": 1, + "value": 21738.954, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.cohort", + "parameters": { + "id": "flags.single.cohort", + "mode": "single", + "flag_count": 10, + "flag_index": 5, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "4b1771937bc181170041d2a503f5940e689e0a7c111e97422c1a540b49c54313", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 13332.7291875, + "operations": 16000, + "elapsed_ns": 213323667 + }, + { + "repetition": 1, + "value": 12793.724, + "operations": 16000, + "elapsed_ns": 204699584 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17212.25, + "operations": 16000, + "elapsed_ns": 213323667 + }, + { + "repetition": 1, + "value": 16807.5, + "operations": 16000, + "elapsed_ns": 204699584 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 15794.813, + "operations": 16000 + }, + { + "repetition": 1, + "value": 15655.0615, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.early", + "parameters": { + "id": "flags.single.early", + "mode": "single", + "flag_count": 10, + "flag_index": 6, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "6c6b135eb967c9ec62350f5d675329c3972ff12e1c83fb426d9f28a2e7b027b7", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 13651.08075, + "operations": 16000, + "elapsed_ns": 218417292 + }, + { + "repetition": 1, + "value": 13630.9115, + "operations": 16000, + "elapsed_ns": 218094584 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17075.6875, + "operations": 16000, + "elapsed_ns": 218417292 + }, + { + "repetition": 1, + "value": 17093.75, + "operations": 16000, + "elapsed_ns": 218094584 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 19417.508, + "operations": 16000 + }, + { + "repetition": 1, + "value": 19632.0915, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.late", + "parameters": { + "id": "flags.single.late", + "mode": "single", + "flag_count": 10, + "flag_index": 7, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "11fd86f58bda1ae9b3c66a724ea5f877e3cea7c50731df961194b755f00cf981", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14455.5703125, + "operations": 16000, + "elapsed_ns": 231289125 + }, + { + "repetition": 1, + "value": 14381.328125, + "operations": 16000, + "elapsed_ns": 230101250 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17931.625, + "operations": 16000, + "elapsed_ns": 231289125 + }, + { + "repetition": 1, + "value": 17839.5, + "operations": 16000, + "elapsed_ns": 230101250 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 24590.9875, + "operations": 16000 + }, + { + "repetition": 1, + "value": 24576.676, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.single.no-match", + "parameters": { + "id": "flags.single.no-match", + "mode": "single", + "flag_count": 10, + "flag_index": 8, + "operations": 16000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "eadae028c525aa88c0b0488eab6ca2cadc67969c9b7e5e74e668d7d862b6d402", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1912.862, + "operations": 16000, + "elapsed_ns": 30605792 + }, + { + "repetition": 1, + "value": 1930.453125, + "operations": 16000, + "elapsed_ns": 30887250 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3635.75, + "operations": 16000, + "elapsed_ns": 30605792 + }, + { + "repetition": 1, + "value": 3659.4375, + "operations": 16000, + "elapsed_ns": 30887250 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 17220.415, + "operations": 16000 + }, + { + "repetition": 1, + "value": 17570.8535, + "operations": 16000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.10", + "parameters": { + "id": "flags.all.10", + "mode": "all", + "flag_count": 10, + "operations": 1600, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "20165e70c1fd37a263143ada015dd08026d90494faad376c254a79d7b46667cb", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 82473.77625, + "operations": 1600, + "elapsed_ns": 131958042 + }, + { + "repetition": 1, + "value": 81382.318125, + "operations": 1600, + "elapsed_ns": 130211709 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 167243.75, + "operations": 1600, + "elapsed_ns": 131958042 + }, + { + "repetition": 1, + "value": 162017.5, + "operations": 1600, + "elapsed_ns": 130211709 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 151583.105, + "operations": 1600 + }, + { + "repetition": 1, + "value": 155848.125, + "operations": 1600 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.100", + "parameters": { + "id": "flags.all.100", + "mode": "all", + "flag_count": 100, + "operations": 160, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "133de92e110e6ee3c2bde38226242aac4302fff953ad423dbeecfc92abc164c7", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 655395.56875, + "operations": 160, + "elapsed_ns": 104863291 + }, + { + "repetition": 1, + "value": 677104.94375, + "operations": 160, + "elapsed_ns": 108336791 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1295731.25, + "operations": 160, + "elapsed_ns": 104863291 + }, + { + "repetition": 1, + "value": 1341325, + "operations": 160, + "elapsed_ns": 108336791 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 1474595.55, + "operations": 160 + }, + { + "repetition": 1, + "value": 1461157.9, + "operations": 160 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.all.1000", + "parameters": { + "id": "flags.all.1000", + "mode": "all", + "flag_count": 1000, + "operations": 160, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "cfaf7acdd2056c29e867677d5280fd52eb28f93459053e1e62dc839103e669b3", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6647329.425, + "operations": 160, + "elapsed_ns": 1063572708 + }, + { + "repetition": 1, + "value": 6680127.6, + "operations": 160, + "elapsed_ns": 1068820416 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11793781.25, + "operations": 160, + "elapsed_ns": 1063572708 + }, + { + "repetition": 1, + "value": 11838550, + "operations": 160, + "elapsed_ns": 1068820416 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14686163.4, + "operations": 160 + }, + { + "repetition": 1, + "value": 14629267.4, + "operations": 160 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "flags.load.1000", + "parameters": { + "id": "flags.load.1000", + "mode": "all", + "flag_count": 1000, + "operations": 1600, + "concurrency": 8, + "passes": [ + "memory" + ] + }, + "fixture_digest": "9c35736c1c79903e10fcc2288b1ccfc3016590a22029795a65a32888b94a6075", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 354271232, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 353189888, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386007040, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380108800, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386023424, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380125184, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 386072576, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 380174336, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 9127592, + "operations": 1600, + "elapsed_ns": 14539833167 + }, + { + "repetition": 1, + "value": 9116720, + "operations": 1600, + "elapsed_ns": 14683788958 + } + ] + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6f498a6af80b614c92d658628decdbfd4b56402974da502015c082bd7a14610e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 225476608, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 224215040, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 230883328, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 229703680, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 228753408, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 227573760, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 11239984, + "operations": 40, + "elapsed_ns": 4196690625 + }, + { + "repetition": 1, + "value": 11242336, + "operations": 40, + "elapsed_ns": 4194177625 + } + ] + } + ] + }, + { + "id": "capture.enqueue.32", + "parameters": { + "id": "capture.enqueue.32", + "mode": "capture", + "payload_bytes": 32, + "operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "65961f598e5917166a73bc754237304803c5f76536ed23cfe61f6c441ecfaf39", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4105.4375, + "operations": 2000, + "elapsed_ns": 8210875 + }, + { + "repetition": 1, + "value": 4073.3335, + "operations": 2000, + "elapsed_ns": 8146667 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6975, + "operations": 2000, + "elapsed_ns": 8210875 + }, + { + "repetition": 1, + "value": 6932, + "operations": 2000, + "elapsed_ns": 8146667 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 8906.228, + "operations": 2000 + }, + { + "repetition": 1, + "value": 8420.576, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "capture.enqueue.4096", + "parameters": { + "id": "capture.enqueue.4096", + "mode": "capture", + "payload_bytes": 4096, + "operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "06edc0baaf6e302e2cc37659a605f64eb24282f5ca8c87ef2915ae907e376826", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "process.hrtime.bigint", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4064.458, + "operations": 2000, + "elapsed_ns": 8128916 + }, + { + "repetition": 1, + "value": 4113.729, + "operations": 2000, + "elapsed_ns": 8227458 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "process.cpuUsage", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 6951.5, + "operations": 2000, + "elapsed_ns": 8128916 + }, + { + "repetition": 1, + "value": 6998, + "operations": 2000, + "elapsed_ns": 8227458 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "v8.HeapProfiler.startSampling", + "quality": "estimated", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 8568.984, + "operations": 2000 + }, + { + "repetition": 1, + "value": 8131.808, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "status": "unsupported", + "reason": "V8 samples do not expose total allocation count; sample count is not object count." + } + ] + }, + { + "id": "capture.drain.4096", + "parameters": { + "id": "capture.drain.4096", + "mode": "drain", + "payload_bytes": 4096, + "operations": 10000, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "ba591d6aa9238b5f2776ac74287bd19a4f07d0e81bfb1c02af29c60e20c43f0e", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 74399744, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 74235904, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 89817088, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 89636864, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "process.resourceUsage.maxRSS", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 89817088, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 89636864, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "process.memoryUsage.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 86720512, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 86573056, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "method": "process.memoryUsage.heapUsed", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 9604016, + "operations": 10000, + "elapsed_ns": 86248667 + }, + { + "repetition": 1, + "value": 9603952, + "operations": 10000, + "elapsed_ns": 85617375 + } + ] + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json new file mode 100644 index 0000000..64bf0bd --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-base.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "sdk": "posthog-python", + "revision": { + "role": "base", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "dirty": false + }, + "suite": { + "id": "python-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "77449378b7ba55b91d978a5383ee32f8bceaab69e83e7d262f4669f4e42ed317", + "profile": "full", + "repetitions": 2, + "allocation_tracking": { + "trace_python_allocators": true, + "file_format": "ALL_ALLOCATIONS", + "scope": "PYMALLOC_MALLOC/CALLOC/REALLOC requests; all worker threads" + }, + "transport": "loopback HTTP receiver in coordinator process; uncompressed v0", + "settling": "two gc.collect calls separated by sleep(0)", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup", + "capture": { + "flush_at": 100000, + "flush_interval_seconds": 3600, + "consumer_threads": 1, + "drain_every": 100 + }, + "flag_poll_interval_seconds": 3600 + }, + "execution": { + "runtime": "cpython", + "runtime_version": "3.12.12", + "implementation": "3.12.12 (main, Oct 31 2025, 23:21:56) [Clang 21.1.4 ]", + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "arm", + "entry": "/synthetic/sdk/entry", + "build_digest": "730edbd69df5111e43ada6fc395e2eb329a2a3a5c86a4ac2731d0d056ea8c718", + "packages": { + "pygments": "2.21.0", + "rich": "15.0.0", + "requests": "2.33.0", + "charset-normalizer": "3.4.2", + "posthog": "7.76.0", + "mdit-py-plugins": "0.6.1", + "urllib3": "2.8.0", + "mdurl": "0.1.2", + "textual": "8.2.8", + "idna": "3.19", + "linkify-it-py": "2.2.0", + "jinja2": "3.1.6", + "markdown-it-py": "4.2.0", + "distro": "1.9.0", + "certifi": "2026.10.19", + "typing-extensions": "4.16.0", + "markupsafe": "3.0.4", + "memray": "1.20.0", + "backoff": "2.2.1", + "psutil": "7.2.2", + "platformdirs": "4.13.5" + }, + "sdk_version": "7.76.0", + "measurement_tools": { + "memray": "1.20.0", + "psutil": "7.2.2" + }, + "pair_id": "example-pair", + "runtime_flags": [ + "-B" + ], + "python_hash_seed": 0, + "bytecode_policy": "uv precompiles both environments; workers do not write bytecode", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "f9c1329b3eb2ef295d5913a328d8e67c92bcb19a6142a2a918ac9b61b8744a6a", + "resource_limits": { + "process_rss_limit_bytes": 536870912, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "allocation_operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "a18a5143bd9edebbe2586a84ab72d2041d90c17b4eeb55423718f68fa4b01131", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "time.perf_counter_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3914.2265625, + "operations": 16000, + "elapsed_ns": 62627625 + }, + { + "repetition": 1, + "value": 3942.0260625, + "operations": 16000, + "elapsed_ns": 63072417 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "time.process_time_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 3879.9375, + "operations": 16000, + "elapsed_ns": 62627625 + }, + { + "repetition": 1, + "value": 3937.875, + "operations": 16000, + "elapsed_ns": 63072417 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14805.996, + "operations": 2000 + }, + { + "repetition": 1, + "value": 14805.996, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 160.8935, + "operations": 2000 + }, + { + "repetition": 1, + "value": 160.8935, + "operations": 2000 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 62627625, + "elapsed_time_per_op": 3914.2265625, + "process_cpu_time_per_op": 3879.9375, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 63072417, + "elapsed_time_per_op": 3942.0260625, + "process_cpu_time_per_op": 3937.875, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6c8bab719abd9b61a1073abe748dbc91fd03ff6feadbc6bc59113224046fec39", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 49659904, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 49364992, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51200000, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50364416, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "resource.getrusage.ru_maxrss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51216384, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50380800, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 51200000, + "operations": 40, + "elapsed_ns": 3435970917 + }, + { + "repetition": 1, + "value": 50364416, + "operations": 40, + "elapsed_ns": 2891613125 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "status": "unsupported", + "reason": "CPython has no uninstrumented total managed-heap byte counter; tracing would distort this RSS pass." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49659904 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49692672 + }, + { + "operations": 10, + "rss": 51150848 + }, + { + "operations": 15, + "rss": 51167232 + }, + { + "operations": 20, + "rss": 51167232 + }, + { + "operations": 25, + "rss": 51167232 + }, + { + "operations": 30, + "rss": 51200000 + }, + { + "operations": 35, + "rss": 51200000 + }, + { + "operations": 40, + "rss": 51200000 + } + ], + "elapsed_ns": 3435970917, + "after_work": { + "rss": 51200000 + }, + "observed_peak_rss": 51200000, + "process_lifetime_peak_rss": 51216384, + "after_settling": { + "rss": 51200000 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49364992 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49397760 + }, + { + "operations": 10, + "rss": 49430528 + }, + { + "operations": 15, + "rss": 49807360 + }, + { + "operations": 20, + "rss": 50298880 + }, + { + "operations": 25, + "rss": 50331648 + }, + { + "operations": 30, + "rss": 50364416 + }, + { + "operations": 35, + "rss": 50364416 + }, + { + "operations": 40, + "rss": 50364416 + } + ], + "elapsed_ns": 2891613125, + "after_work": { + "rss": 50364416 + }, + "observed_peak_rss": 50364416, + "process_lifetime_peak_rss": 50380800, + "after_settling": { + "rss": 50364416 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json new file mode 100644 index 0000000..6bb12c7 --- /dev/null +++ b/.github/actions/benchmark-report/tests/fixtures/benchmarks/python-head.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "sdk": "posthog-python", + "revision": { + "role": "head", + "commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dirty": false + }, + "suite": { + "id": "python-sdk-performance", + "version": 1, + "source_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source_digest": "77449378b7ba55b91d978a5383ee32f8bceaab69e83e7d262f4669f4e42ed317", + "profile": "full", + "repetitions": 2, + "allocation_tracking": { + "trace_python_allocators": true, + "file_format": "ALL_ALLOCATIONS", + "scope": "PYMALLOC_MALLOC/CALLOC/REALLOC requests; all worker threads" + }, + "transport": "loopback HTTP receiver in coordinator process; uncompressed v0", + "settling": "two gc.collect calls separated by sleep(0)", + "memory_sampling": "operation checkpoints; lifetime high-water mark includes setup and warmup", + "capture": { + "flush_at": 100000, + "flush_interval_seconds": 3600, + "consumer_threads": 1, + "drain_every": 100 + }, + "flag_poll_interval_seconds": 3600 + }, + "execution": { + "runtime": "cpython", + "runtime_version": "3.12.12", + "implementation": "3.12.12 (main, Oct 31 2025, 23:21:56) [Clang 21.1.4 ]", + "os": "darwin", + "arch": "arm64", + "kernel": "25.6.0", + "cpu": "arm", + "entry": "/synthetic/sdk/entry", + "build_digest": "730edbd69df5111e43ada6fc395e2eb329a2a3a5c86a4ac2731d0d056ea8c718", + "packages": { + "pygments": "2.21.0", + "rich": "15.0.0", + "requests": "2.33.0", + "charset-normalizer": "3.4.2", + "posthog": "7.76.0", + "mdit-py-plugins": "0.6.1", + "urllib3": "2.8.0", + "mdurl": "0.1.2", + "textual": "8.2.8", + "idna": "3.19", + "linkify-it-py": "2.2.0", + "jinja2": "3.1.6", + "markdown-it-py": "4.2.0", + "distro": "1.9.0", + "certifi": "2026.10.19", + "typing-extensions": "4.16.0", + "markupsafe": "3.0.4", + "memray": "1.20.0", + "backoff": "2.2.1", + "psutil": "7.2.2", + "platformdirs": "4.13.5" + }, + "sdk_version": "7.76.0", + "measurement_tools": { + "memray": "1.20.0", + "psutil": "7.2.2" + }, + "pair_id": "example-pair", + "runtime_flags": [ + "-B" + ], + "python_hash_seed": 0, + "bytecode_policy": "uv precompiles both environments; workers do not write bytecode", + "runner_class": "local", + "runner_image": null, + "dependency_lock_digest": "f9c1329b3eb2ef295d5913a328d8e67c92bcb19a6142a2a918ac9b61b8744a6a", + "resource_limits": { + "process_rss_limit_bytes": 536870912, + "worker_timeout_ms": 120000 + } + }, + "benchmarks": [ + { + "id": "flags.single.boolean", + "parameters": { + "id": "flags.single.boolean", + "mode": "single", + "flag_count": 10, + "flag_index": 0, + "operations": 16000, + "allocation_operations": 2000, + "concurrency": 1, + "passes": [ + "timing", + "allocation" + ] + }, + "fixture_digest": "a18a5143bd9edebbe2586a84ab72d2041d90c17b4eeb55423718f68fa4b01131", + "status": "ok", + "metrics": [ + { + "id": "elapsed_time_per_op", + "unit": "ns/op", + "method": "time.perf_counter_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4009.78125, + "operations": 16000, + "elapsed_ns": 64156500 + }, + { + "repetition": 1, + "value": 3949.6745, + "operations": 16000, + "elapsed_ns": 63194792 + } + ] + }, + { + "id": "process_cpu_time_per_op", + "unit": "ns/op", + "method": "time.process_time_ns", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 4004.0625, + "operations": 16000, + "elapsed_ns": 64156500 + }, + { + "repetition": 1, + "value": 3946.5625, + "operations": 16000, + "elapsed_ns": 63194792 + } + ] + }, + { + "id": "allocated_bytes_per_op", + "unit": "bytes/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 14805.996, + "operations": 2000 + }, + { + "repetition": 1, + "value": 14805.996, + "operations": 2000 + } + ] + }, + { + "id": "allocations_per_op", + "unit": "count/op", + "method": "memray.python_allocator_requests", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 160.8935, + "operations": 2000 + }, + { + "repetition": 1, + "value": 160.8935, + "operations": 2000 + } + ] + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 64156500, + "elapsed_time_per_op": 4009.78125, + "process_cpu_time_per_op": 4004.0625, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 0, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "timing", + "operations": 16000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "elapsed_ns": 63194792, + "elapsed_time_per_op": 3949.6745, + "process_cpu_time_per_op": 3946.5625, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "allocation", + "operations": 2000, + "warmup_operations": 160, + "entry": "/synthetic/sdk/entry", + "allocated_bytes": 29611992, + "allocation_calls": 321787, + "validation": { + "definitions": 1, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 64 + }, + "stdout": "", + "stderr": "" + } + ] + }, + { + "id": "flags.refresh.1000", + "parameters": { + "id": "flags.refresh.1000", + "mode": "refresh", + "flag_count": 1000, + "operations": 40, + "concurrency": 1, + "passes": [ + "memory" + ] + }, + "fixture_digest": "6c8bab719abd9b61a1073abe748dbc91fd03ff6feadbc6bc59113224046fec39", + "status": "ok", + "metrics": [ + { + "id": "idle_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 49070080, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 49184768, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "observed_workload_peak_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss.operation_checkpoints", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50167808, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50069504, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "process_lifetime_peak_rss", + "unit": "bytes", + "method": "resource.getrusage.ru_maxrss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50184192, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50085888, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "post_settling_process_rss", + "unit": "bytes", + "method": "psutil.Process.memory_info.rss", + "quality": "measured", + "direction": "lower", + "status": "ok", + "samples": [ + { + "repetition": 0, + "value": 50167808, + "operations": 40, + "elapsed_ns": 2886622709 + }, + { + "repetition": 1, + "value": 50069504, + "operations": 40, + "elapsed_ns": 2931467333 + } + ] + }, + { + "id": "post_settling_heap_used", + "unit": "bytes", + "status": "unsupported", + "reason": "CPython has no uninstrumented total managed-heap byte counter; tracing would distort this RSS pass." + } + ], + "runs": [ + { + "repetition": 0, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49070080 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49610752 + }, + { + "operations": 10, + "rss": 49627136 + }, + { + "operations": 15, + "rss": 50118656 + }, + { + "operations": 20, + "rss": 50118656 + }, + { + "operations": 25, + "rss": 50135040 + }, + { + "operations": 30, + "rss": 50167808 + }, + { + "operations": 35, + "rss": 50167808 + }, + { + "operations": 40, + "rss": 50167808 + } + ], + "elapsed_ns": 2886622709, + "after_work": { + "rss": 50167808 + }, + "observed_peak_rss": 50167808, + "process_lifetime_peak_rss": 50184192, + "after_settling": { + "rss": 50167808 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + }, + { + "repetition": 1, + "status": "ok", + "pass": "memory", + "operations": 40, + "warmup_operations": 2, + "entry": "/synthetic/sdk/entry", + "before": { + "rss": 49184768 + }, + "checkpoints": [ + { + "operations": 5, + "rss": 49217536 + }, + { + "operations": 10, + "rss": 49217536 + }, + { + "operations": 15, + "rss": 49217536 + }, + { + "operations": 20, + "rss": 49217536 + }, + { + "operations": 25, + "rss": 50036736 + }, + { + "operations": 30, + "rss": 50069504 + }, + { + "operations": 35, + "rss": 50069504 + }, + { + "operations": 40, + "rss": 50069504 + } + ], + "elapsed_ns": 2931467333, + "after_work": { + "rss": 50069504 + }, + "observed_peak_rss": 50069504, + "process_lifetime_peak_rss": 50085888, + "after_settling": { + "rss": 50069504 + }, + "validation": { + "definitions": 43, + "delivered": 0, + "requests": 0, + "invalid_events": 0, + "delivered_by_user": { + "benchmark-user-0": 0, + "benchmark-user-1": 0, + "benchmark-user-2": 0, + "benchmark-user-3": 0, + "benchmark-user-4": 0, + "benchmark-user-5": 0, + "benchmark-user-6": 0, + "benchmark-user-7": 0, + "benchmark-user-8": 0, + "benchmark-user-9": 0, + "benchmark-user-10": 0, + "benchmark-user-11": 0, + "benchmark-user-12": 0, + "benchmark-user-13": 0, + "benchmark-user-14": 0, + "benchmark-user-15": 0 + }, + "unexpected": [], + "captured": 0, + "evaluations": 736 + }, + "stdout": "", + "stderr": "" + } + ] + } + ] +} diff --git a/.github/actions/benchmark-report/tests/test_benchmark_bundle.py b/.github/actions/benchmark-report/tests/test_benchmark_bundle.py new file mode 100644 index 0000000..6c2095c --- /dev/null +++ b/.github/actions/benchmark-report/tests/test_benchmark_bundle.py @@ -0,0 +1,512 @@ +"""Shard collection, independent environments and data-only publication regressions.""" + +import copy +import html +import json +import os +import stat +import zipfile + +import pytest + +from benchmarks.report import bundle as bundle_module +from benchmarks.report import cli +from benchmarks.report.bundle import ( + collect, + compare_bundle, + encode_bundle, + load_bundle, + validate_bundle, + validate_manifest, +) +from benchmarks.report.github import read_archive +from benchmarks.report.render import MAX_REPORT_BYTES, render +from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json +from tests.test_benchmark_github import archive +from tests.test_benchmark_publish import BASE, HEAD, TARGET, FakeGitHub, bot_comment, publisher +from tests.test_benchmark_report import pair + + +def bundle_fixture(sdk="node", count=2): + documents = pair(sdk) + manifest = {"manifest_version": 1, "sdk": documents[0]["sdk"], "shards": []} + shards = [] + for index in range(count): + identity = f"shard-{index}" + shard = {"id": identity} + for role, original in zip(("base", "head"), documents): + doc = copy.deepcopy(original) + if count == 2: + doc["benchmarks"] = doc["benchmarks"][index::count] + else: + doc["benchmarks"] = [doc["benchmarks"][0]] + doc["benchmarks"][0]["id"] = f"workload-{index}" + doc["execution"].update(shard_id=identity, pair_id=f"pair-{index}", cpu=f"CPU-{index}") + shard[role] = doc + manifest["shards"].append({"id": identity, "workloads": [w["id"] for w in shard["head"]["benchmarks"]]}) + shards.append(shard) + return {"bundle_version": 1, "manifest": manifest, "shards": shards} + + +def collection_fixture(tmp_path, sdk="node"): + bundle = bundle_fixture(sdk) + root = tmp_path / "shards" + root.mkdir() + manifest = tmp_path / "shards.json" + manifest.write_text(json.dumps(bundle["manifest"])) + for shard in bundle["shards"]: + directory = root / f"benchmark-shard-{shard['id']}" + directory.mkdir() + for role in ("base", "head"): + (directory / f"{role}.json").write_text(json.dumps(shard[role])) + (directory / "revisions.json").write_text(json.dumps({"base": BASE, "head": HEAD, "target": TARGET})) + return manifest, root, bundle + + +@pytest.mark.parametrize("sdk", ["node", "python"]) +def test_collect_preserves_documents_and_offline_bundle_render(tmp_path, monkeypatch, sdk): + manifest, root, expected = collection_fixture(tmp_path, sdk) + monkeypatch.setattr(cli, "GitHub", lambda *a: pytest.fail("Offline collection constructed GitHub client")) + output = tmp_path / "out" + args = [ + "--mode", + "collect", + "--shard-manifest", + str(manifest), + "--shard-directory", + str(root), + "--output-directory", + str(output), + ] + assert cli.main(args, {"GH_TOKEN": "unused"}) == 0 + bundle_path = output / "bundle.json" + assert load_bundle(bundle_path) == expected + assert json.loads((output / "revisions.json").read_text()) == {"base": BASE, "head": HEAD, "target": TARGET} + comparison = json.loads((output / "comparison.json").read_text()) + assert comparison["comparison_version"] == 2 + assert all(row["shard_id"] == comparison["workload_shards"][row["workload"]] for row in comparison["rows"]) + assert all(not shard["compatibility"] for shard in comparison["shards"].values()) + assert any("change_percent" in row for row in comparison["rows"]) + report = (output / "report.md").read_text() + decoded = html.unescape(report).replace("\u200b", "") + assert "Shard: shard-1" in decoded + assert comparison["shards"]["shard-0"]["context"]["head"]["execution"]["cpu"] == "CPU-0" + assert comparison["shards"]["shard-1"]["context"]["head"]["execution"]["cpu"] == "CPU-1" + assert cli.main(["--bundle-results", str(bundle_path), "--output-directory", str(tmp_path / "render")], {}) == 0 + assert (tmp_path / "render/report.md").read_text() == report + + +@pytest.mark.parametrize("field", ["cpu", "runtime_version", "runner_image", "measurement_tools"]) +def test_environment_mismatch_only_suppresses_affected_pair(field): + bundle = bundle_fixture() + bundle["shards"][1]["head"]["execution"][field] = ( + {"tool": "different"} if field == "measurement_tools" else "different" + ) + result = compare_bundle(bundle) + assert not result["shards"]["shard-0"]["compatibility"] + assert result["shards"]["shard-1"]["compatibility"] + assert any("change_percent" in row for row in result["rows"] if row["shard_id"] == "shard-0") + assert all("change_percent" not in row for row in result["rows"] if row["shard_id"] == "shard-1") + + +def test_shard_build_provenance_is_retained_not_compared_across_runners(): + bundle = bundle_fixture() + for index, shard in enumerate(bundle["shards"]): + for role in ("base", "head"): + shard[role]["execution"]["build_digest"] = str(index + (1 if role == "base" else 3)) * 64 + result = compare_bundle(bundle) + for shard in bundle["shards"]: + assert not result["shards"][shard["id"]]["compatibility"] + for role in ("base", "head"): + assert result["shards"][shard["id"]]["context"][role]["execution"] == shard[role]["execution"] + + +@pytest.mark.parametrize( + "status", ["failure", "cancelled", "timed_out", "action_required", "neutral", "skipped", "stale", "startup_failure"] +) +def test_collect_failure_preserves_raw_data_and_suppresses_every_shard(tmp_path, status): + manifest, root, expected = collection_fixture(tmp_path) + output = tmp_path / "out" + assert ( + cli.main( + [ + "--mode", + "collect", + "--shard-manifest", + str(manifest), + "--shard-directory", + str(root), + "--source-status", + status, + "--output-directory", + str(output), + ], + {}, + ) + == 1 + ) + assert load_bundle(output / "bundle.json") == expected + result = json.loads((output / "comparison.json").read_text()) + assert result["execution_failed"] and result["source_execution_status"] == status + assert all("change_percent" not in row and "base_median" not in row for row in result["rows"]) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda m: m.update(manifest_version=True), + lambda m: m.update(manifest_version=2), + lambda m: m.update(extra="unexpected"), + lambda m: m.update(sdk=""), + lambda m: m.update(shards=[]), + lambda m: m.update(shards=m["shards"] * 9), + lambda m: m["shards"].append(copy.deepcopy(m["shards"][0])), + lambda m: m["shards"][0].update(id="../escape"), + lambda m: m["shards"][0].update(id="Uppercase"), + lambda m: m["shards"][0].update(id="a" * 65), + lambda m: m["shards"][0].update(id="name\n"), + lambda m: m["shards"][0].update(id=[]), + lambda m: m["shards"][0].update(workloads=[]), + lambda m: m["shards"][0]["workloads"].append(m["shards"][0]["workloads"][0]), + lambda m: m["shards"][1]["workloads"].append(m["shards"][0]["workloads"][0]), + lambda m: m["shards"][0]["workloads"].append({}), + ], +) +def test_manifest_rejects_invalid_partition(mutation): + manifest = bundle_fixture()["manifest"] + mutation(manifest) + with pytest.raises(InvalidResult): + validate_manifest(manifest) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda b: b.update(bundle_version=True), + lambda b: b.update(extra="unexpected"), + lambda b: b["shards"].pop(), + lambda b: b["shards"].append(copy.deepcopy(b["shards"][0])), + lambda b: b["shards"][1].update(id=b["shards"][0]["id"]), + lambda b: b["shards"][1].update(id="undeclared"), + lambda b: b["shards"][1].update(extra={}), + lambda b: b["shards"][1]["base"]["benchmarks"].pop(), + lambda b: b["shards"][1]["head"]["benchmarks"].pop(), + lambda b: b["shards"][1]["head"].update(sdk="other"), + lambda b: b["shards"][1]["head"]["revision"].update(role="base"), + lambda b: b["shards"][1]["head"]["revision"].update(commit="d" * 40), + lambda b: b["shards"][1]["base"]["revision"].update(dirty=True), + lambda b: b["shards"][1]["base"]["suite"].update(profile="other"), + lambda b: b["shards"][1]["head"]["suite"].update(source_commit="d" * 40), + lambda b: b["shards"][1]["head"]["suite"].update(source_digest="d" * 64), + lambda b: b["shards"][1]["head"]["execution"].update(shard_id="other"), + lambda b: b["shards"][1]["head"]["execution"].update(pair_id="other"), + lambda b: [b["shards"][1][r]["execution"].update(pair_id="pair-0") for r in ("base", "head")], + ], +) +def test_bundle_checks_every_shard(mutation): + bundle = bundle_fixture() + mutation(bundle) + with pytest.raises(InvalidResult): + validate_bundle(bundle) + + +@pytest.mark.parametrize( + "change", + [ + "missing-directory", + "extra-directory", + "duplicate-directory", + "extra-file", + "missing-file", + "file-symlink", + "directory-symlink", + "root-symlink", + "manifest-symlink", + "fifo", + "directory-member", + ], +) +def test_collect_rejects_unsafe_or_incomplete_tree(tmp_path, change): + manifest, root, _ = collection_fixture(tmp_path) + directory = root / "benchmark-shard-shard-1" + member = directory / "head.json" + if change == "missing-directory": + directory.rename(tmp_path / "removed") + elif change in ("extra-directory", "duplicate-directory"): + (root / ("undeclared" if change == "extra-directory" else "benchmark-shard-shard-1-copy")).mkdir() + elif change == "extra-file": + (directory / "script.py").write_text("raise Exception('must not run')") + elif change == "missing-file": + member.unlink() + elif change in ("file-symlink", "directory-symlink", "root-symlink", "manifest-symlink"): + path = { + "file-symlink": member, + "directory-symlink": directory, + "root-symlink": root, + "manifest-symlink": manifest, + }[change] + saved = tmp_path / "saved" + path.rename(saved) + path.symlink_to(saved) + else: + member.unlink() + if change == "fifo": + os.mkfifo(member) + else: + member.mkdir() + with pytest.raises(InvalidResult): + collect(str(manifest), str(root)) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda r: r.update(target="d" * 40), + lambda r: r.update(head="d" * 40), + lambda r: r.update(base="d" * 40), + lambda r: r.update(extra="unexpected"), + lambda r: r.update(target="not-sha"), + ], +) +def test_collect_requires_identical_exact_revisions(tmp_path, mutation): + manifest, root, _ = collection_fixture(tmp_path) + path = root / "benchmark-shard-shard-1/revisions.json" + doc = json.loads(path.read_text()) + mutation(doc) + path.write_text(json.dumps(doc)) + with pytest.raises(InvalidResult): + collect(str(manifest), str(root)) + + +@pytest.mark.parametrize("field", ["base", "head"]) +def test_collect_matches_documents_against_revision_sidecar(tmp_path, field): + manifest, root, _ = collection_fixture(tmp_path) + for path in root.glob("*/revisions.json"): + revisions = json.loads(path.read_text()) + revisions[field] = "d" * 40 + path.write_text(json.dumps(revisions)) + with pytest.raises(InvalidResult, match="differs"): + collect(str(manifest), str(root)) + + +def test_collect_checks_source_identity(tmp_path): + manifest, root, _ = collection_fixture(tmp_path) + for path in [*root.glob("*/base.json"), *root.glob("*/head.json")]: + doc = json.loads(path.read_text()) + doc["suite"]["source_commit"] = "d" * 40 + path.write_text(json.dumps(doc)) + with pytest.raises(InvalidResult, match="source differs"): + collect(str(manifest), str(root)) + + +def test_aggregate_budget_is_checked_before_result_reads(tmp_path, monkeypatch): + manifest, root, _ = collection_fixture(tmp_path) + for path in root.glob("*/head.json"): + with path.open("wb") as stream: + stream.truncate(MAX_DOCUMENT_BYTES // 2) + reads = [] + original = bundle_module.read_regular + monkeypatch.setattr(bundle_module, "read_regular", lambda path, *a: reads.append(path) or original(path, *a)) + with pytest.raises(InvalidResult, match="Aggregate shard inputs"): + collect(str(manifest), str(root)) + assert reads == [manifest] + + +def test_bundle_byte_bounds_and_duplicate_json_keys(tmp_path): + path = tmp_path / "bundle.json" + for data in (b" " * (MAX_DOCUMENT_BYTES + 1), b'{"bundle_version":1,"bundle_version":1}'): + path.write_bytes(data) + with pytest.raises(InvalidResult): + load_bundle(path) + with pytest.raises(InvalidResult, match="Bundle exceeds byte limit"): + encode_bundle({"data": "." * MAX_DOCUMENT_BYTES}) + + +def test_sixteen_shard_report_is_bounded_and_escaped(): + bundle = bundle_fixture(count=16) + for shard in bundle["shards"]: + for role in ("base", "head"): + doc = shard[role] + for field in ("runtime", "runtime_version", "os", "arch", "cpu", "runner_class"): + doc["execution"][field] = "