ci: add trusted SDK benchmark report workflow

.github/actions/benchmark-report/.gitignore

@@ -0,0 +1,4 @@
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/

.github/actions/benchmark-report/action.yml

@@ -0,0 +1,82 @@
+name: SDK benchmark report
+description: Validate and render paired SDK results, or publish from a verified workflow_run
+inputs:
+  mode:
+    description: render or collect (offline), or publish (trusted workflow_run only)
+    default: render
+  source-status:
+    description: Measurement or matrix outcome for render/collect mode; publish mode uses the verified source run conclusion
+    default: ''
+  base-results:
+    description: Baseline result JSON path for render mode
+    default: ''
+  head-results:
+    description: Head result JSON path for render mode
+    default: ''
+  bundle-results:
+    description: Saved bundle JSON path for render mode (instead of base/head)
+    default: ''
+  shard-manifest:
+    description: Producer JSON manifest path for collect mode
+    default: ''
+  shard-directory:
+    description: Directory containing benchmark-shard-ID downloads for collect mode
+    default: ''
+  output-directory:
+    description: Directory for report.md and comparison.json; collect also writes bundle.json and revisions.json
+    default: benchmark-report
+  workflow-path:
+    description: Trusted literal producer workflow path for publish mode
+    default: ''
+  artifact-name:
+    description: Trusted literal producer artifact name for publish mode
+    default: ''
+  report-key:
+    description: Trusted stable comment namespace for publish mode
+    default: ''
+  sdk:
+    description: Trusted literal SDK identity for publish mode
+    default: ''
+outputs:
+  report-path:
+    description: Absolute Markdown report path
+    value: ${{ steps.report.outputs.report-path }}
+  comparison-path:
+    description: Absolute comparison JSON path
+    value: ${{ steps.report.outputs.comparison-path }}
+runs:
+  using: composite
+  steps:
+    - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      with:
+        version: '0.13.4'
+        python-version: '3.12.12'
+        enable-cache: 'false'
+    - id: report
+      shell: bash
+      env:
+        REPORT_ACTION_ROOT: ${{ github.action_path }}
+        REPORT_MODE: ${{ inputs.mode }}
+        REPORT_SOURCE_STATUS: ${{ inputs.source-status }}
+        REPORT_BASE: ${{ inputs.base-results }}
+        REPORT_HEAD: ${{ inputs.head-results }}
+        REPORT_BUNDLE: ${{ inputs.bundle-results }}
+        REPORT_MANIFEST: ${{ inputs.shard-manifest }}
+        REPORT_SHARDS: ${{ inputs.shard-directory }}
+        REPORT_OUTPUT: ${{ inputs.output-directory }}
+        REPORT_WORKFLOW: ${{ inputs.workflow-path }}
+        REPORT_ARTIFACT: ${{ inputs.artifact-name }}
+        REPORT_KEY: ${{ inputs.report-key }}
+        REPORT_SDK: ${{ inputs.sdk }}
+      run: |
+        set -euo pipefail
+        root="$(cd "$REPORT_ACTION_ROOT" && pwd)"
+        uv sync --no-config --project "$root" --frozen --no-dev --no-install-project
+        uv run --no-config --project "$root" --no-sync \
+          python -I "$root/benchmarks/report/cli.py" \
+          --mode "$REPORT_MODE" --source-status "$REPORT_SOURCE_STATUS" \
+          --base-results "$REPORT_BASE" --head-results "$REPORT_HEAD" \
+          --bundle-results "$REPORT_BUNDLE" --shard-manifest "$REPORT_MANIFEST" \
+          --shard-directory "$REPORT_SHARDS" \
+          --output-directory "$REPORT_OUTPUT" --workflow-path "$REPORT_WORKFLOW" \
+          --artifact-name "$REPORT_ARTIFACT" --report-key "$REPORT_KEY" --sdk "$REPORT_SDK"

.github/actions/benchmark-report/benchmarks/report/bundle.py

@@ -0,0 +1,195 @@
+"""Bounded shard collection and independent paired comparisons; inputs are data only."""
+
+import json
+import os
+import re
+import stat
+from pathlib import Path
+
+from .compare import compare
+from .validate import MAX_DOCUMENT_BYTES, bounded_json, check_bounds, require, validate
+
+MAX_SHARDS = 16
+SHARD_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,63}")
+
+
+def validate_manifest(manifest: dict) -> dict:
+    check_bounds(manifest)
+    require(
+        isinstance(manifest, dict) and set(manifest) == {"manifest_version", "sdk", "shards"},
+        "Invalid shard manifest fields",
+    )
+    require(type(manifest["manifest_version"]) is int and manifest["manifest_version"] == 1, "Invalid manifest version")
+    require(isinstance(manifest["sdk"], str) and 0 < len(manifest["sdk"]) <= 256, "Invalid manifest SDK")
+    shards = manifest["shards"]
+    require(isinstance(shards, list) and 0 < len(shards) <= MAX_SHARDS, "Invalid manifest shard count")
+    ids, workloads = set(), set()
+    for shard in shards:
+        require(isinstance(shard, dict) and set(shard) == {"id", "workloads"}, "Invalid manifest shard fields")
+        identity = shard["id"]
+        require(isinstance(identity, str) and SHARD_ID.fullmatch(identity), "Unsafe shard ID")
+        require(identity not in ids, "Duplicate manifest shard")
+        ids.add(identity)
+        members = shard["workloads"]
+        require(isinstance(members, list) and 0 < len(members) <= 128, "Invalid shard workload count")
+        for member in members:
+            require(isinstance(member, str) and 0 < len(member) <= 256, "Invalid manifest workload ID")
+            require(member not in workloads, "Duplicate manifest workload")
+            workloads.add(member)
+    return manifest
+
+
+def validate_revisions(revisions: dict) -> dict:
+    require(isinstance(revisions, dict) and set(revisions) == {"target", "head", "base"}, "Invalid revisions document")
+    require(
+        all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value) for value in revisions.values()),
+        "Invalid revision SHA",
+    )
+    return revisions
+
+
+def validate_bundle(bundle: dict) -> dict:
+    check_bounds(bundle)
+    require(
+        isinstance(bundle, dict) and set(bundle) == {"bundle_version", "manifest", "shards"}, "Invalid bundle fields"
+    )
+    require(type(bundle["bundle_version"]) is int and bundle["bundle_version"] == 1, "Invalid bundle version")
+    manifest = validate_manifest(bundle["manifest"])
+    expected = {shard["id"]: set(shard["workloads"]) for shard in manifest["shards"]}
+    shards = bundle["shards"]
+    require(isinstance(shards, list) and len(shards) == len(expected), "Bundle shard count differs")
+    seen, pairs, revisions = set(), set(), {}
+    suite = None
+    for shard in shards:
+        require(isinstance(shard, dict) and set(shard) == {"id", "base", "head"}, "Invalid bundle shard fields")
+        identity = shard["id"]
+        require(
+            isinstance(identity, str) and identity in expected and identity not in seen,
+            "Unexpected or duplicate bundle shard",
+        )
+        seen.add(identity)
+        for role in ("base", "head"):
+            doc = validate(shard[role])
+            require(doc["sdk"] == manifest["sdk"], "Shard SDK differs")
+            require(doc["revision"]["role"] == role, "Wrong shard revision role")
+            require(doc["execution"].get("shard_id") == identity, "Result shard ID differs")
+            require({w["id"] for w in doc["benchmarks"]} == expected[identity], "Shard workload coverage differs")
+            if suite is None:
+                suite = doc["suite"]
+            require(doc["suite"] == suite, "Shard suite/source/profile differs")
+            revisions.setdefault(role, doc["revision"])
+            require(doc["revision"] == revisions[role], "Shard revision identity differs")
+        pair_id = shard["base"]["execution"]["pair_id"]
+        require(pair_id == shard["head"]["execution"]["pair_id"], "Shard pair IDs differ")
+        require(pair_id not in pairs, "Pair ID reused across shards")
+        pairs.add(pair_id)
+    return bundle
+
+
+def compare_bundle(bundle: dict, *, source_status: str | None = None) -> dict:
+    validate_bundle(bundle)
+    shards = {}
+    rows, workloads, workload_shards = [], {}, {}
+    for shard in bundle["shards"]:
+        identity = shard["id"]
+        comparison = compare(shard["base"], shard["head"], source_status=source_status)
+        shards[identity] = {key: comparison[key] for key in ("context", "compatibility", "execution_failed")}
+        rows.extend(dict(row, shard_id=identity) for row in comparison["rows"])
+        workloads.update(comparison["workloads"])
+        workload_shards.update({workload: identity for workload in comparison["workloads"]})
+    rows.sort(
+        key=lambda r: (not r.get("execution_failed", False), r["status"] == "inconclusive", r["workload"], r["metric"])
+    )
+    return {
+        "comparison_version": 2,
+        "sdk": bundle["manifest"]["sdk"],
+        "manifest": bundle["manifest"],
+        "source_execution_status": source_status,
+        "execution_failed": any(shard["execution_failed"] for shard in shards.values()),
+        "shards": shards,
+        "workload_shards": workload_shards,
+        "workloads": workloads,
+        "rows": rows,
+    }
+
+
+def read_regular(path: Path, limit: int = MAX_DOCUMENT_BYTES) -> bytes:
+    require(stat.S_ISREG(path.lstat().st_mode), "Input must be a regular file")
+    with path.open("rb") as stream:
+        require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), "Input must be a regular file")
+        require(os.fstat(stream.fileno()).st_size <= limit, "Input exceeds byte limit")
+        data = stream.read(limit + 1)
+    require(len(data) <= limit, "Input exceeds byte limit")
+    return data
+
+
+def load_bundle(path: str | Path) -> dict:
+    return validate_bundle(bounded_json(read_regular(Path(path))))
+
+
+def directory_members(path: Path, maximum: int) -> set[str]:
+    require(stat.S_ISDIR(path.lstat().st_mode), "Shard input must be a regular directory")
+    names = set()
+    with os.scandir(path) as entries:
+        for entry in entries:
+            names.add(entry.name)
+            require(len(names) <= maximum, "Too many shard directory members")
+    return names
+
+
+def encode_bundle(bundle: dict) -> bytes:
+    # Compact encoding keeps the canonical document within the same input byte budget.
+    data = json.dumps(bundle, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode() + b"\n"
+    require(len(data) <= MAX_DOCUMENT_BYTES, "Bundle exceeds byte limit")
+    return data
+
+
+def collect(manifest_path: str, shard_directory: str) -> tuple[dict, dict]:
+    manifest_bytes = read_regular(Path(manifest_path))
+    manifest = validate_manifest(bounded_json(manifest_bytes))
+    root = Path(shard_directory)
+    expected = {f"benchmark-shard-{shard['id']}" for shard in manifest["shards"]}
+    require(directory_members(root, MAX_SHARDS) == expected, "Missing or unexpected shard directory")
+    paths = []
+    total = len(manifest_bytes)
+    # Preflight all sizes before retaining any result collection.
+    for shard in manifest["shards"]:
+        directory = root / f"benchmark-shard-{shard['id']}"
+        require(
+            directory_members(directory, 3) == {"base.json", "head.json", "revisions.json"},
+            "Unexpected or missing shard member",
+        )
+        for name in ("base.json", "head.json", "revisions.json"):
+            path = directory / name
+            info = path.lstat()
+            require(stat.S_ISREG(info.st_mode), "Shard member must be a regular file")
+            limit = 8192 if name == "revisions.json" else MAX_DOCUMENT_BYTES
+            require(info.st_size <= limit, "Shard member exceeds byte limit")
+            total += info.st_size
+            require(total <= MAX_DOCUMENT_BYTES, "Aggregate shard inputs exceed byte limit")
+            paths.append((shard["id"], name, path, limit))
+    shards = {shard["id"]: {"id": shard["id"]} for shard in manifest["shards"]}
+    revisions = None
+    consumed = len(manifest_bytes)
+    for identity, name, path, limit in paths:
+        data = read_regular(path, min(limit, MAX_DOCUMENT_BYTES - consumed))
+        consumed += len(data)
+        doc = bounded_json(data, limit=limit)
+        if name == "revisions.json":
+            validate_revisions(doc)
+            if revisions is None:
+                revisions = doc
+            require(doc == revisions, "Shard revisions documents differ")
+        else:
+            shards[identity][name.removesuffix(".json")] = doc
+    bundle = validate_bundle({"bundle_version": 1, "manifest": manifest, "shards": list(shards.values())})
+    for shard in bundle["shards"]:
+        for role in ("base", "head"):
+            require(
+                shard[role]["revision"]["commit"] == revisions[role], "Shard revision differs from revisions document"
+            )
+            require(
+                shard[role]["suite"]["source_commit"] == revisions["head"], "Shard source differs from head revision"
+            )
+    encode_bundle(bundle)
+    return bundle, revisions

.github/actions/benchmark-report/benchmarks/report/cli.py

@@ -0,0 +1,99 @@
+"""Offline renderer and trusted workflow_run publisher entry point."""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Support absolute-path invocation from a composite action without installing the project.
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from benchmarks.report.bundle import collect, compare_bundle, encode_bundle, load_bundle
+from benchmarks.report.compare import SOURCE_STATUSES, compare
+from benchmarks.report.github import GitHub
+from benchmarks.report.publish import Publisher
+from benchmarks.report.render import json_text, render, write_outputs
+from benchmarks.report.validate import MAX_DOCUMENT_BYTES, InvalidResult, bounded_json, load_result, require
+
+
+def main(argv=None, environ=None) -> int:
+    env = os.environ if environ is None else environ
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--mode", choices=("render", "collect", "publish"), default="render")
+    parser.add_argument("--source-status", choices=sorted(SOURCE_STATUSES | {""}), default="")
+    parser.add_argument("--base-results", default="")
+    parser.add_argument("--head-results", default="")
+    parser.add_argument("--bundle-results", default="")
+    parser.add_argument("--shard-manifest", default="")
+    parser.add_argument("--shard-directory", default="")
+    parser.add_argument("--output-directory", default="benchmark-report")
+    parser.add_argument("--workflow-path", default="")
+    parser.add_argument("--artifact-name", default="")
+    parser.add_argument("--report-key", default="")
+    parser.add_argument("--sdk", default="")
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "collect":
+            require(not (args.base_results or args.head_results or args.bundle_results), "Ambiguous collection inputs")
+            require(args.shard_manifest and args.shard_directory, "Manifest and shard directory are required")
+            bundle, revisions = collect(args.shard_manifest, args.shard_directory)
+            comparison = compare_bundle(bundle, source_status=args.source_status or None)
+            report = render(comparison, repository=env.get("GITHUB_REPOSITORY"), run_id=env.get("GITHUB_RUN_ID"))
+            directory = Path(args.output_directory)
+            directory.mkdir(parents=True, exist_ok=True)
+            (directory / "bundle.json").write_bytes(encode_bundle(bundle))
+            (directory / "revisions.json").write_text(json_text(revisions), encoding="utf-8")
+        elif args.mode == "render":
+            require(not (args.shard_manifest or args.shard_directory), "Shard inputs require collect mode")
+            if args.bundle_results:
+                require(not (args.base_results or args.head_results), "Ambiguous rendering inputs")
+                comparison = compare_bundle(load_bundle(args.bundle_results), source_status=args.source_status or None)
+            else:
+                require(args.base_results and args.head_results, "Both result paths are required")
+                comparison = compare(
+                    load_result(args.base_results),
+                    load_result(args.head_results),
+                    source_status=args.source_status or None,
+                )
+            report = render(comparison, repository=env.get("GITHUB_REPOSITORY"), run_id=env.get("GITHUB_RUN_ID"))
+        else:
+            require(
+                not (
+                    args.base_results
+                    or args.head_results
+                    or args.bundle_results
+                    or args.shard_manifest
+                    or args.shard_directory
+                ),
+                "Publish accepts only verified artifact inputs",
+            )
+            require(env.get("GITHUB_EVENT_NAME") == "workflow_run", "Publish requires a workflow_run event")
+            require(bool(env.get("GITHUB_EVENT_PATH")), "Missing GitHub event file")
+            with Path(env["GITHUB_EVENT_PATH"]).open("rb") as stream:
+                event = bounded_json(stream.read(MAX_DOCUMENT_BYTES + 1), text_limit=65536)
+            publisher = Publisher(
+                GitHub(env.get("GH_TOKEN", "")),
+                env.get("GITHUB_REPOSITORY", ""),
+                args.workflow_path,
+                args.artifact_name,
+                args.report_key,
+                args.sdk,
+            )
+            comparison, report = publisher.publish(event, report_run_id=env.get("GITHUB_RUN_ID"))
+        write_outputs(args.output_directory, comparison, report, env)
+        return 1 if comparison.get("execution_failed") else 0
+    except (InvalidResult, OSError, ValueError) as exc:
+        # Never print paths, API responses, raw logs or arbitrary exception text.
+        reason = str(exc) if isinstance(exc, InvalidResult) else "Report input or output could not be processed"
+        report = f"## SDK performance\n\n**Reporting failed:** {reason}.\n"
+        try:
+            write_outputs(args.output_directory, {"comparison_version": 1, "error": reason}, report, env)
+        except (OSError, ValueError):
+            pass
+        print(reason, file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())