compare.py

"""Put several models' results side by side, and render the reference table.

Emitted as markdown so RESULTS.md is generated rather than hand-maintained: a table
someone edits by hand drifts from the runs it claims to describe, and the whole point
of publishing a baseline is that a reader can re-derive it.

Every rate here is one pass. That is stated in the output because the benchmark's
own run-to-run noise is wide enough to swallow small differences: two identical
passes over the same generation tasks, byte-identical prompts at temperature 0,
disagreed on 8.7% of them. A ranking is only worth reading where the gap is bigger
than that.
"""

from __future__ import annotations

import glob
import json
import os
import sys
from collections import defaultdict

CLASSES = ["dockerfile", "helm_chart", "workflow", "manifest_set", "compose",
           "terraform_module", "ansible_role"]
TYPES = ["repair", "completion", "generation"]
HARNESS = ("no files parsed", "model error")


def rate(rows: list[dict]) -> tuple[int, int, float]:
    passed = sum(1 for r in rows if r["passed"])
    return passed, len(rows), 100 * passed / max(len(rows), 1)


def newest_per_model(paths: list[str]) -> dict[str, tuple[str, list[dict]]]:
    """Latest full run per model. A partial run is not a baseline."""
    best: dict[str, tuple[str, list[dict]]] = {}
    for path in sorted(paths):
        rows = [json.loads(line) for line in open(path)]
        if len(rows) < 600:
            continue
        model = rows[0].get("model") or os.path.basename(path).rsplit("-", 1)[0]
        best[model] = (path, rows)
    return best


def when(path: str) -> str:
    """The run's timestamp, taken from the filename it was written under."""
    stamp = os.path.basename(path)[: -len(".jsonl")].rsplit("-", 1)[-1]
    try:
        return f"{stamp[:4]}-{stamp[4:6]}-{stamp[6:8]} {stamp[9:11]}:{stamp[11:13]}"
    except IndexError:
        return stamp


def table(models: dict[str, list[dict]], key: str, names: list[str]) -> list[str]:
    header = "| Model | " + " | ".join(f"`{n}`" for n in names) + " |"
    divider = "|---" * (len(names) + 1) + "|"
    lines = [header, divider]
    for model, rows in models.items():
        groups = defaultdict(list)
        for record in rows:
            groups[record[key]].append(record)
        cells = []
        for name in names:
            if not groups[name]:
                cells.append("-")
                continue
            _, _, r = rate(groups[name])
            cells.append(f"{r:.1f}%")
        lines.append(f"| **{model}** | " + " | ".join(cells) + " |")
    return lines


def main() -> int:
    found = newest_per_model(sys.argv[1:] or glob.glob("results/*.jsonl"))
    if not found:
        print("no complete runs found in results/", file=sys.stderr)
        return 2
    ordered = dict(sorted(found.items(), key=lambda kv: -rate(kv[1][1])[2]))
    models = {model: rows for model, (_, rows) in ordered.items()}

    out = ["# Reference results", "",
           "What one-shot answers from a few models score against the tools that gate "
           "real deployments. One pass per model over the same 674 tasks, temperature "
           "0, no retries, no tool access, no iteration, offline tier.", "",
           "Generated by `compare.py` from the run files; do not edit by hand.", "",
           "| Model | Pass rate | Unreadable replies | Output tokens | Run |",
           "|---|---|---|---|---|"]
    for model, (path, rows) in ordered.items():
        passed, total, r = rate(rows)
        unreadable = sum(1 for x in rows if not x["passed"]
                         and any(x["reason"].startswith(h) for h in HARNESS))
        tokens = sum(x.get("usage", {}).get("completion_tokens", 0) for x in rows)
        out.append(f"| **{model}** | **{passed}/{total} = {r:.1f}%** | "
                   f"{unreadable} | {tokens:,} | {when(path)} |")

    out += ["", "### By task type", ""] + table(models, "task_type", TYPES)
    out += ["", "### By class", ""] + table(models, "unit_type", CLASSES)
    out += [
        "",
        "### How to read these",
        "",
        "**One pass is not a measurement.** Two identical passes over the same "
        "generation tasks, with byte-identical prompts at temperature 0, disagreed on "
        "8.7% of them in both directions: continuous batching makes the same prompt a "
        "different computation depending on what else is in the batch. Treat each "
        "number as roughly +-4 points on the generation slice, and do not read a "
        "ranking from gaps narrower than that.",
        "",
        "**Unreadable replies count as failures.** A reply the parser could not turn "
        "into files is scored wrong, because the alternative is quietly excluding a "
        "model's worst output. The column is published so you can see how much of a "
        "rate is that rather than the model's infrastructure code. `analyse.py` breaks "
        "it down per run.",
        "",
        "**These are not equal-footing comparisons of raw capability.** Every model "
        "was prompted identically and given no retries, no tool access and no "
        "iteration, which is not how any of them is used in practice. What the numbers "
        "compare is one-shot output against the tools that gate real deployments.",
        "",
        "**Model names are whatever the endpoint called them.** They are not canonical "
        "upstream identifiers: resolve each against the configuration of the endpoint "
        "the run targeted, since two deployments of the same weights can differ in "
        "quantisation, context length and sampling defaults. Some of these are served "
        "directly, others routed onward to a third-party provider, and that too is a "
        "property of the endpoint rather than of the model.",
    ]
    print("\n".join(out))
    return 0


if __name__ == "__main__":
    sys.exit(main())