diff --git a/scripts/plot_bench.py b/scripts/plot_bench.py new file mode 100644 index 0000000..99c1800 --- /dev/null +++ b/scripts/plot_bench.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Compare one metric across bench-results.json runs, or across pipelines. + +One row per benchmark (readable with dozens of projects), sorted by effect size, +percentages relative to a baseline. Each series is a dot (median) with whiskers +(min-max over its iterations); the baseline is a grey band, so the band is the +noise floor -- a dot inside it is not distinguishable from run jitter. + +Two things vary in a result file: which run it came from and which pipeline +compiled it. --compare picks which of the two is put side by side; the other +becomes one panel per value. + + # runs against each other, one panel per pipeline (the default) + ./.venv/bin/python scripts/plot_bench.py \ + base=out-muh1/bench-results.json \ + feat=out-muh3/bench-results.json --min-baseline 1 + + # pipelines against each other, one panel per run + ./.venv/bin/python scripts/plot_bench.py --compare pipelines \ + may07=out-feat2/bench-results.json +""" + +import argparse +import json +import os + +import matplotlib + +matplotlib.use("Agg") +import pandas as pd # noqa: E402 +import seaborn as sns # noqa: E402 +from matplotlib.lines import Line2D # noqa: E402 +from matplotlib.patches import Patch # noqa: E402 +from scipy.stats import gmean # noqa: E402 + +BAND_COLOR = "0.85" + +# metric -> (how to print a baseline value, word for "less", word for "more") +UNITS = { + "wall_time": ("{:.2f}s", "faster", "slower"), + "cpu_time": ("{:.2f}s", "faster", "slower"), + "peak_rss": ("{:.0f}MiB", "smaller", "larger"), + "bytecode_size": ("{:,.0f}B", "smaller", "larger"), + "creation_size": ("{:,.0f}B", "smaller", "larger"), + "runtime_size": ("{:,.0f}B", "smaller", "larger"), + "cycles": ("{:.3g}", "fewer", "more"), + "instructions": ("{:.3g}", "fewer", "more"), +} +GENERIC_UNIT = ("{:.3g}", "lower", "higher") + + +def load(spec): + label, _, path = spec.partition("=") + if not path: + path, label = label, os.path.basename(os.path.dirname(label)) or label + with open(path) as fh: + return label, json.load(fh) + + +def available_metrics(run): + """Metric names that carry a stats block (median/values), not a bare count.""" + return sorted({ + metric + for bench in run.get("results", {}).values() + for pipeline in bench.values() + for metric, block in pipeline.items() + if isinstance(block, dict) and "median" in block + }) + + +def samples(runs, metric): + """Long form: one row per (benchmark, pipeline, run, iteration sample).""" + rows = [] + for label, run in runs: + for bench, pipelines in run.get("results", {}).items(): + for pipeline, blocks in pipelines.items(): + block = blocks.get(metric) + if not isinstance(block, dict) or not block.get("median"): + continue + for value in block.get("values") or [block["median"]]: + rows.append(dict(benchmark=bench, pipeline=pipeline, + run=label, value=value)) + return pd.DataFrame(rows) + + +def against_baseline(df, series, facet, baseline, min_baseline): + """Percentages vs `baseline`'s median, within each (benchmark, facet) cell. + + Cells the baseline does not cover, or that some series is missing from, are + dropped -- a percentage against a missing reference means nothing. + """ + ref = (df[df[series] == baseline].groupby(["benchmark", facet]).value.median() + .rename("ref")) + df = df.merge(ref, on=["benchmark", facet]) + df = df[df.ref >= min_baseline] + complete = df.groupby(["benchmark", facet])[series].transform("nunique") + df = df[complete == df[series].nunique()] + return df.assign(pct=(df.value / df.ref - 1) * 100) + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("runs", nargs="+", metavar="[label=]bench-results.json", + help="with --compare runs, the first one is the baseline") + ap.add_argument("-c", "--compare", choices=("runs", "pipelines"), default="runs", + help="what to put side by side (default: runs)") + ap.add_argument("-b", "--baseline", + help="with --compare pipelines, the reference pipeline " + "(default: the alphabetically first one)") + ap.add_argument("--only", metavar="A,B", + help="restrict to these runs/pipelines (the baseline is " + "always kept); useful when one series is so far from " + "the rest that it flattens the others") + ap.add_argument("-m", "--metric", default="wall_time", + help="metric to plot (default: wall_time); " + "an unknown name lists what the baseline file offers") + ap.add_argument("-o", "--output", help="default: .png") + ap.add_argument("--min-baseline", "--min-seconds", type=float, default=0.0, + dest="min_baseline", + help="drop benchmarks whose baseline value is below this, " + "in the metric's own units") + args = ap.parse_args() + + runs = [load(s) for s in args.runs] + labels = [l for l, _ in runs] + if len(set(labels)) != len(labels): + raise SystemExit(f"run labels must be unique, got: {', '.join(labels)}") + metric = args.metric + output = args.output or f"{metric.replace('_', '-')}.png" + + offered = available_metrics(runs[0][1]) + if metric not in offered: + raise SystemExit(f"unknown metric {metric!r}; {labels[0]} has: {', '.join(offered)}") + unit_fmt, less, more = UNITS.get(metric, GENERIC_UNIT) + + raw = samples(runs, metric) + if raw.empty: + raise SystemExit(f"no benchmark carries {metric}") + + series, facet = ("run", "pipeline") if args.compare == "runs" else ("pipeline", "run") + order_of = {"run": labels, "pipeline": sorted(raw.pipeline.unique())} + baseline = args.baseline or order_of[series][0] + if baseline not in order_of[series]: + raise SystemExit(f"unknown baseline {baseline!r}; " + f"{series}s are: {', '.join(order_of[series])}") + hue_order = [baseline] + [s for s in order_of[series] if s != baseline] + if args.only: + keep = {baseline, *args.only.split(",")} + unknown = keep - set(hue_order) + if unknown: + raise SystemExit(f"--only: unknown {series}(s) {', '.join(sorted(unknown))}; " + f"have: {', '.join(hue_order)}") + hue_order = [s for s in hue_order if s in keep] + raw = raw[raw[series].isin(keep)] + others = hue_order[1:] + if not others: + raise SystemExit( + f"only one {series} to plot; with --compare {args.compare} you need " + f"at least two (pass more files, or --compare " + f"{'pipelines' if args.compare == 'runs' else 'runs'})" + ) + + df = against_baseline(raw, series, facet, baseline, args.min_baseline) + if df.empty: + raise SystemExit( + f"--min-baseline {args.min_baseline:g} dropped everything" + if args.min_baseline else + f"no benchmark has {metric} for every {series} in a common {facet}" + ) + + # the baseline value belongs in the row label only when it is unambiguous: + # with several panels the same benchmark has a different baseline in each + if df[facet].nunique() == 1: + df["benchmark"] += " (" + df.ref.map(unit_fmt.format) + ")" + + medians = df.groupby([facet, "benchmark", series]).pct.median() + # rows are shared across panels, so a benchmark in several panels needs one + # position: sort by its typical effect in the last series listed + order = (medians.xs(hue_order[-1], level=series) + .groupby("benchmark").median().sort_values().index) + colors = sns.color_palette("tab10", len(others)) + + sns.set_theme(style="whitegrid", rc={"axes.edgecolor": "0.8"}) + height = max(0.24 * len(order) + 1.6, 3.2) + g = sns.catplot( + data=df[df[series] != baseline], kind="point", x="pct", y="benchmark", + hue=series, col=facet, order=order, hue_order=others, palette=colors, + estimator="median", errorbar=("pi", 100), linestyle="none", + dodge=0.5 if len(others) > 1 else False, + marker="o", markersize=4, err_kws={"linewidth": 1.2}, + height=height, aspect=7.5 / height, legend=False, + ) + g.refline(x=0, color="0.3", linewidth=1) + g.set_axis_labels( + f"{metric.replace('_', ' ')} vs {baseline} (%) ← {less} / {more} →", "" + ) + for ax in g.axes.flat: + ax.tick_params(axis="y", labelsize=8) + + spread = (df[df[series] == baseline].groupby([facet, "benchmark"]).pct + .agg(["min", "max"])) + + handles = [Line2D([], [], marker="o", linestyle="none", color=c, label=l) + for l, c in zip(others, colors)] + if (spread["max"] > spread["min"]).any(): + handles.append(Patch(facecolor=BAND_COLOR, label=f"{baseline} spread")) + g.figure.legend(handles=handles, loc="center left", bbox_to_anchor=(1.0, 0.5), + frameon=False) + + # keep the axis on the real signal: one outlier iteration on a small benchmark + # must not squash everything else; whiskers past the edge are clipped. Panels + # share the axis, so this spans them all -- no dot can fall off-panel. + lo, hi = min(medians.min(), 0.0), max(medians.max(), 0.0) + pad = max((hi - lo) * 0.35, 0.25) + g.axes.flat[0].set_xlim(lo - pad, hi + pad) + + for ax, panel in zip(g.axes.flat, g.col_names): + # the baseline's own iteration spread, as a band behind each row + for i, bench in enumerate(order): + if (panel, bench) in spread.index: + blo, bhi = spread.loc[(panel, bench)] + ax.barh(i, bhi - blo, left=blo, height=0.56, color=BAND_COLOR, + linewidth=0, zorder=0.5) + + med = medians.xs(panel, level=facet) + summary = " ".join( + f"{s} {(gmean(med.xs(s, level=series) / 100 + 1) - 1) * 100:+.2f}%" + for s in others + ) + ax.set_title(f"[{panel}] {metric} geomean: {summary}", fontsize=10) + print(f"[{panel}] {metric}, " + f"{len(med.index.get_level_values(0).unique())} benchmarks, " + f"baseline {baseline}: {summary}") + + g.figure.savefig(output, dpi=160, bbox_inches="tight") + print(f"wrote {output}") + + +if __name__ == "__main__": + main() diff --git a/src/solc_bench/benchmark.py b/src/solc_bench/benchmark.py index ecafb92..0481e08 100644 --- a/src/solc_bench/benchmark.py +++ b/src/solc_bench/benchmark.py @@ -207,16 +207,13 @@ def _write_error_log(self, result, name, pipeline): log_path.write_text("\n".join(error_messages), encoding="utf-8") return str(log_path) - def run_file(self, input_file, pipeline, no_optimize): + def run_file(self, input_file, pipelines, no_optimize): """Run benchmark on a single .sol or .json input file. - pipeline is a pipeline name (str) or None for all pipelines. + pipelines is a list of pipeline names, or None for all pipelines. """ name = Path(input_file).stem - pipeline_runs = self._pipeline_runs( - [pipeline] if pipeline else DEFAULT_PIPELINES, - no_optimize, - ) + pipeline_runs = self._pipeline_runs(pipelines or DEFAULT_PIPELINES, no_optimize) for label, solc_settings, ethdebug in pipeline_runs: if input_file.endswith(".sol"): @@ -231,13 +228,13 @@ def run_suite( self, benchmark_dir, only, - pipeline, + pipelines, no_optimize, tags=None, ): """Run configured benchmarks from benchmarks.toml. - pipeline is a pipeline name (str) or None for per-project defaults. + pipelines is a list of pipeline names, or None for per-project defaults. tags is a list of lowercase tag names; benchmarks must carry at least one of them to be selected (combined with `only` via AND). """ @@ -266,10 +263,7 @@ def run_suite( ) continue - if pipeline: - pipelines = [pipeline] - else: - pipelines = config.get("pipelines", DEFAULT_PIPELINES) + bench_pipelines = pipelines or config.get("pipelines", DEFAULT_PIPELINES) gas_project_dir = None if config.get("gas"): @@ -287,7 +281,7 @@ def run_suite( ) for label, solc_settings, ethdebug in self._pipeline_runs( - pipelines, + bench_pipelines, no_optimize, ): with override_json_settings( diff --git a/src/solc_bench/cli.py b/src/solc_bench/cli.py index 3c1dbe8..e1b5f52 100644 --- a/src/solc_bench/cli.py +++ b/src/solc_bench/cli.py @@ -15,7 +15,12 @@ compare_compiler_versions, load_results, ) -from solc_bench.config import DEFAULT_RESULT_FILENAME, RUN_PIPELINES, load_benchmarks +from solc_bench.config import ( + DEFAULT_PIPELINES, + DEFAULT_RESULT_FILENAME, + RUN_PIPELINES, + load_benchmarks, +) from solc_bench.extract import extract_inputs from solc_bench.fetch import FetchError, fetch_solc from solc_bench.host import check_variance_factors @@ -113,14 +118,15 @@ def cmd_run(args): for w in check_variance_factors(): print(f"warning: {w}", file=sys.stderr) + pipelines = _resolve_pipelines(args) if args.input_file: - suite.run_file(args.input_file, args.pipeline, args.no_optimize) + suite.run_file(args.input_file, pipelines, args.no_optimize) else: tags = _split_tags(args.tags) suite.run_suite( args.benchmark_dir, args.only, - args.pipeline, + pipelines, args.no_optimize, tags, ) @@ -129,6 +135,20 @@ def cmd_run(args): return 0 +def _resolve_pipelines(args): + """The pipelines `run` was asked for, or None for the per-benchmark default.""" + if args.pipelines: + names = [p.strip() for p in args.pipelines.split(",") if p.strip()] + unknown = [p for p in names if p not in RUN_PIPELINES] + if unknown: + raise ValueError( + f"unknown pipeline(s) {', '.join(unknown)}; " + f"choose from {', '.join(RUN_PIPELINES)}" + ) + return names + return [args.pipeline] if args.pipeline else None + + def cmd_compare(args): if args.pipelines and len(args.results) != 1: raise ValueError("--pipelines cannot be combined with a second file") @@ -471,11 +491,22 @@ def build_parser(): "input_file is given. Populate with `solc-bench extract`." ), ) - run_parser.add_argument( + pipeline_group = run_parser.add_mutually_exclusive_group() + pipeline_group.add_argument( "--pipeline", choices=RUN_PIPELINES, default=None, - help="Compilation pipeline (default: all pipelines)", + help="Single compilation pipeline (default: per-benchmark defaults)", + ) + pipeline_group.add_argument( + "--pipelines", + default=None, + metavar="A,B", + help=( + "Comma-separated pipelines to run for every benchmark, e.g. " + f"'{','.join(DEFAULT_PIPELINES)}'. Overrides the per-benchmark " + "`pipelines` in benchmarks.toml" + ), ) run_parser.add_argument( "--no-optimize",