From 378be9a0d00f5ad3cc569d34ceb2884634c35089 Mon Sep 17 00:00:00 2001 From: Matheus Aguiar Date: Tue, 4 Aug 2026 00:33:11 -0300 Subject: [PATCH 1/2] README --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 1d097b1..eaaa440 100644 --- a/README.md +++ b/README.md @@ -157,12 +157,48 @@ the suite and needs no `--benchmark-dir`). Results land in | `--stdout` | off | Also print results to stdout | | `--pipeline P` | (all) | Single pipeline: `evmasm`/`ir`/`ir-ssacfg`/`ir-ethdebug` | | `--no-optimize` | off | Disable the optimizer | +| `--extra-solc-flags FLAGS` | (none) | Extra flags passed to solc, repeatable (see [Extra solc flags](#extra-solc-flags)) | ```bash solc-bench run --solc ./solc --benchmark-dir ./my-suite --only openzeppelin-5.6.1 solc-bench run --solc ./solc contract.sol --pipeline ir # single file ``` +### Extra solc flags + +`--extra-solc-flags` passes optional command-line flags to solc, inserted before +`--standard-json`. +For example: + +```bash +solc-bench run --solc ./solc --benchmark-dir ./benchmark_data \ + --extra-solc-flags='--log-level debug' -o ./logging-on.json +``` +/ +**Prefer the use of `=`** so it works with every flag. + +```bash +--extra-solc-flags='--optimize-runs 1000' # always works +--extra-solc-flags "--optimize-runs 1000" # always works +--extra-solc-flags "--optimize" # does not work +``` + +It is possible to group and isolate related flags in different specifications of the option: + +```bash +--extra-solc-flags='--log-level warn' --extra-solc-flags='--log yul.ssa=debug' +``` + +The flags are recorded in the result JSON as `extra_solc_flags` and shown in every `compare` table, so datasets that differ only by flags stay distinguishable. +`compare` warns when the two sides do not match. + +Gas benchmarks are skipped, because forge runs solc itself via `--use` and cannot forward these flags. +`--standard-json` is rejected; solc-bench passes it by default already. + +The flags are validated with a trivial compilation before the suite starts, so a typo fails immediately rather than after the run. + +solc's stderr is discarded. Flags that log heavily cost nothing extra in I/O wait, but their output is not retained. + ### ETHDebug overhead `ir-ethdebug` is a regular pipeline: the same unoptimized IR compilation as From 7086ddf6997c02c7c620c8cf37aedf4d2124ddc3 Mon Sep 17 00:00:00 2001 From: Matheus Aguiar Date: Tue, 4 Aug 2026 17:50:55 -0300 Subject: [PATCH 2/2] Add extra-solc-flags option to allow additional flags to be passed to solc --- src/solc_bench/benchmark.py | 87 ++++++++++++++++++++++++++----------- src/solc_bench/cli.py | 36 ++++++++++++++- src/solc_bench/compare.py | 2 + src/solc_bench/reporter.py | 42 +++++++++++++++--- src/solc_bench/solidity.py | 50 +++++++++++++++++++++ 5 files changed, 186 insertions(+), 31 deletions(-) diff --git a/src/solc_bench/benchmark.py b/src/solc_bench/benchmark.py index a11b2a5..73e134d 100644 --- a/src/solc_bench/benchmark.py +++ b/src/solc_bench/benchmark.py @@ -2,6 +2,7 @@ import shutil import subprocess import sys +import tempfile import time from pathlib import Path @@ -49,9 +50,10 @@ def _ru_maxrss_mib(ru_maxrss): class Benchmark: """Runs solc and collects all metrics.""" - def __init__(self, solc, use_perf=None): + def __init__(self, solc, use_perf=None, extra_flags=()): self.solc = solc self.use_perf = use_perf if use_perf is not None else perf_available() + self.extra_flags = tuple(extra_flags) def run(self, input_file, iterations): """Run solc N times, return aggregated metrics or None on failure.""" @@ -88,36 +90,56 @@ def invoke_solc(self, input_file): Returns (metrics_dict, stdout_bytes). See https://docs.python.org/3/library/os.html#os.wait4 + + The compiler's stderr is always discarded, and perf is told to write its + counters to a file rather than stderr. Sharing fd 2 between the two would + deadlock: stdout is read to EOF before stderr is drained, so any solc flag + that writes more than the 64 KiB pipe buffer to stderr - which + --extra-solc-flags makes easy to request - would hang the harness. """ - cmd = [self.solc, "--standard-json"] - if self.use_perf: - cmd = ["perf", "stat", "-e", "instructions,cycles", "-x", ";", "--", *cmd] + cmd = [self.solc, *self.extra_flags, "--standard-json"] - stderr = subprocess.PIPE if self.use_perf else subprocess.DEVNULL + perf_output = None + if self.use_perf: + handle = tempfile.NamedTemporaryFile( + suffix=".perf", prefix="solc-bench-", delete=False + ) + handle.close() + perf_output = handle.name + cmd = [ + "perf", "stat", + "--output", perf_output, + "-e", "instructions,cycles", "-x", ";", "--", + *cmd, + ] - with open(input_file, encoding="utf-8") as f: - wall_start = time.monotonic() + try: + with open(input_file, encoding="utf-8") as f: + wall_start = time.monotonic() - proc = subprocess.Popen( - cmd, stdin=f, stdout=subprocess.PIPE, stderr=stderr, - ) + proc = subprocess.Popen( + cmd, stdin=f, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) - stdout = proc.stdout.read() - perf_stderr = proc.stderr.read() if self.use_perf else None - _, status, rusage = os.wait4(proc.pid, 0) - proc.returncode = os.waitstatus_to_exitcode(status) + stdout = proc.stdout.read() + _, status, rusage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(status) - wall_time = time.monotonic() - wall_start + wall_time = time.monotonic() - wall_start - metrics = { - "cpu_time": rusage.ru_utime + rusage.ru_stime, - "wall_time": wall_time, - "peak_rss": _ru_maxrss_mib(rusage.ru_maxrss), - "exit_code": proc.returncode, - } + metrics = { + "cpu_time": rusage.ru_utime + rusage.ru_stime, + "wall_time": wall_time, + "peak_rss": _ru_maxrss_mib(rusage.ru_maxrss), + "exit_code": proc.returncode, + } - if self.use_perf: - metrics.update(parse_perf_output(perf_stderr.decode(errors="replace"))) + if self.use_perf: + with open(perf_output, encoding="utf-8", errors="replace") as pf: + metrics.update(parse_perf_output(pf.read())) + finally: + if perf_output is not None: + os.unlink(perf_output) return metrics, stdout @@ -132,9 +154,14 @@ def __init__( output_dir, keep_inputs=False, output_file=None, + extra_flags=(), ): + # Deliberately not passed to get_solc_version: solc rejects some options + # outside their input mode, and version detection must not be able to + # fail because of a benchmark flag. self.solc_version = get_solc_version(solc) - self.benchmark = Benchmark(solc) + self.extra_flags = tuple(extra_flags) + self.benchmark = Benchmark(solc, extra_flags=self.extra_flags) self.output_dir = Path(output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) self.output_file = Path(output_file) if output_file else None @@ -173,6 +200,16 @@ def _run_gas(self, result, project_dir, name, pipeline, solc_settings): if pipeline == "ir-ssacfg": # TODO: forge doesn't support --viaSSACFG yet, skip gas for ir-ssacfg return + if self.extra_flags: + # forge drives solc itself via --use and offers no way to forward + # extra flags, so gas would be measured without them while the + # compile metrics were measured with them. Skip rather than emit a + # dataset whose metrics disagree about what was compiled. + print( + " [gas] skipped: --extra-solc-flags cannot be forwarded through forge", + file=sys.stderr, + ) + return via_ir = solc_settings.get("viaIR", False) log_path = self.output_dir / f"{name}-{pipeline}.gas.log" print(" [gas] running...", file=sys.stderr, end="", flush=True) @@ -338,7 +375,7 @@ def write_results(self, stdout=False): return output = reporter.build_result_json( - self.results, self.solc_version, self.iterations + self.results, self.solc_version, self.iterations, self.extra_flags ) result_path = self.output_file or self.output_dir / DEFAULT_RESULT_FILENAME reporter.write_result_json(output, result_path, stdout=stdout) diff --git a/src/solc_bench/cli.py b/src/solc_bench/cli.py index 3c1dbe8..8b9db12 100644 --- a/src/solc_bench/cli.py +++ b/src/solc_bench/cli.py @@ -2,6 +2,7 @@ import json import os +import shlex import sys from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter from collections import Counter @@ -21,7 +22,7 @@ from solc_bench.host import check_variance_factors from solc_bench.metrics import ALL_METRICS from solc_bench import reporter -from solc_bench.solidity import validate_standard_json +from solc_bench.solidity import check_solc_flags, validate_standard_json from solc_bench.sourcify import extract as extract_sourcify DEFAULT_ITERATIONS = 3 @@ -42,6 +43,20 @@ def _split_tags(raw): return out or None +def _split_extra_solc_flags(raw_values): + """Flatten repeated --extra-solc-flags values into a single list.""" + flags = [] + for raw in raw_values or []: + flags.extend(shlex.split(raw)) + + if "--standard-json" in flags: + raise ValueError( + "--extra-solc-flags must not contain --standard-json; " + "solc-bench already passes it by default" + ) + return flags + + def solc_binary(value): """argparse type for --solc: existing executable, returned as absolute path.""" path = Path(value).resolve() @@ -94,14 +109,20 @@ def cmd_run(args): "Populate one with `solc-bench extract`." ) + extra_flags = _split_extra_solc_flags(args.extra_solc_flags) + check_solc_flags(args.solc, extra_flags) + suite = BenchmarkSuite( args.solc, args.iterations, output_dir, keep_inputs=args.keep_inputs, output_file=args.output_file, + extra_flags=extra_flags, ) print(f"solc: {suite.solc_version}", file=sys.stderr) + if extra_flags: + print(f"extra solc flags: {' '.join(extra_flags)}", file=sys.stderr) print(f"iterations: {args.iterations}", file=sys.stderr) perf_str = ( "available (using hardware counters)" @@ -483,6 +504,19 @@ def build_parser(): default=False, help="Disable optimizer (default: optimizer enabled)", ) + run_parser.add_argument( + "--extra-solc-flags", + action="append", + default=[], + metavar="FLAGS", + help=( + "Extra command-line flags passed to solc before --standard-json, " + "repeatable. Use the '=' form, since a value " + "starting with '-' is only accepted as a value when it contains a " + "space: --extra-solc-flags='--optimize-runs 1000'. Gas benchmarks " + "are skipped, as forge cannot forward these flags." + ), + ) run_parser.add_argument( "--keep-inputs", action="store_true", diff --git a/src/solc_bench/compare.py b/src/solc_bench/compare.py index 956ab7c..0536a5b 100644 --- a/src/solc_bench/compare.py +++ b/src/solc_bench/compare.py @@ -130,6 +130,7 @@ def _side_meta(result): """Pick out the metadata fields that describe a single result file.""" return { "solc_version": result.get("solc_version", "unknown"), + "extra_solc_flags": result.get("extra_solc_flags", []), "timestamp": result.get("timestamp", ""), "iterations": result.get("iterations"), "hardware": result.get("hardware", {}), @@ -165,6 +166,7 @@ def compare_pipelines(results, ref_pipeline, target_pipeline): return { "solc_version": results.get("solc_version", "unknown"), + "extra_solc_flags": results.get("extra_solc_flags", []), "timestamp": results.get("timestamp", ""), "iterations": results.get("iterations"), "ref_pipeline": ref_pipeline, diff --git a/src/solc_bench/reporter.py b/src/solc_bench/reporter.py index df3b2b4..544b753 100644 --- a/src/solc_bench/reporter.py +++ b/src/solc_bench/reporter.py @@ -133,10 +133,11 @@ def missing_input_file(name, input_file, source, version, benchmark_dir): ) -def build_result_json(results, solc_version, iterations): +def build_result_json(results, solc_version, iterations, extra_solc_flags=()): return { "solc_bench_version": VERSION, "solc_version": solc_version, + "extra_solc_flags": list(extra_solc_flags), "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), "iterations": iterations, "hardware": host.hardware(), @@ -178,8 +179,14 @@ def _format_metric_cell(comparison, side, metric): def cross_version_table(result): baseline = result["baseline"] target = result["target"] - print(f"Baseline: {baseline['solc_version']}{_iterations_suffix(baseline)}") - print(f"Target: {target['solc_version']}{_iterations_suffix(target)}") + print( + f"Baseline: {baseline['solc_version']}{_iterations_suffix(baseline)}" + f"{extra_solc_flags_suffix(baseline)}" + ) + print( + f"Target: {target['solc_version']}{_iterations_suffix(target)}" + f"{extra_solc_flags_suffix(target)}" + ) print( "Values are mean \u00b1 sample stddev. \u0394% = " "(target mean - baseline mean) / baseline mean. Negative = " @@ -190,6 +197,7 @@ def cross_version_table(result): f"|Δ%| ≥ {MIN_DELTA_PCT:g}%." ) _print_host_mismatch_banner(baseline, target) + _print_flags_mismatch_banner(baseline, target) print() metric_names = list(dict.fromkeys( @@ -298,7 +306,7 @@ def cross_version_per_function_table(result, sort_by="median", max_func_width=60 def cross_pipeline_table(result): - print(f"solc: {result['solc_version']}") + print(f"solc: {result['solc_version']}{extra_solc_flags_suffix(result)}") print(f"timestamp: {result['timestamp']}") if result.get("iterations") is not None: print(f"iterations: {result['iterations']}") @@ -363,7 +371,7 @@ def dataset_pairs_table(result): print("Datasets:") for label, dataset in result["datasets"].items(): print( - f" {label}: {dataset['solc_version']} " + f" {label}: {dataset['solc_version']}{extra_solc_flags_suffix(dataset)} " f"({dataset['pipeline']}{_iterations_suffix(dataset, ', ')}, " f"{dataset['path']})" ) @@ -387,6 +395,9 @@ def dataset_pairs_table(result): print() print(f"Comparison: {target} vs {ref}") _print_host_mismatch_banner(result["datasets"][ref], result["datasets"][target]) + _print_flags_mismatch_banner( + result["datasets"][ref], result["datasets"][target] + ) print() metric_names = list( @@ -451,3 +462,24 @@ def _format_winner(delta_pct, significant, target, ref): def _iterations_suffix(meta, prefix=" "): iterations = meta.get("iterations") return f"{prefix}n={iterations}" if iterations is not None else "" + + +def extra_solc_flags_suffix(meta, prefix=" "): + """Render the extra solc flags a dataset was measured with, or ''.""" + flags = meta.get("extra_solc_flags") or [] + return f"{prefix}[solc {' '.join(flags)}]" if flags else "" + + +def _print_flags_mismatch_banner(baseline_meta, target_meta): + """Warn when the two sides were compiled with different extra solc flags.""" + baseline_flags = baseline_meta.get("extra_solc_flags") or [] + target_flags = target_meta.get("extra_solc_flags") or [] + if baseline_flags == target_flags: + return + print() + print( + "warning: the two sides were measured with different --extra-solc-flags; " + "the delta includes that difference." + ) + print(f" baseline: {' '.join(baseline_flags) or '(none)'}") + print(f" target: {' '.join(target_flags) or '(none)'}") diff --git a/src/solc_bench/solidity.py b/src/solc_bench/solidity.py index 96570b9..4e8cc40 100644 --- a/src/solc_bench/solidity.py +++ b/src/solc_bench/solidity.py @@ -51,6 +51,56 @@ def get_solc_version(solc): raise ValueError(f"not a solc binary: {solc}") +_PREFLIGHT_INPUT = { + "language": "Solidity", + "sources": {"preflight.sol": {"content": "contract C {}"}}, + "settings": {"outputSelection": {"*": {"*": ["evm.bytecode.object"]}}}, +} + + +def check_solc_flags(solc, extra_flags): + """Compile a trivial input to verify extra solc flags are actually usable. + + Raises ValueError carrying solc's own diagnostics. Without this a typo only + surfaces once the suite has already been running for a while, and a flag + that makes solc emit non-JSON would instead show up as every benchmark + silently producing no metrics. + """ + if not extra_flags: + return + + quoted = " ".join(extra_flags) + result = subprocess.run( + [solc, *extra_flags, "--standard-json"], + input=json.dumps(_PREFLIGHT_INPUT), + capture_output=True, + text=True, + ) + + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip().splitlines() + raise ValueError( + f"solc rejected --extra-solc-flags {quoted!r} " + f"(exit {result.returncode}): {detail[0] if detail else 'no diagnostics'}" + ) + + try: + output = json.loads(result.stdout) + except json.JSONDecodeError: + raise ValueError( + f"--extra-solc-flags {quoted!r} made solc emit output that is not " + "standard-json; benchmarks would record no metrics" + ) + + errors = [e for e in output.get("errors", []) if e.get("severity") == "error"] + if errors: + message = errors[0].get("formattedMessage") or errors[0].get("message", "") + raise ValueError( + f"--extra-solc-flags {quoted!r} caused a compilation error: " + f"{message.strip().splitlines()[0] if message else 'unknown error'}" + ) + + def _serialized_json_size(value): return len(json.dumps(value, separators=(",", ":"), sort_keys=True).encode("utf-8"))