Add CLI option solc-extra-flags to allow other flags to be passed to solc

README.md

@@ -157,12 +157,48 @@ the suite and needs no `--benchmark-dir`). Results land in
 | `--stdout` | off | Also print results to stdout |
 | `--pipeline P` | (all) | Single pipeline: `evmasm`/`ir`/`ir-ssacfg`/`ir-ethdebug` |
 | `--no-optimize` | off | Disable the optimizer |
+| `--extra-solc-flags FLAGS` | (none) | Extra flags passed to solc, repeatable (see [Extra solc flags](#extra-solc-flags)) |
 
 ```bash
 solc-bench run --solc ./solc --benchmark-dir ./my-suite --only openzeppelin-5.6.1
 solc-bench run --solc ./solc contract.sol --pipeline ir       # single file
 ```
 
+### Extra solc flags
+
+`--extra-solc-flags` passes optional command-line flags to solc, inserted before
+`--standard-json`.
+For example:
+
+```bash
+solc-bench run --solc ./solc --benchmark-dir ./benchmark_data \
+  --extra-solc-flags='--log-level debug' -o ./logging-on.json
+```
+/
+**Prefer the use of `=`** so it works with every flag.
+
+```bash
+--extra-solc-flags='--optimize-runs 1000'   # always works
+--extra-solc-flags "--optimize-runs 1000"   # always works
+--extra-solc-flags "--optimize"             # does not work
+```
+
+It is possible to group and isolate related flags in different specifications of the option: 
+
+```bash
+--extra-solc-flags='--log-level warn' --extra-solc-flags='--log yul.ssa=debug'
+```
+
+The flags are recorded in the result JSON as `extra_solc_flags` and shown in every `compare` table, so datasets that differ only by flags stay distinguishable.
+`compare` warns when the two sides do not match.
+
+Gas benchmarks are skipped, because forge runs solc itself via `--use` and cannot forward these flags.
+`--standard-json` is rejected; solc-bench passes it by default already.
+
+The flags are validated with a trivial compilation before the suite starts, so a typo fails immediately rather than after the run.
+
+solc's stderr is discarded. Flags that log heavily cost nothing extra in I/O wait, but their output is not retained.
+
 ### ETHDebug overhead
 
 `ir-ethdebug` is a regular pipeline: the same unoptimized IR compilation as

src/solc_bench/benchmark.py

@@ -2,6 +2,7 @@
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
 from pathlib import Path
 
@@ -49,9 +50,10 @@ def _ru_maxrss_mib(ru_maxrss):
 class Benchmark:
     """Runs solc and collects all metrics."""
 
-    def __init__(self, solc, use_perf=None):
+    def __init__(self, solc, use_perf=None, extra_flags=()):
         self.solc = solc
         self.use_perf = use_perf if use_perf is not None else perf_available()
+        self.extra_flags = tuple(extra_flags)
 
     def run(self, input_file, iterations):
         """Run solc N times, return aggregated metrics or None on failure."""
@@ -88,36 +90,56 @@ def invoke_solc(self, input_file):
 
         Returns (metrics_dict, stdout_bytes).
         See https://docs.python.org/3/library/os.html#os.wait4
+
+        The compiler's stderr is always discarded, and perf is told to write its
+        counters to a file rather than stderr. Sharing fd 2 between the two would
+        deadlock: stdout is read to EOF before stderr is drained, so any solc flag
+        that writes more than the 64 KiB pipe buffer to stderr - which
+        --extra-solc-flags makes easy to request - would hang the harness.
         """
-        cmd = [self.solc, "--standard-json"]
-        if self.use_perf:
-            cmd = ["perf", "stat", "-e", "instructions,cycles", "-x", ";", "--", *cmd]
+        cmd = [self.solc, *self.extra_flags, "--standard-json"]
 
-        stderr = subprocess.PIPE if self.use_perf else subprocess.DEVNULL
+        perf_output = None
+        if self.use_perf:
+            handle = tempfile.NamedTemporaryFile(
+                suffix=".perf", prefix="solc-bench-", delete=False
+            )
+            handle.close()
+            perf_output = handle.name
+            cmd = [
+                "perf", "stat",
+                "--output", perf_output,
+                "-e", "instructions,cycles", "-x", ";", "--",
+                *cmd,
+            ]
 
-        with open(input_file, encoding="utf-8") as f:
-            wall_start = time.monotonic()
+        try:
+            with open(input_file, encoding="utf-8") as f:
+                wall_start = time.monotonic()
 
-            proc = subprocess.Popen(
-                cmd, stdin=f, stdout=subprocess.PIPE, stderr=stderr,
-            )
+                proc = subprocess.Popen(
+                    cmd, stdin=f, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                )
 
-            stdout = proc.stdout.read()
-            perf_stderr = proc.stderr.read() if self.use_perf else None
-            _, status, rusage = os.wait4(proc.pid, 0)
-            proc.returncode = os.waitstatus_to_exitcode(status)
+                stdout = proc.stdout.read()
+                _, status, rusage = os.wait4(proc.pid, 0)
+                proc.returncode = os.waitstatus_to_exitcode(status)
 
-            wall_time = time.monotonic() - wall_start
+                wall_time = time.monotonic() - wall_start
 
-        metrics = {
-            "cpu_time": rusage.ru_utime + rusage.ru_stime,
-            "wall_time": wall_time,
-            "peak_rss": _ru_maxrss_mib(rusage.ru_maxrss),
-            "exit_code": proc.returncode,
-        }
+            metrics = {
+                "cpu_time": rusage.ru_utime + rusage.ru_stime,
+                "wall_time": wall_time,
+                "peak_rss": _ru_maxrss_mib(rusage.ru_maxrss),
+                "exit_code": proc.returncode,
+            }
 
-        if self.use_perf:
-            metrics.update(parse_perf_output(perf_stderr.decode(errors="replace")))
+            if self.use_perf:
+                with open(perf_output, encoding="utf-8", errors="replace") as pf:
+                    metrics.update(parse_perf_output(pf.read()))
+        finally:
+            if perf_output is not None:
+                os.unlink(perf_output)
 
         return metrics, stdout
 
@@ -132,9 +154,14 @@ def __init__(
         output_dir,
         keep_inputs=False,
         output_file=None,
+        extra_flags=(),
     ):
+        # Deliberately not passed to get_solc_version: solc rejects some options
+        # outside their input mode, and version detection must not be able to
+        # fail because of a benchmark flag.
         self.solc_version = get_solc_version(solc)
-        self.benchmark = Benchmark(solc)
+        self.extra_flags = tuple(extra_flags)
+        self.benchmark = Benchmark(solc, extra_flags=self.extra_flags)
         self.output_dir = Path(output_dir)
         self.output_dir.mkdir(parents=True, exist_ok=True)
         self.output_file = Path(output_file) if output_file else None
@@ -173,6 +200,16 @@ def _run_gas(self, result, project_dir, name, pipeline, solc_settings):
         if pipeline == "ir-ssacfg":
             # TODO: forge doesn't support --viaSSACFG yet, skip gas for ir-ssacfg
             return
+        if self.extra_flags:
+            # forge drives solc itself via --use and offers no way to forward
+            # extra flags, so gas would be measured without them while the
+            # compile metrics were measured with them. Skip rather than emit a
+            # dataset whose metrics disagree about what was compiled.
+            print(
+                "    [gas] skipped: --extra-solc-flags cannot be forwarded through forge",
+                file=sys.stderr,
+            )
+            return
         via_ir = solc_settings.get("viaIR", False)
         log_path = self.output_dir / f"{name}-{pipeline}.gas.log"
         print("    [gas] running...", file=sys.stderr, end="", flush=True)
@@ -338,7 +375,7 @@ def write_results(self, stdout=False):
             return
 
         output = reporter.build_result_json(
-            self.results, self.solc_version, self.iterations
+            self.results, self.solc_version, self.iterations, self.extra_flags
         )
         result_path = self.output_file or self.output_dir / DEFAULT_RESULT_FILENAME
         reporter.write_result_json(output, result_path, stdout=stdout)

src/solc_bench/cli.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import shlex
 import sys
 from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter
 from collections import Counter
@@ -21,7 +22,7 @@
 from solc_bench.host import check_variance_factors
 from solc_bench.metrics import ALL_METRICS
 from solc_bench import reporter
-from solc_bench.solidity import validate_standard_json
+from solc_bench.solidity import check_solc_flags, validate_standard_json
 from solc_bench.sourcify import extract as extract_sourcify
 
 DEFAULT_ITERATIONS = 3
@@ -42,6 +43,20 @@ def _split_tags(raw):
     return out or None
 
 
+def _split_extra_solc_flags(raw_values):
+    """Flatten repeated --extra-solc-flags values into a single list."""
+    flags = []
+    for raw in raw_values or []:
+        flags.extend(shlex.split(raw))
+
+    if "--standard-json" in flags:
+        raise ValueError(
+            "--extra-solc-flags must not contain --standard-json; "
+            "solc-bench already passes it by default"
+        )
+    return flags
+
+
 def solc_binary(value):
     """argparse type for --solc: existing executable, returned as absolute path."""
     path = Path(value).resolve()
@@ -94,14 +109,20 @@ def cmd_run(args):
             "Populate one with `solc-bench extract`."
         )
 
+    extra_flags = _split_extra_solc_flags(args.extra_solc_flags)
+    check_solc_flags(args.solc, extra_flags)
+
     suite = BenchmarkSuite(
         args.solc,
         args.iterations,
         output_dir,
         keep_inputs=args.keep_inputs,
         output_file=args.output_file,
+        extra_flags=extra_flags,
     )
     print(f"solc: {suite.solc_version}", file=sys.stderr)
+    if extra_flags:
+        print(f"extra solc flags: {' '.join(extra_flags)}", file=sys.stderr)
     print(f"iterations: {args.iterations}", file=sys.stderr)
     perf_str = (
         "available (using hardware counters)"
@@ -483,6 +504,19 @@ def build_parser():
         default=False,
         help="Disable optimizer (default: optimizer enabled)",
     )
+    run_parser.add_argument(
+        "--extra-solc-flags",
+        action="append",
+        default=[],
+        metavar="FLAGS",
+        help=(
+            "Extra command-line flags passed to solc before --standard-json, "
+            "repeatable. Use the '=' form, since a value "
+            "starting with '-' is only accepted as a value when it contains a "
+            "space: --extra-solc-flags='--optimize-runs 1000'. Gas benchmarks "
+            "are skipped, as forge cannot forward these flags."
+        ),
+    )
     run_parser.add_argument(
         "--keep-inputs",
         action="store_true",

src/solc_bench/compare.py

@@ -130,6 +130,7 @@ def _side_meta(result):
     """Pick out the metadata fields that describe a single result file."""
     return {
         "solc_version": result.get("solc_version", "unknown"),
+        "extra_solc_flags": result.get("extra_solc_flags", []),
         "timestamp": result.get("timestamp", ""),
         "iterations": result.get("iterations"),
         "hardware": result.get("hardware", {}),
@@ -165,6 +166,7 @@ def compare_pipelines(results, ref_pipeline, target_pipeline):
 
     return {
         "solc_version": results.get("solc_version", "unknown"),
+        "extra_solc_flags": results.get("extra_solc_flags", []),
         "timestamp": results.get("timestamp", ""),
         "iterations": results.get("iterations"),
         "ref_pipeline": ref_pipeline,

src/solc_bench/reporter.py

@@ -133,10 +133,11 @@ def missing_input_file(name, input_file, source, version, benchmark_dir):
     )
 
 
-def build_result_json(results, solc_version, iterations):
+def build_result_json(results, solc_version, iterations, extra_solc_flags=()):
     return {
         "solc_bench_version": VERSION,
         "solc_version": solc_version,
+        "extra_solc_flags": list(extra_solc_flags),
         "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
         "iterations": iterations,
         "hardware": host.hardware(),
@@ -178,8 +179,14 @@ def _format_metric_cell(comparison, side, metric):
 def cross_version_table(result):
     baseline = result["baseline"]
     target = result["target"]
-    print(f"Baseline: {baseline['solc_version']}{_iterations_suffix(baseline)}")
-    print(f"Target:   {target['solc_version']}{_iterations_suffix(target)}")
+    print(
+        f"Baseline: {baseline['solc_version']}{_iterations_suffix(baseline)}"
+        f"{extra_solc_flags_suffix(baseline)}"
+    )
+    print(
+        f"Target:   {target['solc_version']}{_iterations_suffix(target)}"
+        f"{extra_solc_flags_suffix(target)}"
+    )
     print(
         "Values are mean \u00b1 sample stddev. \u0394% = "
         "(target mean - baseline mean) / baseline mean. Negative = "
@@ -190,6 +197,7 @@ def cross_version_table(result):
         f"|Δ%| ≥ {MIN_DELTA_PCT:g}%."
     )
     _print_host_mismatch_banner(baseline, target)
+    _print_flags_mismatch_banner(baseline, target)
     print()
 
     metric_names = list(dict.fromkeys(
@@ -298,7 +306,7 @@ def cross_version_per_function_table(result, sort_by="median", max_func_width=60
 
 
 def cross_pipeline_table(result):
-    print(f"solc:      {result['solc_version']}")
+    print(f"solc: {result['solc_version']}{extra_solc_flags_suffix(result)}")
     print(f"timestamp: {result['timestamp']}")
     if result.get("iterations") is not None:
         print(f"iterations: {result['iterations']}")
@@ -363,7 +371,7 @@ def dataset_pairs_table(result):
     print("Datasets:")
     for label, dataset in result["datasets"].items():
         print(
-            f"  {label}: {dataset['solc_version']} "
+            f"  {label}: {dataset['solc_version']}{extra_solc_flags_suffix(dataset)} "
             f"({dataset['pipeline']}{_iterations_suffix(dataset, ', ')}, "
             f"{dataset['path']})"
         )
@@ -387,6 +395,9 @@ def dataset_pairs_table(result):
         print()
         print(f"Comparison: {target} vs {ref}")
         _print_host_mismatch_banner(result["datasets"][ref], result["datasets"][target])
+        _print_flags_mismatch_banner(
+            result["datasets"][ref], result["datasets"][target]
+        )
         print()
 
         metric_names = list(
@@ -451,3 +462,24 @@ def _format_winner(delta_pct, significant, target, ref):
 def _iterations_suffix(meta, prefix=" "):
     iterations = meta.get("iterations")
     return f"{prefix}n={iterations}" if iterations is not None else ""
+
+
+def extra_solc_flags_suffix(meta, prefix=" "):
+    """Render the extra solc flags a dataset was measured with, or ''."""
+    flags = meta.get("extra_solc_flags") or []
+    return f"{prefix}[solc {' '.join(flags)}]" if flags else ""
+
+
+def _print_flags_mismatch_banner(baseline_meta, target_meta):
+    """Warn when the two sides were compiled with different extra solc flags."""
+    baseline_flags = baseline_meta.get("extra_solc_flags") or []
+    target_flags = target_meta.get("extra_solc_flags") or []
+    if baseline_flags == target_flags:
+        return
+    print()
+    print(
+        "warning: the two sides were measured with different --extra-solc-flags; "
+        "the delta includes that difference."
+    )
+    print(f"  baseline: {' '.join(baseline_flags) or '(none)'}")
+    print(f"  target:   {' '.join(target_flags) or '(none)'}")