Add a 'runners' list to longitudinal and flag-effect plots, to have them

only plot a subset of runners. (Flag effects could already do this by
mapping runners to themselves, but it's a bit unwieldy in practice.)

Author: Yhg1s

README.md

@@ -149,6 +149,7 @@ In `bench_runner.toml`, the `longitudinal_plot` table has a `subplots` key which
 - `base`: The base version to compare to. Should be a fully-specified version, e.g. "3.13.0".
 - `version`: The version series to use as a head. Should be a 2-part version, e.g. "3.14"
 - `flags`: (optional) A list of flags to match to for the head versions
+- `runners`: (optional) A list of nicknames of runners to plot. Defaults to all runners.
 
 For example:
 
@@ -157,7 +158,8 @@ For example:
 subplots = [
     { base = "3.10.4", version = "3.11" },
     { base = "3.12.0", version = "3.13" },
-    { base = "3.13.0", version = "3.14" },
+    { base = "3.13.0", version = "3.14", runners = ["linux1", "linux2"] },
+    { base = "3.13.0", version = "3.14", runners = ["windows1", "macos1"] },
     { base = "3.13.0", version = "3.14", flags = ["JIT"] }
 ]
 ```
@@ -172,6 +174,7 @@ In `bench_runner.toml`, the `flag_effect_plot` table has a `subplots` key which
 - `version`: The version series to compare. Should be a 2-part version, e.g. "3.14"
 - `head_flags`: A list of flags to use as the head.
 - `base_flags`: (optional) A list of flags to use as the base. By default, this is a default build, i.e. no flags.
+- `runners`: (optional) A list of nicknames of runners to plot. Defaults to all runners.
 - `runner_map`: (optional) If you need to map a runner to a base in a
   different runner, you can provide that mapping here. For example, with
   tail-calling, you may want to compare runners configured to use clang
@@ -192,6 +195,7 @@ head_flags = ["JIT"]
 name = "Tail calling interpreter"
 version = "3.14"
 head_flags = ["TAILCALL"]
+runners = ["linux_clang"]
 runner_map = { linux_clang = "linux" }
 ```
 

bench_runner/plot.py

@@ -93,6 +93,7 @@ def get_longitudinal_plot_config():
         assert "version" in subplot
         if "flags" not in subplot:
             subplot["flags"] = []
+        subplot["runners"] = set(subplot.get("runners", ()))
 
     return plots
 
@@ -113,6 +114,7 @@ def get_flag_effect_plot_config():
             subplot["base_flags"] = []
         else:
             subplot["base_flags"] = sorted(set(subplot["base_flags"]))
+        subplot["runners"] = set(subplot.get("runners", ()))
         if "runner_map" not in subplot:
             subplot["runner_map"] = {}
 
@@ -329,6 +331,10 @@ def get_comparison_value(ref, r, base):
         ver_results = [
             r for r in results if list(r.parsed_version.release[0:2]) == version
         ]
+        if cfg["runners"]:
+            cfg_runners = [r for r in runners if r.nickname in cfg["runners"]]
+        else:
+            cfg_runners = runners
 
         subtitle = f"Python {cfg['version']}.x vs. {cfg['base']}"
         if len(cfg["flags"]):
@@ -337,7 +343,7 @@ def get_comparison_value(ref, r, base):
 
         first_runner = True
 
-        for runner in runners:
+        for runner in cfg_runners:
             runner_results = [
                 r
                 for r in ver_results
@@ -493,6 +499,8 @@ def get_comparison_value(ref, r, force_valid):
         )
 
         for runner in mrunners.get_runners():
+            if subplot["runners"] and runner.nickname not in subplot["runners"]:
+                continue
             runner_is_mapped = runner.nickname in subplot["runner_map"]
             if subplot["runner_map"] and not runner_is_mapped:
                 continue

tests/data/bench_runner.toml

@@ -12,13 +12,14 @@ plot = { name = "Linux", color = "C0" }
 subplots = [
     { base = "3.10.4", version = "3.11" },
     { base = "3.12.0", version = "3.13" },
-    { base = "3.13.0b2", version = "3.14" },
+    { base = "3.13.0b2", version = "3.14", runners = ["linux"] },
     { base = "3.13.0b2", version = "3.14", flags = ["JIT"] }
 ]
 
 [flag_effect_plot]
 subplots = [
     { name = "JIT", version = "3.14", head_flags = ["JIT"] },
+    { name = "NOGIL", version = "3.14", head_flags = ["NOGIL"], runners = ["linux"] },
 ]
 
 [benchmark_longitudinal_plot]