Fix #422: Don't hardcode 4th plot as JIT

Author: mdboom

CHANGELOG.md

@@ -83,6 +83,21 @@ flags = ["TAILCALL"]
 runners = ["linux_clang", "darwin", "windows_clang"]
 ```
 
+### Configurable flags in longitudinal plots
+
+The flags to use for each of the longitudinal plots is now configurable. (Previously, the fourth one was always for JIT).
+
+The plot configuration, for example, will create two plots. The first will
+compare default builds on the 3.14 branch against 3.13.0. The second will
+compare JIT builds on the 3.14 branch against 3.13.0.
+
+```
+[plot]
+bases = ["3.13.0", "3.13.0"]
+versions = [[3, 14], [3, 14]]
+flags = [[], ["JIT"]]
+```
+
 ### Long-running tests
 
 `bench-runner` has some new long-running end-to-end integration tests. To avoid running them, use:

bench_runner/plot.py

@@ -89,6 +89,9 @@ def get_plot_config():
         == len(content["markers"])
     )
 
+    if "flags" not in content:
+        content["flags"] = [[]] * len(content["bases"])
+
     return content
 
 
@@ -276,8 +279,8 @@ def get_comparison_value(ref, r, base):
         ]
 
         subtitle = f"Python {version_str}.x vs. {base}"
-        if i == 3:
-            subtitle += " (with JIT)"
+        if len(cfg["flags"][i]):
+            subtitle += f" ({','.join(cfg['flags'][i])})"
         ax.set_title(subtitle)
 
         for runner_i, (runner, name, color, style, marker) in enumerate(
@@ -289,13 +292,11 @@ def get_comparison_value(ref, r, base):
                 cfg["markers"],
             )
         ):
-            runner_results = [r for r in ver_results if r.nickname == runner]
-
-            # The last plot is JIT-only
-            if i == 3:
-                runner_results = [r for r in runner_results if r.flags == ["JIT"]]
-            else:
-                runner_results = [r for r in runner_results if r.flags == []]
+            runner_results = [
+                r
+                for r in ver_results
+                if r.nickname == runner and set(r.flags) == set(cfg["flags"][i])
+            ]
 
             for r in results:
                 if r.nickname == runner and r.version == base and r.flags == []:

bench_runner/templates/bench_runner.toml

@@ -59,6 +59,8 @@ env.BUILD_DEST = "PCbuild/amd64"
 [plot]
 bases = ["3.11.0"]
 versions = [[3, 12]]
+# The flags to use for each of the versions. The base is always "default" (without flags)
+flags = [[]]
 runners = ["linux"]
 names = ["linux"]
 colors = ["C0"]