Merge pull request #367 from faster-cpython/reduce-all

Don't include all machines in "all"

Author: mdboom

README.md

@@ -44,7 +44,7 @@ git push origin main
 
 The `bench_runner.toml` file created at the root of your repository contains configuration specific to your instance.
 More details about this configuration are below.
-Every time you make a change to the `bench_runner.toml` file, you will need to rerun `python -m bench_runer install` to have the changes reflected.
+Every time you make a change to the `bench_runner.toml` file, you will need to rerun `python -m bench_runner install` to have the changes reflected.
 
 ### Add some self-hosted runners
 
@@ -87,6 +87,12 @@ hostname = "cpython-benchmarking-azure"
 available = false
 ```
 
+If you don't want a machine to be included when the user selects "machine == 'all'", add:
+
+```
+include_in_all = false
+```
+
 ### Try a benchmarking run
 
 There are instructions for running a benchmarking action already in the `README.md` of your repo. Look there and give it a try!
@@ -144,7 +150,7 @@ python -m bench_runner purge
 
 To see more options that control what is deleted, run `python -m bench_runner purge --help`.
 
-After purging the results, you will usually want to squash the git history down to a single commit to save space in your repository.  **NOTE THAT THIS IS A DESTRUCTIVE OPERATION THAT WILL DELETE OLD DATA.**
+After purging the results, you will usually want to squash the git history down to a single commit to save space in your repository. **NOTE THAT THIS IS A DESTRUCTIVE OPERATION THAT WILL DELETE OLD DATA.**
 
 ```
 git checkout --orphan new-main main
@@ -155,7 +161,7 @@ git branch -M new-main main
 git push -f origin main
 ```
 
-### Running 
+### Running
 
 ## Developer
 

bench_runner/runners.py

@@ -20,6 +20,7 @@ def __init__(
         # Override the Github self-hosted runner name if different from
         # os-arch-nickname
         github_runner_name: str | None,
+        include_in_all: bool = True,
     ):
         self.nickname = nickname
         self.os = os
@@ -30,6 +31,7 @@ def __init__(
         if github_runner_name is None:
             github_runner_name = self.name
         self.github_runner_name = github_runner_name
+        self.include_in_all = include_in_all
 
     @property
     def name(self) -> str:
@@ -54,6 +56,7 @@ def get_runners() -> list[Runner]:
                 section.get("available", True),
                 section.get("env", {}),
                 section.get("github_runner_name"),
+                section.get("include_in_all", True),
             )
         )
 

bench_runner/scripts/install.py

@@ -147,9 +147,15 @@ def generate__benchmark(src: Any) -> Any:
         runner_template["steps"].insert(0, setup_environment)
 
         runner_template["runs-on"].append(runner.github_runner_name)
-        runner_template["if"] = (
-            f"${{{{ (inputs.machine == '{runner.name}' || inputs.machine == 'all') }}}}"
-        )
+
+        machine_clauses = [
+            f"inputs.machine == '{runner.name}'",
+            "inputs.machine == '__really_all'",
+        ]
+        if runner.include_in_all:
+            machine_clauses.append("inputs.machine == 'all'")
+        runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
+
         dst["jobs"][f"benchmark-{runner.name}"] = runner_template
 
     add_flag_env(dst["jobs"])
@@ -174,7 +180,7 @@ def generate_benchmark(dst: Any) -> Any:
     user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    runner_choices = [*[x.name for x in available_runners], "all"]
+    runner_choices = [*[x.name for x in available_runners], "all", "__really_all"]
 
     dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices
 

bench_runner/templates/_weekly.src.yml

@@ -34,7 +34,7 @@ jobs:
     with:
       fork: python
       ref: ${{ needs.determine_head.outputs.commit }}
-      machine: all
+      machine: __really_all
       benchmarks: all_and_excluded
       pgo: true
       tier2: false
@@ -46,7 +46,7 @@ jobs:
     with:
       fork: python
       ref: ${{ needs.determine_head.outputs.commit }}
-      machine: all
+      machine: __really_all
       benchmarks: all_and_excluded
       pgo: true
       tier2: false
@@ -59,7 +59,7 @@ jobs:
     with:
       fork: python
       ref: ${{ needs.determine_head.outputs.commit }}
-      machine: all
+      machine: __really_all
       benchmarks: all_and_excluded
       pgo: true
       tier2: false
@@ -73,7 +73,7 @@ jobs:
     with:
       fork: python
       ref: ${{ needs.determine_head.outputs.commit }}
-      machine: all
+      machine: __really_all
       benchmarks: all_and_excluded
       pgo: true
       tier2: false