Limit cores on specific machines

Author: mdboom

README.md

@@ -93,6 +93,12 @@ If you don't want a machine to be included when the user selects "machine == 'al
 include_in_all = false
 ```
 
+You may limit the number of cores used to build Python with the `use_cores` option. This may be necessary, for example, on cloud VMs.
+
+```
+use_cores = 2
+```
+
 ### Try a benchmarking run
 
 There are instructions for running a benchmarking action already in the `README.md` of your repo. Look there and give it a try!

bench_runner/config.py

@@ -4,16 +4,26 @@
 
 import functools
 from pathlib import Path
+from typing import Any
 
 try:
     import tomllib
 except ImportError:
     import tomli as tomllib  # type: ignore
 
 
+from . import runners
+
+
 @functools.cache
 def get_bench_runner_config(
     filepath: Path | str = Path("bench_runner.toml"),
 ):
     with Path(filepath).open("rb") as fd:
         return tomllib.load(fd)
+
+
+def get_config_for_current_runner() -> dict[str, Any]:
+    config = get_bench_runner_config()
+    runner = runners.get_runner_for_hostname()
+    return config.get("runners", {}).get(runner.nickname, {})

bench_runner/result.py

@@ -9,7 +9,6 @@
 from operator import itemgetter
 from pathlib import Path
 import re
-import socket
 import subprocess
 import sys
 from typing import Any, Callable, Iterable, Sequence
@@ -524,7 +523,7 @@ def from_scratch(
         flags: Iterable[str] | None = None,
     ) -> "Result":
         result = cls(
-            _clean(runners.get_nickname_for_hostname(socket.gethostname())),
+            _clean(runners.get_nickname_for_hostname()),
             _clean(_get_architecture(python)),
             _clean_for_url(fork),
             _clean(ref[:20]),

bench_runner/runners.py

@@ -3,6 +3,7 @@
 
 import functools
 import os
+import socket
 
 
 from . import config
@@ -80,13 +81,19 @@ def get_runners_by_nickname() -> dict[str, Runner]:
     return {x.nickname: x for x in get_runners()}
 
 
-def get_nickname_for_hostname(hostname: str) -> str:
+def get_nickname_for_hostname(hostname: str | None = None) -> str:
     # The envvar BENCHMARK_MACHINE_NICKNAME is used to override the machine that
     # results are reported for.
     if "BENCHMARK_MACHINE_NICKNAME" in os.environ:
         return os.environ["BENCHMARK_MACHINE_NICKNAME"]
-    return get_runners_by_hostname().get(hostname, unknown_runner).nickname
+    return get_runner_for_hostname(hostname).nickname
 
 
 def get_runner_by_nickname(nickname: str) -> Runner:
     return get_runners_by_nickname().get(nickname, unknown_runner)
+
+
+def get_runner_for_hostname(hostname: str | None = None) -> Runner:
+    if hostname is None:
+        hostname = socket.gethostname()
+    return get_runners_by_hostname().get(hostname, unknown_runner)

bench_runner/scripts/workflow.py

@@ -15,6 +15,7 @@
 
 
 from bench_runner import benchmark_definitions
+from bench_runner import config
 from bench_runner import flags as mflags
 from bench_runner import git
 from bench_runner.result import has_result
@@ -132,6 +133,7 @@ def checkout_benchmarks():
 
 def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool) -> None:
     cpython = Path(cpython)
+    cfg = config.get_config_for_current_runner()
 
     env = os.environ.copy()
     if "CLANG" in flags:
@@ -169,9 +171,15 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
     if configure_flags := os.environ.get("PYTHON_CONFIGURE_FLAGS"):
         args.extend(shlex.split(configure_flags))
 
+    make_args = []
+    if cores := cfg.get("use_cores", None):
+        make_args.extend(["-j", str(cores)])
+    else:
+        make_args.extend(["-j"])
+
     with contextlib.chdir(cpython):
         subprocess.check_call(["./configure", *args], env=env)
-        subprocess.check_call(["make", "-j"], env=env)
+        subprocess.check_call(["make", *make_args], env=env)
 
 
 def compile_windows(