Simplify tests

Author: mdboom

.github/workflows/ci.yml

@@ -42,10 +42,6 @@ jobs:
   long_test_linux:
     runs-on: ubuntu-latest
     steps:
-      - name: Install packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y linux-tools-common linux-tools-generic
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:

CHANGELOG.md

@@ -83,6 +83,14 @@ flags = ["TAILCALL"]
 runners = ["linux_clang", "darwin", "windows_clang"]
 ```
 
+### Long-running tests
+
+`bench-runner` has some new long-running end-to-end integration tests. To avoid running them, use:
+
+```
+python -m pytest -m "not long_running"
+```
+
 ## v1.8.0
 
 ### bench_runner.toml change

tests/test_notify.py

@@ -1,6 +1,5 @@
 import contextlib
 from pathlib import Path
-import textwrap
 
 
 from bench_runner import gh
@@ -31,17 +30,18 @@ def send_notification(message):
         )
 
     captured = capsys.readouterr()
-    assert (
-        captured.out.strip()
-        == "::notice ::@test_actor: [test_fork/test_ref](https://github.com/test_repo-public/tree/main/results/bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
+    assert captured.out.strip() == (
+        "::notice ::@test_actor: "
+        "[test_fork/test_ref]"
+        "(https://github.com/test_repo-public/tree/main/results/"
+        "bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
     )
 
-    expected = textwrap.dedent(
-        """
-        🤖 This is the friendly benchmarking bot with some new results!
-
-        @test_actor: [test_fork/test_ref](https://github.com/test_repo-public/tree/main/results/bm-20231001-3.10.4-test_he-JIT,TAILCALL)
-        """
-    ).strip()
+    expected = (
+        "@test_actor: "
+        "[test_fork/test_ref]"
+        "(https://github.com/test_repo-public/tree/main/results/"
+        "bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
+    )
 
-    assert message_sent[0].strip() == expected
+    assert expected in message_sent[0].strip()

tests/test_workflow.py

@@ -239,51 +239,6 @@ def test_whole_workflow(tmpdir):
                 "--check",
             ]
         )
-        with open("requirements.txt", "w") as fd:
-            fd.write(f"{str(bench_runner_checkout)}\n")
-        subprocess.check_call(
-            [
-                str(binary),
-                "workflow_bootstrap.py",
-                "python",
-                "main",
-                "linux-x86_64-linux",
-                "deltablue",
-                ",,,",
-                "--_fast",
-            ]
-        )
-
-
-@pytest.mark.long_running
-def test_check_install_fail(tmpdir):
-    repo = tmpdir / "repo"
-    venv_dir = repo / "outer_venv"
-    bench_runner_checkout = DATA_PATH.parents[1]
-    if sys.platform.startswith("win"):
-        binary = venv_dir / "Scripts" / "python.exe"
-    else:
-        binary = venv_dir / "bin" / "python"
-
-    repo.mkdir()
-
-    with contextlib.chdir(repo):
-        subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
-        subprocess.check_call(
-            [
-                str(binary),
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                "pip",
-            ]
-        )
-        subprocess.check_call(
-            [str(binary), "-m", "pip", "install", f"{bench_runner_checkout}[test]"]
-        )
-        subprocess.check_call([str(binary), "-m", "bench_runner", "install"])
-
         # Now edit one of the generated files to make the check fail
         with open("workflow_bootstrap.py", "a") as fd:
             fd.write("# EXTRA CONTENT\n\n")
@@ -299,41 +254,6 @@ def test_check_install_fail(tmpdir):
                 ]
             )
 
-
-@pytest.mark.long_running
-@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only")
-def test_linux_perf(tmpdir):
-    """
-    Tests the whole workflow from a clean benchmarking repo.
-    """
-    repo = tmpdir / "repo"
-    venv_dir = repo / "outer_venv"
-    bench_runner_checkout = DATA_PATH.parents[1]
-    if sys.platform.startswith("win"):
-        binary = venv_dir / "Scripts" / "python.exe"
-    else:
-        binary = venv_dir / "bin" / "python"
-    profiling_dir = Path(repo / "profiling" / "results")
-
-    repo.mkdir()
-    Path(profiling_dir).mkdir(parents=True)
-
-    with contextlib.chdir(repo):
-        subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
-        subprocess.check_call(
-            [
-                str(binary),
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                "pip",
-            ]
-        )
-        subprocess.check_call(
-            [str(binary), "-m", "pip", "install", f"{bench_runner_checkout}[test]"]
-        )
-        subprocess.check_call([str(binary), "-m", "bench_runner", "install"])
         with open("requirements.txt", "w") as fd:
             fd.write(f"{str(bench_runner_checkout)}\n")
         subprocess.check_call(
@@ -346,17 +266,5 @@ def test_linux_perf(tmpdir):
                 "deltablue",
                 ",,,",
                 "--_fast",
-                "--perf",
             ]
         )
-
-        csv_file = profiling_dir / "deltablue.perf.csv"
-        assert csv_file.is_file()
-
-        with open(csv_file, "r") as fd:
-            lines = iter(fd.readlines())
-            first_line = next(lines)
-            assert first_line.strip() == "self,pid,command,shared_obj,symbol"
-            for line in fd.readlines():
-                assert line.strip().endswith("_PyEval_EvalFrameDefault")
-                break