Merge pull request #419 from mdboom/more-test-coverage

More test coverage

Author: mdboom

CHANGELOG.md

@@ -83,6 +83,14 @@ flags = ["TAILCALL"]
 runners = ["linux_clang", "darwin", "windows_clang"]
 ```
 
+### Long-running tests
+
+`bench-runner` has some new long-running end-to-end integration tests. To avoid running them, use:
+
+```
+python -m pytest -m "not long_running"
+```
+
 ## v1.8.0
 
 ### bench_runner.toml change

pyproject.toml

@@ -52,6 +52,6 @@ markers = [
 [tool.coverage.paths]
 source = [
     "bench_runner",
-    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/outer_venv/lib/python*/site-packages/bench_runner",
-    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/venv/lib/python*/site-packages/bench_runner",
+    "/tmp/pytest-of-*/pytest-*/*/repo/outer_venv/lib/python*/site-packages/bench_runner",
+    "/tmp/pytest-of-*/pytest-*/*/repo/venv/lib/python*/site-packages/bench_runner",
 ]

tests/data/loops.json

@@ -0,0 +1,47 @@
+import contextlib
+from pathlib import Path
+
+
+from bench_runner import gh
+from bench_runner.scripts import notify
+
+
+DATA_PATH = Path(__file__).parent / "data"
+
+
+def test_notify(monkeypatch, capsys):
+    message_sent = [None]
+
+    def send_notification(message):
+        message_sent[0] = message
+
+    monkeypatch.setattr(gh, "send_notification", send_notification)
+    monkeypatch.setenv("GITHUB_ACTOR", "test_actor")
+    monkeypatch.setenv("GITHUB_REPOSITORY", "test_repo")
+
+    with contextlib.chdir(DATA_PATH):
+        notify._main(
+            fork="test_fork",
+            ref="test_ref",
+            head="test_head",
+            date="2023-10-01",
+            version="3.10.4",
+            flags=["JIT", "TAILCALL"],
+        )
+
+    captured = capsys.readouterr()
+    assert captured.out.strip() == (
+        "::notice ::@test_actor: "
+        "[test_fork/test_ref]"
+        "(https://github.com/test_repo-public/tree/main/results/"
+        "bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
+    )
+
+    expected = (
+        "@test_actor: "
+        "[test_fork/test_ref]"
+        "(https://github.com/test_repo-public/tree/main/results/"
+        "bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
+    )
+
+    assert expected in message_sent[0].strip()

tests/test_notify.py

@@ -239,6 +239,21 @@ def test_whole_workflow(tmpdir):
                 "--check",
             ]
         )
+        # Now edit one of the generated files to make the check fail
+        with open("workflow_bootstrap.py", "a") as fd:
+            fd.write("# EXTRA CONTENT\n\n")
+
+        with pytest.raises(subprocess.CalledProcessError):
+            subprocess.check_call(
+                [
+                    str(binary),
+                    "-m",
+                    "bench_runner",
+                    "install",
+                    "--check",
+                ]
+            )
+
         with open("requirements.txt", "w") as fd:
             fd.write(f"{str(bench_runner_checkout)}\n")
         subprocess.check_call(
@@ -253,3 +268,62 @@ def test_whole_workflow(tmpdir):
                 "--_fast",
             ]
         )
+
+
+@pytest.mark.long_running
+@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only")
+def test_pystats(tmpdir):
+    """
+    Tests the whole workflow from a clean benchmarking repo.
+    """
+    tmpdir = Path(tmpdir)
+    repo = tmpdir / "repo"
+    venv_dir = repo / "outer_venv"
+    bench_runner_checkout = DATA_PATH.parents[1]
+    if sys.platform.startswith("win"):
+        binary = venv_dir / "Scripts" / "python.exe"
+    else:
+        binary = venv_dir / "bin" / "python"
+    profiling_dir = Path(repo / "profiling" / "results")
+
+    repo.mkdir()
+    profiling_dir.mkdir(parents=True)
+
+    shutil.copyfile(DATA_PATH / "loops.json", repo / "loops.json")
+
+    with contextlib.chdir(repo):
+        subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
+        subprocess.check_call(
+            [
+                str(binary),
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                "pip",
+            ]
+        )
+        subprocess.check_call(
+            [str(binary), "-m", "pip", "install", f"{bench_runner_checkout}[test]"]
+        )
+        subprocess.check_call([str(binary), "-m", "bench_runner", "install"])
+        with open("requirements.txt", "w") as fd:
+            fd.write(f"{str(bench_runner_checkout)}\n")
+        subprocess.check_call(
+            [
+                str(binary),
+                "workflow_bootstrap.py",
+                "python",
+                "main",
+                "linux-x86_64-linux",
+                "deltablue",
+                ",,,",
+                "--_fast",
+                "--pystats",
+            ]
+        )
+
+        deltablue_output = list((repo / "results").glob("**/*-pystats-deltablue.*"))
+        assert len(deltablue_output) == 2
+        all_output = list((repo / "results").glob("**/*-pystats.*"))
+        assert len(all_output) == 2