More test coverage

Author: mdboom

.github/workflows/ci.yml

@@ -42,6 +42,10 @@ jobs:
   long_test_linux:
     runs-on: ubuntu-latest
     steps:
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y linux-tools-common linux-tools-generic
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:

pyproject.toml

@@ -52,6 +52,6 @@ markers = [
 [tool.coverage.paths]
 source = [
     "bench_runner",
-    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/outer_venv/lib/python*/site-packages/bench_runner",
-    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/venv/lib/python*/site-packages/bench_runner",
+    "/tmp/pytest-of-*/pytest-*/*/repo/outer_venv/lib/python*/site-packages/bench_runner",
+    "/tmp/pytest-of-*/pytest-*/*/repo/venv/lib/python*/site-packages/bench_runner",
 ]

tests/test_notify.py

@@ -0,0 +1,47 @@
+import contextlib
+from pathlib import Path
+import textwrap
+
+
+from bench_runner import gh
+from bench_runner.scripts import notify
+
+
+DATA_PATH = Path(__file__).parent / "data"
+
+
+def test_notify(monkeypatch, capsys):
+    message_sent = [None]
+
+    def send_notification(message):
+        message_sent[0] = message
+
+    monkeypatch.setattr(gh, "send_notification", send_notification)
+    monkeypatch.setenv("GITHUB_ACTOR", "test_actor")
+    monkeypatch.setenv("GITHUB_REPOSITORY", "test_repo")
+
+    with contextlib.chdir(DATA_PATH):
+        notify._main(
+            fork="test_fork",
+            ref="test_ref",
+            head="test_head",
+            date="2023-10-01",
+            version="3.10.4",
+            flags=["JIT", "TAILCALL"],
+        )
+
+    captured = capsys.readouterr()
+    assert (
+        captured.out.strip()
+        == "::notice ::@test_actor: [test_fork/test_ref](https://github.com/test_repo-public/tree/main/results/bm-20231001-3.10.4-test_he-JIT,TAILCALL)"
+    )
+
+    expected = textwrap.dedent(
+        """
+        🤖 This is the friendly benchmarking bot with some new results!
+
+        @test_actor: [test_fork/test_ref](https://github.com/test_repo-public/tree/main/results/bm-20231001-3.10.4-test_he-JIT,TAILCALL)
+        """
+    ).strip()
+
+    assert message_sent[0].strip() == expected

tests/test_workflow.py

@@ -298,3 +298,65 @@ def test_check_install_fail(tmpdir):
                     "--check",
                 ]
             )
+
+
+@pytest.mark.long_running
+@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only")
+def test_linux_perf(tmpdir):
+    """
+    Tests the whole workflow from a clean benchmarking repo.
+    """
+    repo = tmpdir / "repo"
+    venv_dir = repo / "outer_venv"
+    bench_runner_checkout = DATA_PATH.parents[1]
+    if sys.platform.startswith("win"):
+        binary = venv_dir / "Scripts" / "python.exe"
+    else:
+        binary = venv_dir / "bin" / "python"
+    profiling_dir = Path(repo / "profiling" / "results")
+
+    repo.mkdir()
+    Path(profiling_dir).mkdir(parents=True)
+
+    with contextlib.chdir(repo):
+        subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
+        subprocess.check_call(
+            [
+                str(binary),
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                "pip",
+            ]
+        )
+        subprocess.check_call(
+            [str(binary), "-m", "pip", "install", f"{bench_runner_checkout}[test]"]
+        )
+        subprocess.check_call([str(binary), "-m", "bench_runner", "install"])
+        with open("requirements.txt", "w") as fd:
+            fd.write(f"{str(bench_runner_checkout)}\n")
+        subprocess.check_call(
+            [
+                str(binary),
+                "workflow_bootstrap.py",
+                "python",
+                "main",
+                "linux-x86_64-linux",
+                "deltablue",
+                ",,,",
+                "--_fast",
+                "--perf",
+            ]
+        )
+
+        csv_file = profiling_dir / "deltablue.perf.csv"
+        assert csv_file.is_file()
+
+        with open(csv_file, "r") as fd:
+            lines = iter(fd.readlines())
+            first_line = next(lines)
+            assert first_line.strip() == "self,pid,command,shared_obj,symbol"
+            for line in fd.readlines():
+                assert line.strip().endswith("_PyEval_EvalFrameDefault")
+                break