Adding unit tests (#319)

Author: ngc92

.github/workflows/runner_ci.yml

@@ -83,3 +83,11 @@ jobs:
     env:
       CUDA_VISIBLE_DEVICES: 0
 
+  unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --extra dev
+      - run: uv run pytest unit-tests -v

src/libkernelbot/submission.py

@@ -47,7 +47,7 @@ def prepare_submission(
         )
 
     if profanity.contains_profanity(req.file_name):
-        raise KernelBotError("Please provide a non rude filename")
+        raise KernelBotError("Please provide a non-rude filename")
 
     # check file extension
     if not req.file_name.endswith((".py", ".cu", ".cuh", ".cpp")):
@@ -137,7 +137,7 @@ def handle_popcorn_directives(req: SubmissionRequest) -> SubmissionRequest:
     return req
 
 
-def _get_popcorn_directives(submission: str) -> dict:
+def _get_popcorn_directives(submission: str) -> dict:  # noqa: C901
     popcorn_info = {"gpus": None, "leaderboard": None}
     for line in submission.splitlines():
         # only process the first comment block of the file.
@@ -148,10 +148,26 @@ def _get_popcorn_directives(submission: str) -> dict:
         args = line.split()
         if args[0] in ["//!POPCORN", "#!POPCORN"]:
             arg = args[1].strip().lower()
-            if arg in ["gpu", "gpus"]:
+            if len(args) < 3:
+                raise KernelBotError(f"!POPCORN directive missing argument: {line}")
+            #  allow both versions of the argument
+            if arg == "gpu":
+                arg = "gpus"
+
+            if arg not in popcorn_info:
+                raise KernelBotError(f"Invalid !POPCORN directive: {arg}")
+
+            if popcorn_info[arg] is not None:
+                raise KernelBotError(f"Found multiple values for !POPCORN directive {arg}")
+
+            if arg == "gpus":
                 popcorn_info["gpus"] = args[2:]
             elif arg == "leaderboard":
-                popcorn_info["leaderboard"] = args[2]
+                popcorn_info["leaderboard"] = args[2].strip()
+                if len(popcorn_info["leaderboard"]) == 0:
+                    raise KernelBotError(
+                        "No leaderboard specified in !POPCORN Leaderboard directive"
+                    )
     return popcorn_info
 
 

src/libkernelbot/task.py

@@ -62,6 +62,12 @@ class LeaderboardTask:
     ranking_by: RankCriterion = RankCriterion.LAST
     seed: Optional[int] = None
 
+    def __post_init__(self):
+        if self.lang == Language.Python and not isinstance(self.config, PythonTaskData):
+            raise TypeError("Python language requires PythonTaskData config")
+        if self.lang == Language.CUDA and not isinstance(self.config, CudaTaskData):
+            raise TypeError("CUDA language requires CudaTaskData config")
+
     @classmethod
     def from_dict(cls, data: dict):
         data_ = copy.copy(data)
@@ -147,7 +153,7 @@ def make_task_definition(yaml_file: str | Path) -> LeaderboardDefinition:
 
 
 def build_task_config(
-    task: "LeaderboardTask" = None,
+    task: LeaderboardTask = None,
     submission_content: str = None,
     arch: str = None,
     mode: SubmissionMode = None,

src/libkernelbot/utils.py

@@ -60,9 +60,9 @@ def __init__(self, max_size: int):
         self._max_size = max_size
         self._q = []
 
-    def __getitem__(self, key: Any, default: Any = None) -> Any | None:
+    def __getitem__(self, key: Any) -> Any | None:
         if key not in self._cache:
-            return default
+            return None
 
         self._q.remove(key)
         self._q.append(key)
@@ -95,50 +95,51 @@ def invalidate(self):
         self._q.clear()
 
 
-def format_time(value: float | str, err: Optional[float | str] = None, scale=None):  # noqa: C901
-    if value is None:
+def format_time(nanoseconds: float | str, err: Optional[float | str] = None):  # noqa: C901
+    if nanoseconds is None:
         logging.warning("Expected a number, got None", stack_info=True)
         return "–"
 
     # really ugly, but works for now
-    value = float(value)
+    nanoseconds = float(nanoseconds)
 
     scale = 1  # nanoseconds
     unit = "ns"
-    if value > 2_000_000:
+    if nanoseconds > 2_000_000:
         scale = 1000_000
         unit = "ms"
-    elif value > 2000:
+    elif nanoseconds > 2000:
         scale = 1000
         unit = "µs"
 
-    value /= scale
+    time_in_unit = nanoseconds / scale
     if err is not None:
         err = float(err)
         err /= scale
-    if value < 1:
+    if time_in_unit < 1:
         if err:
-            return f"{value} ± {err} {unit}"
+            return f"{time_in_unit} ± {err} {unit}"
         else:
-            return f"{value} {unit}"
-    elif value < 10:
+            return f"{time_in_unit} {unit}"
+    elif time_in_unit < 10:
         if err:
-            return f"{value:.2f} ± {err:.3f} {unit}"
+            return f"{time_in_unit:.2f} ± {err:.3f} {unit}"
         else:
-            return f"{value:.2f} {unit}"
-    elif value < 100:
+            return f"{time_in_unit:.2f} {unit}"
+    elif time_in_unit < 100:
         if err:
-            return f"{value:.1f} ± {err:.2f} {unit}"
+            return f"{time_in_unit:.1f} ± {err:.2f} {unit}"
         else:
-            return f"{value:.1f} {unit}"
+            return f"{time_in_unit:.1f} {unit}"
     else:
         if err:
-            return f"{value:.0f} ± {err:.1f} {unit}"
+            return f"{time_in_unit:.0f} ± {err:.1f} {unit}"
         else:
-            return f"{value:.0f} {unit}"
+            return f"{time_in_unit:.0f} {unit}"
 
 
 def limit_length(text: str, maxlen: int):
+    assert maxlen > 6
     if len(text) > maxlen:
         return text[: maxlen - 6] + " [...]"
     else: