Libkernelbot (#316)

* create a kernelbot library

* remove path hacks

* updated imports

* adapted path

* cleanup

* deploy modal in test environment

* fix pythonpath

* Package libkernelbot

* update

* remove requirements.txt

* update

* update

* update

* update

---------

Co-authored-by: ngc92 <7938269+ngc92@users.noreply.github.com>

Author: Mark Saroufim

.github/workflows/amd_workflow.yml

@@ -62,14 +62,15 @@ jobs:
         EOL
         pip install -r "requirements.txt"
         fi
+        pip install -e .
 
     - name: Run script
       shell: bash
       run: |
         if [[ "${{ github.event.inputs.runner }}" == "amdgpu-mi250-x86-64" ]]; then
           source ${VENV_DIR}/bin/activate
         fi
-        python3 .github/workflows/runner.py
+        python3 src/runners/github-runner.py
 
     - name: Upload training artifacts
       uses: actions/upload-artifact@v4

.github/workflows/nvidia_workflow.yml

@@ -31,6 +31,11 @@ jobs:
       with:
         python-version: '3.10'
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        version: "latest"
+
     - name: Create input files
       shell: bash
       run: |
@@ -62,11 +67,12 @@ jobs:
         EOL
         uv pip install -r "requirements.txt"
         fi
+        uv pip install -e .
 
     - name: Run script
       shell: bash
       run: |
-        python .github/workflows/runner.py
+        python src/runners/github-runner.py
         cat result.json  # Debug: show output
 
     - name: Upload training artifacts

.github/workflows/runner_ci.yml

@@ -24,9 +24,21 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install pytest
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Setup Python environment
+        run: |
+          uv venv .venv
+          echo "VIRTUAL_ENV=$PWD/.venv" >> $GITHUB_ENV
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+          uv pip install pytest
+
+      - name: Install package
         shell: bash
-        run: pip install pytest
+        run: uv pip install -e .
 
       - name: Run script
         shell: bash
@@ -60,6 +72,10 @@ jobs:
           echo "$PWD/.venv/bin" >> $GITHUB_PATH
           uv pip install numpy torch setuptools ninja pytest
 
+      - name: Install package
+        shell: bash
+        run: uv pip install -e .
+
       - name: Run script
         shell: bash
         run: pytest scripts/ci_test_python.py

examples/identity_py/submission.py

@@ -1,4 +1,4 @@
-#!POPCORN leaderboard identity_py
+#!POPCORN leaderboard identity_py-dev
 
 from task import input_t, output_t
 

pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "discord-cluster-manager"
+version = "0.1.0"
+description = "Discord bot for managing compute clusters and running kernel benchmarks"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "PyGithub",
+    "aiohttp",
+    "discord.py",
+    "audioop-lts; python_version >= '3.13'",
+    "python-dotenv",
+    "requests",
+    "modal",
+    "psycopg2-binary",
+    "yoyo-migrations",
+    "better_profanity",
+    "PyYAML",
+    "fastapi[all]",
+    "uvicorn",
+    "jinja2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff",
+    "pre-commit",
+    "pytest",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["scripts", "tests"]
+python_files = ["test_*.py", "*_test.py", "ci_test_*.py"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"

requirements.txt

@@ -1,17 +1,10 @@
-import os
-import sys
 import tempfile
 from pathlib import Path
 
 import pytest
 
-if Path().resolve().name == "scripts":
-    os.chdir("..")
-
-sys.path.append("src/discord-cluster-manager")
-
-from consts import ExitCode, SubmissionMode
-from run_eval import compile_cuda_script, run_cuda_script
+from libkernelbot.consts import ExitCode, SubmissionMode
+from libkernelbot.run_eval import compile_cuda_script, run_cuda_script
 
 ref = Path("examples/identity_cuda/reference.cuh").read_text()
 task_h = Path("examples/identity_cuda/task.h").read_text()

scripts/ci_test_cuda.py

@@ -1,14 +1,7 @@
-import os
-import sys
 from pathlib import Path
 
-if Path().resolve().name == "scripts":
-    os.chdir("..")
-
-sys.path.append("src/discord-cluster-manager")
-
-from consts import ExitCode, SubmissionMode
-from run_eval import run_pytorch_script
+from libkernelbot.consts import ExitCode, SubmissionMode
+from libkernelbot.run_eval import run_pytorch_script
 
 ref = Path("examples/identity_py/reference.py").read_text()
 task = Path("examples/identity_py/task.py").read_text()