remove SCRIPT run mode (#303)

Author: ngc92

examples/eval.py

@@ -366,7 +366,7 @@ def main():
             elif mode == "profile":
                 run_profiling(logger, tests)
             else:
-                # TODO: Implement script mode
+                # invalid mode
                 return 2
 
 

scripts/ci_test_cuda.py

@@ -204,7 +204,8 @@ def test_include_dirs(tmp_path: Path):
         {"eval.cu": eval_cu, "submission.cu": sub},
         header_files,
         flags=["-I.", f"-I{tmp_path}"],
-        mode="script",
+        mode=SubmissionMode.TEST.value,
+        tests="size: 256; seed: 42\n",
     )
 
     assert result.compilation.success is True

src/discord-cluster-manager/api/main.py

@@ -335,7 +335,6 @@ async def run_submission(  # noqa: C901
     allowed_modes = [
         SubmissionMode.TEST,
         SubmissionMode.BENCHMARK,
-        SubmissionMode.SCRIPT,
         SubmissionMode.LEADERBOARD,
     ]
     if submission_mode_enum not in allowed_modes:

src/discord-cluster-manager/bot.py

@@ -11,7 +11,6 @@
 from cogs.admin_cog import AdminCog
 from cogs.leaderboard_cog import LeaderboardCog
 from cogs.misc_cog import BotManagerCog
-from cogs.submit_cog import SubmitCog
 from cogs.verify_run_cog import VerifyRunCog
 from discord import app_commands
 from discord.ext import commands
@@ -36,13 +35,6 @@ def __init__(self, debug_mode=False):
         super().__init__(intents=intents, command_prefix="!")
         self.debug_mode = debug_mode
 
-        # Create the run group for leaderboardless runs. Debugging only.
-        if self.debug_mode:
-            self.run_group = app_commands.Group(
-                name="run", description="Run jobs on different platforms"
-            )
-            self.tree.add_command(self.run_group)
-
         self.leaderboard_group = app_commands.Group(
             name="leaderboard", description="Leaderboard commands"
         )
@@ -65,7 +57,6 @@ async def setup_hook(self):
         logger.info(f"Syncing commands for staging guild {DISCORD_CLUSTER_STAGING_ID}")
         try:
             # Load cogs
-            await self.add_cog(SubmitCog(self))
             await self.add_cog(BotManagerCog(self))
             await self.add_cog(LeaderboardCog(self))
             await self.add_cog(VerifyRunCog(self))

src/discord-cluster-manager/cogs/submit_cog.py

@@ -8,7 +8,6 @@ class Timeout(IntEnum):
     BENCHMARK = 180
     RANKED = 180
     COMPILE = 120
-    SCRIPT = 120
 
 
 class SchedulerType(Enum):
@@ -88,15 +87,13 @@ class SubmissionMode(Enum):
         limited (no stdout/stderr).
     Private: Special run that does test followed by leaderboard (on a secret seed), but gives only
         very limited feedback.
-    Script: Submit an arbitrary script.
     """
 
     TEST = "test"
     BENCHMARK = "benchmark"
     PROFILE = "profile"
     LEADERBOARD = "leaderboard"
     PRIVATE = "private"
-    SCRIPT = "script"
 
 
 class Language(Enum):

src/discord-cluster-manager/consts.py

@@ -342,20 +342,6 @@ def generate_report(result: FullResult) -> RunResultReport:  # noqa: C901
             make_benchmark_log(bench_run),
         )
 
-    if "script" in runs:
-        run = runs["script"]
-        if run.compilation is not None and not run.compilation.success:
-            _generate_compile_report(report, run.compilation)
-            return report
-
-        run = run.run
-        # OK, we were successful
-        message = "# Success!\n"
-        message += "Command "
-        message += f"```bash\n{limit_length(run.command, 1000)}```\n"
-        message += f"ran successfully in {run.duration:.2f} seconds.\n"
-        report.add_text(message)
-
     if len(runs) == 1:
         run = next(iter(runs.values()))
         if len(run.run.stderr.strip()) > 0:

src/discord-cluster-manager/report.py

@@ -306,8 +306,7 @@ def run_single_evaluation(
             bench_file.flush()
             return run_program(call + [mode, bench_file.name], seed=seed, timeout=timeout)
     else:
-        assert mode == "script"
-        return run_program(call, seed=seed, timeout=Timeout.SCRIPT)
+        raise ValueError(f"Invalid mode {mode}")
 
 
 def make_system_info() -> SystemInfo:
@@ -506,12 +505,12 @@ def run_evaluation(
     """
     Given a "runner" function `call`, interprets the mode
     and calls the runner with the right arguments.
-    Simple modes (test, benchmark, profile, script) just
+    Simple modes (test, benchmark, profile) just
     invoke the runner once, but private/leaderboard
     require multiple runner calls.
     """
     results: dict[str, EvalResult] = {}
-    if mode in ["test", "benchmark", "profile", "script"]:
+    if mode in ["test", "benchmark", "profile"]:
         results[mode] = call(mode=mode)
     elif mode in ["private", "leaderboard"]:
         # first, run the tests

src/discord-cluster-manager/run_eval.py

@@ -142,65 +142,43 @@ def build_task_config(
     arch: str = None,
     mode: SubmissionMode = None,
 ) -> dict:
-    if task is None:
-        assert mode == SubmissionMode.SCRIPT
-        # TODO detect language
-        lang = "py"
-
-        config = {
-            "lang": lang,
-            "arch": arch,
-        }
-
-        eval_name = {"py": "eval.py", "cu": "eval.cu"}[lang]
-
-        if lang == "py":
-            config["main"] = "eval.py"
-
+    all_files = {}
+    for n, c in task.files.items():
+        if c == "@SUBMISSION@":
+            all_files[n] = submission_content
+        else:
+            all_files[n] = c
+
+    common = {
+        "lang": task.lang.value,
+        "arch": arch,
+        "benchmarks": task.benchmarks,
+        "tests": task.tests,
+        "mode": mode.value,
+        "test_timeout": task.test_timeout,
+        "benchmark_timeout": task.benchmark_timeout,
+        "ranked_timeout": task.ranked_timeout,
+        "ranking_by": task.ranking_by.value,
+        "seed": task.seed,
+    }
+
+    if task.lang == Language.Python:
         return {
-            **config,
-            "sources": {
-                eval_name: submission_content,
-            },
+            "main": task.config.main,
+            "sources": all_files,
+            **common,
         }
     else:
-        all_files = {}
-        for n, c in task.files.items():
-            if c == "@SUBMISSION@":
-                all_files[n] = submission_content
+        sources = {}
+        headers = {}
+        for f in all_files:
+            if f in task.config.sources:
+                sources[f] = all_files[f]
             else:
-                all_files[n] = c
-
-        common = {
-            "lang": task.lang.value,
-            "arch": arch,
-            "benchmarks": task.benchmarks,
-            "tests": task.tests,
-            "mode": mode.value,
-            "test_timeout": task.test_timeout,
-            "benchmark_timeout": task.benchmark_timeout,
-            "ranked_timeout": task.ranked_timeout,
-            "ranking_by": task.ranking_by.value,
-            "seed": task.seed,
-        }
+                headers[f] = all_files[f]
 
-        if task.lang == Language.Python:
-            return {
-                "main": task.config.main,
-                "sources": all_files,
-                **common,
-            }
-        else:
-            sources = {}
-            headers = {}
-            for f in all_files:
-                if f in task.config.sources:
-                    sources[f] = all_files[f]
-                else:
-                    headers[f] = all_files[f]
-
-            return {
-                "sources": sources,
-                "headers": headers,
-                "include_dirs": task.config.include_dirs,
-            }
+        return {
+            "sources": sources,
+            "headers": headers,
+            "include_dirs": task.config.include_dirs,
+        }