From 8d52177626f168318f7a2d0030e7b03b65e03be5 Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Thu, 16 Jul 2026 18:41:05 +0900 Subject: [PATCH 1/2] fix reusable-container-path-quoting --- src/ale_bench/tool_wrappers/case_runner.py | 27 ++-- tests/judge/test_reuse_containers.py | 24 +++ tests/tool_wrappers/test_case_runner.py | 167 +++++++++++++++++---- 3 files changed, 179 insertions(+), 39 deletions(-) diff --git a/src/ale_bench/tool_wrappers/case_runner.py b/src/ale_bench/tool_wrappers/case_runner.py index cef49adb..b8babf05 100644 --- a/src/ale_bench/tool_wrappers/case_runner.py +++ b/src/ale_bench/tool_wrappers/case_runner.py @@ -220,11 +220,11 @@ def build_batch_run_command( """ run_command = get_run_command(code_language, judge_version) - run_command += f" < {input_file} > {output_file}" + run_command += f" < {shlex.quote(input_file)} > {shlex.quote(output_file)}" run_command = ( "/usr/bin/time " f'-f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' - f"-o {profiles_file} {run_command}" + f"-o {shlex.quote(profiles_file)} {run_command}" ) # NOTE: We use the GNU Time to measure the resource usage # NOTE: the profiles by GNU Time update every 1 sec (from observations while debugging) time_limit_ceil = math.ceil(time_limit + 0.1) @@ -294,7 +294,7 @@ def build_batch_judge_command( str: The judging command. """ - return f"{ale_bench.constants.TESTER_BIN} {input_file} {output_file}" + return f"{ale_bench.constants.TESTER_BIN} {shlex.quote(input_file)} {shlex.quote(output_file)}" class HostPathsReactiveJudge(BaseModel): @@ -402,11 +402,11 @@ def build_reactive_judge_command( """ run_command = get_run_command(code_language, judge_version) - run_command += f" < {input_file} > {output_file}" + run_command += f" < {shlex.quote(input_file)} > {shlex.quote(output_file)}" run_command = ( f"{ale_bench.constants.TESTER_BIN} /usr/bin/time " f'-f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' - f"-o {profiles_file} {run_command}" + f"-o {shlex.quote(profiles_file)} {run_command}" ) # NOTE: We use the GNU Time to measure the resource usage # NOTE: the profiles by GNU Time update every 1 sec (from observations while debugging) time_limit_ceil = math.ceil(time_limit + 0.1) @@ -500,7 +500,7 @@ def build_vis_command( str: The visualization command. """ - return f"{ale_bench.constants.VIS_BIN} {input_file} {output_file}" + return f"{ale_bench.constants.VIS_BIN} {shlex.quote(input_file)} {shlex.quote(output_file)}" def run_compile_container( @@ -1171,6 +1171,12 @@ def parse_profiles( return execution_time, memory_usage # Return the execution time and memory usage if all checks pass +def build_case_file_prefix(problem_id: str, case_idx: int) -> str: + """Build a filesystem-safe prefix for per-case scratch files.""" + safe_problem_id = re.sub(r"[^A-Za-z0-9._-]+", "_", problem_id) + return f"{safe_problem_id}_{case_idx:06d}_" + + def case_iter_func( problem_id: str, time_limit: float, @@ -1196,15 +1202,14 @@ def case_iter_func( result_input_str = input_str if return_details else None host_paths_judge: HostPathsBatchJudge | HostPathsReactiveJudge execution_time_host = -1.0 + case_file_prefix = build_case_file_prefix(problem_id, case_idx) case_temp_dir = ( reusable_submission_container_pool.scratch_dir if reusable_submission_container_pool is not None else temp_dir ) if problem_type == ProblemType.BATCH: # Run the submission code and generate the output file - host_paths_run = setup_paths_batch_run( - host_paths_compile, case_temp_dir, input_str, f"{problem_id}_{case_idx:06d}_" - ) + host_paths_run = setup_paths_batch_run(host_paths_compile, case_temp_dir, input_str, case_file_prefix) if reusable_submission_container_pool is None: run_volumes = get_batch_run_volumes(host_paths_run, temp_dir) run_result = run_batch_run_container( @@ -1286,7 +1291,7 @@ def case_iter_func( host_paths_compile, case_temp_dir, input_str, - f"{problem_id}_{case_idx:06d}_", + case_file_prefix, ) if reusable_submission_container_pool is None: judge_volumes = get_reactive_judge_volumes(host_paths_judge, temp_dir, tool_dir) @@ -1365,7 +1370,7 @@ def case_iter_func( if not skip_local_visualization and problem_id not in ale_bench.constants.NO_LOCAL_VIS: # Run the local visualization command in the Docker container vis_temp_dir = case_temp_dir if reusable_tool_container_pool is not None else temp_dir - host_paths_vis = setup_paths_vis(host_paths_judge, vis_temp_dir, problem_id, f"{problem_id}_{case_idx:06d}_") + host_paths_vis = setup_paths_vis(host_paths_judge, vis_temp_dir, problem_id, case_file_prefix) if reusable_tool_container_pool is None: vis_volumes = get_vis_volumes(host_paths_vis, tool_dir) run_vis_container(vis_command, vis_volumes) diff --git a/tests/judge/test_reuse_containers.py b/tests/judge/test_reuse_containers.py index acd583be..5fa44ec2 100644 --- a/tests/judge/test_reuse_containers.py +++ b/tests/judge/test_reuse_containers.py @@ -90,6 +90,30 @@ def test_reuse_containers_matches_default_batch( assert all(result.local_visualization is not None for result in default_results) assert all(result.local_visualization is not None for result in reused_results) + def test_reuse_containers_handles_unsafe_problem_id_characters( + self, + inputs: dict[str, str], + ac_codes: dict[str, str], + ahc001_session: Session, + ) -> None: + results = run_cases( + inputs=[inputs["ahc001"]], + code=ac_codes["ahc001"], + code_language=self.CODE_LANGUAGE, + judge_version=self.JUDGE_VERSION, + time_limit=5.0, + memory_limit=256 * 1024 * 1024, + problem_id=r"problem/name (variant); $(command)&", + problem_type=ProblemType.BATCH, + tool_dir=ahc001_session.tool_dir, + return_details=True, + skip_local_visualization=True, + num_workers=1, + reuse_containers=True, + ) + + assert [result.judge_result for result in results] == [JudgeResult.ACCEPTED] + def test_reuse_containers_matches_default_reactive( self, inputs: dict[str, str], diff --git a/tests/tool_wrappers/test_case_runner.py b/tests/tool_wrappers/test_case_runner.py index 76603505..e8b1b18b 100644 --- a/tests/tool_wrappers/test_case_runner.py +++ b/tests/tool_wrappers/test_case_runner.py @@ -16,6 +16,7 @@ HostPathsVis, build_batch_judge_command, build_batch_run_command, + build_case_file_prefix, build_compile_command, build_reactive_judge_command, build_vis_command, @@ -454,21 +455,50 @@ def test_build_batch_run_command( assert run_command == expected -def test_build_batch_run_command_custom_paths() -> None: +@pytest.mark.parametrize( + ("input_file", "output_file", "profiles_file", "expected"), + [ + pytest.param( + "/reuse/input.txt", + "/reuse/output.txt", + "/reuse/profiles.json", + ( + "timeout 2.2 prlimit --cpu=2.1 " + f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' + "-o /reuse/profiles.json " + "./a.out < /reuse/input.txt > /reuse/output.txt; sync" + ), + id="standard", + ), + pytest.param( + "/reuse/problem files (input).txt", + "/reuse/problem files (output).txt", + "/reuse/problem files (profiles).json", + ( + "timeout 2.2 prlimit --cpu=2.1 " + f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' + "-o '/reuse/problem files (profiles).json' " + "./a.out < '/reuse/problem files (input).txt' > '/reuse/problem files (output).txt'; sync" + ), + id="unsafe", + ), + ], +) +def test_build_batch_run_command_custom_paths( + input_file: str, + output_file: str, + profiles_file: str, + expected: str, +) -> None: run_command = build_batch_run_command( CodeLanguage.CPP20, JudgeVersion.V202301, 1.0, - input_file="/reuse/input.txt", - output_file="/reuse/output.txt", - profiles_file="/reuse/profiles.json", - ) - assert run_command == ( - "timeout 2.2 prlimit --cpu=2.1 " - f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' - "-o /reuse/profiles.json " - "./a.out < /reuse/input.txt > /reuse/output.txt; sync" + input_file=input_file, + output_file=output_file, + profiles_file=profiles_file, ) + assert run_command == expected def test_setup_paths_batch_judge() -> None: @@ -512,12 +542,29 @@ def test_build_batch_judge_command() -> None: ) -def test_build_batch_judge_command_custom_paths() -> None: +@pytest.mark.parametrize( + ("input_file", "output_file", "expected"), + [ + pytest.param( + "/reuse/input.txt", + "/reuse/output.txt", + f"{ale_bench.constants.TESTER_BIN} /reuse/input.txt /reuse/output.txt", + id="standard", + ), + pytest.param( + "/reuse/problem files (input).txt", + "/reuse/problem files (output).txt", + f"{ale_bench.constants.TESTER_BIN} '/reuse/problem files (input).txt' '/reuse/problem files (output).txt'", + id="unsafe", + ), + ], +) +def test_build_batch_judge_command_custom_paths(input_file: str, output_file: str, expected: str) -> None: judge_command = build_batch_judge_command( - input_file="/reuse/input.txt", - output_file="/reuse/output.txt", + input_file=input_file, + output_file=output_file, ) - assert judge_command == f"{ale_bench.constants.TESTER_BIN} /reuse/input.txt /reuse/output.txt" + assert judge_command == expected @pytest.mark.parametrize( @@ -837,22 +884,52 @@ def test_build_reactive_judge_command( assert run_command == expected -def test_build_reactive_judge_command_custom_paths() -> None: +@pytest.mark.parametrize( + ("input_file", "output_file", "profiles_file", "expected"), + [ + pytest.param( + "/reuse/input.txt", + "/reuse/output.txt", + "/reuse/profiles.json", + ( + "timeout 2.2 prlimit --cpu=2.1 " + f"{ale_bench.constants.TESTER_BIN} " + f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' + "-o /reuse/profiles.json " + "./a.out < /reuse/input.txt > /reuse/output.txt; sync" + ), + id="standard", + ), + pytest.param( + "/reuse/problem files (input).txt", + "/reuse/problem files (output).txt", + "/reuse/problem files (profiles).json", + ( + "timeout 2.2 prlimit --cpu=2.1 " + f"{ale_bench.constants.TESTER_BIN} " + f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' + "-o '/reuse/problem files (profiles).json' " + "./a.out < '/reuse/problem files (input).txt' > '/reuse/problem files (output).txt'; sync" + ), + id="unsafe", + ), + ], +) +def test_build_reactive_judge_command_custom_paths( + input_file: str, + output_file: str, + profiles_file: str, + expected: str, +) -> None: run_command = build_reactive_judge_command( CodeLanguage.CPP20, JudgeVersion.V202301, 1.0, - input_file="/reuse/input.txt", - output_file="/reuse/output.txt", - profiles_file="/reuse/profiles.json", - ) - assert run_command == ( - "timeout 2.2 prlimit --cpu=2.1 " - f"{ale_bench.constants.TESTER_BIN} " - f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" ' - "-o /reuse/profiles.json " - "./a.out < /reuse/input.txt > /reuse/output.txt; sync" + input_file=input_file, + output_file=output_file, + profiles_file=profiles_file, ) + assert run_command == expected @pytest.mark.parametrize( @@ -1020,9 +1097,29 @@ def test_build_vis_command() -> None: ) -def test_build_vis_command_custom_paths() -> None: - vis_command = build_vis_command(input_file="/reuse/input.txt", output_file="/reuse/output.txt") - assert vis_command == f"{ale_bench.constants.VIS_BIN} /reuse/input.txt /reuse/output.txt" +@pytest.mark.parametrize( + ("input_file", "output_file", "expected"), + [ + pytest.param( + "/reuse/input.txt", + "/reuse/output.txt", + f"{ale_bench.constants.VIS_BIN} /reuse/input.txt /reuse/output.txt", + id="standard", + ), + pytest.param( + "/reuse/problem files (input).txt", + "/reuse/problem files (output).txt", + f"{ale_bench.constants.VIS_BIN} '/reuse/problem files (input).txt' '/reuse/problem files (output).txt'", + id="unsafe", + ), + ], +) +def test_build_vis_command_custom_paths(input_file: str, output_file: str, expected: str) -> None: + vis_command = build_vis_command( + input_file=input_file, + output_file=output_file, + ) + assert vis_command == expected def test_run_batch_judge_reusable_container() -> None: @@ -1531,3 +1628,17 @@ def test_parse_profiles( expected: CaseResult | tuple[float, int], ) -> None: assert parse_profiles(time_limit, memory_limit, profiles_content, execution_time_host, None, None, None) == expected + + +@pytest.mark.parametrize( + ("problem_id", "expected_safe_problem_id"), + [ + pytest.param("problem.alpha-1_beta", "problem.alpha-1_beta", id="safe-ascii"), + pytest.param("problem name (variant)", "problem_name_variant_", id="whitespace-and-parentheses"), + pytest.param(r"problem/path\segment", "problem_path_segment", id="path-separators"), + pytest.param("problem;$()|&<>`'\"*?name", "problem_name", id="shell-metacharacters"), + pytest.param("problem-問題", "problem-_", id="non-ascii"), + ], +) +def test_build_case_file_prefix(problem_id: str, expected_safe_problem_id: str) -> None: + assert build_case_file_prefix(problem_id, 12) == f"{expected_safe_problem_id}_000012_" From 407141de407949bf550b96aa70f5cd1505e04dbe Mon Sep 17 00:00:00 2001 From: Yuki Imajuku Date: Thu, 16 Jul 2026 19:15:22 +0900 Subject: [PATCH 2/2] bugfix: move vis workdir in reuse container mode to /tmp --- src/ale_bench/tool_wrappers/case_runner.py | 9 ++++++--- tests/tool_wrappers/test_case_runner.py | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ale_bench/tool_wrappers/case_runner.py b/src/ale_bench/tool_wrappers/case_runner.py index b8babf05..972114ce 100644 --- a/src/ale_bench/tool_wrappers/case_runner.py +++ b/src/ale_bench/tool_wrappers/case_runner.py @@ -1025,7 +1025,9 @@ def run_vis_reusable_container( f"cp {shlex.quote(generated_file_path)} {shlex.quote(local_visualization_file)}" ) timed_command = f"timeout {ale_bench.constants.VISUALIZE_TIMEOUT} bash -c {shlex.quote(inner_command)}" - _execution_time_host, exit_code, _stderr = reusable_tool_container_pool.run(timed_command) + _execution_time_host, exit_code, _stderr = reusable_tool_container_pool.run( + timed_command, workdir=ale_bench.constants.TMP_DIR + ) if exit_code == TIMEOUT_EXIT_CODE: msg = "Timeout while running the visualization command. Something went wrong." raise RuntimeError(msg) @@ -1375,11 +1377,12 @@ def case_iter_func( vis_volumes = get_vis_volumes(host_paths_vis, tool_dir) run_vis_container(vis_command, vis_volumes) else: - generated_file_path = ( + generated_file_name = Path( ale_bench.constants.LOCAL_VIS_SVG if host_paths_vis.local_visualization_file.suffix == ".svg" else ale_bench.constants.LOCAL_VIS_HTML - ) + ).name + generated_file_path = f"{ale_bench.constants.TMP_DIR}/{generated_file_name}" reusable_vis_command = build_vis_command( input_file=reusable_tool_container_pool.container_path(host_paths_vis.input_file), output_file=reusable_tool_container_pool.container_path(host_paths_vis.output_file), diff --git a/tests/tool_wrappers/test_case_runner.py b/tests/tool_wrappers/test_case_runner.py index e8b1b18b..bbf3de45 100644 --- a/tests/tool_wrappers/test_case_runner.py +++ b/tests/tool_wrappers/test_case_runner.py @@ -1163,15 +1163,15 @@ def run(self, command: str, *, workdir: str = ale_bench.constants.WORK_DIR) -> t pool, f"vis {reusable_input_file} {reusable_output_file}", reusable_local_visualization_file, - ale_bench.constants.LOCAL_VIS_HTML, + f"{ale_bench.constants.TMP_DIR}/vis.html", ) assert pool.calls == [ ( - "timeout 10 bash -c 'rm -f /workdir/vis.html; " + "timeout 10 bash -c 'rm -f /tmp/vis.html; " f"vis {reusable_input_file} {reusable_output_file}; " - f"cp /workdir/vis.html {reusable_local_visualization_file}'", - ale_bench.constants.WORK_DIR, + f"cp /tmp/vis.html {reusable_local_visualization_file}'", + ale_bench.constants.TMP_DIR, ) ]