Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 22 additions & 14 deletions src/ale_bench/tool_wrappers/case_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,11 @@ def build_batch_run_command(

"""
run_command = get_run_command(code_language, judge_version)
run_command += f" < {input_file} > {output_file}"
run_command += f" < {shlex.quote(input_file)} > {shlex.quote(output_file)}"
run_command = (
"/usr/bin/time "
f'-f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
f"-o {profiles_file} {run_command}"
f"-o {shlex.quote(profiles_file)} {run_command}"
) # NOTE: We use the GNU Time to measure the resource usage
# NOTE: the profiles by GNU Time update every 1 sec (from observations while debugging)
time_limit_ceil = math.ceil(time_limit + 0.1)
Expand Down Expand Up @@ -294,7 +294,7 @@ def build_batch_judge_command(
str: The judging command.

"""
return f"{ale_bench.constants.TESTER_BIN} {input_file} {output_file}"
return f"{ale_bench.constants.TESTER_BIN} {shlex.quote(input_file)} {shlex.quote(output_file)}"


class HostPathsReactiveJudge(BaseModel):
Expand Down Expand Up @@ -402,11 +402,11 @@ def build_reactive_judge_command(

"""
run_command = get_run_command(code_language, judge_version)
run_command += f" < {input_file} > {output_file}"
run_command += f" < {shlex.quote(input_file)} > {shlex.quote(output_file)}"
run_command = (
f"{ale_bench.constants.TESTER_BIN} /usr/bin/time "
f'-f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
f"-o {profiles_file} {run_command}"
f"-o {shlex.quote(profiles_file)} {run_command}"
) # NOTE: We use the GNU Time to measure the resource usage
# NOTE: the profiles by GNU Time update every 1 sec (from observations while debugging)
time_limit_ceil = math.ceil(time_limit + 0.1)
Expand Down Expand Up @@ -500,7 +500,7 @@ def build_vis_command(
str: The visualization command.

"""
return f"{ale_bench.constants.VIS_BIN} {input_file} {output_file}"
return f"{ale_bench.constants.VIS_BIN} {shlex.quote(input_file)} {shlex.quote(output_file)}"


def run_compile_container(
Expand Down Expand Up @@ -1025,7 +1025,9 @@ def run_vis_reusable_container(
f"cp {shlex.quote(generated_file_path)} {shlex.quote(local_visualization_file)}"
)
timed_command = f"timeout {ale_bench.constants.VISUALIZE_TIMEOUT} bash -c {shlex.quote(inner_command)}"
_execution_time_host, exit_code, _stderr = reusable_tool_container_pool.run(timed_command)
_execution_time_host, exit_code, _stderr = reusable_tool_container_pool.run(
timed_command, workdir=ale_bench.constants.TMP_DIR
)
if exit_code == TIMEOUT_EXIT_CODE:
msg = "Timeout while running the visualization command. Something went wrong."
raise RuntimeError(msg)
Expand Down Expand Up @@ -1171,6 +1173,12 @@ def parse_profiles(
return execution_time, memory_usage # Return the execution time and memory usage if all checks pass


def build_case_file_prefix(problem_id: str, case_idx: int) -> str:
"""Build a filesystem-safe prefix for per-case scratch files."""
safe_problem_id = re.sub(r"[^A-Za-z0-9._-]+", "_", problem_id)
return f"{safe_problem_id}_{case_idx:06d}_"


def case_iter_func(
problem_id: str,
time_limit: float,
Expand All @@ -1196,15 +1204,14 @@ def case_iter_func(
result_input_str = input_str if return_details else None
host_paths_judge: HostPathsBatchJudge | HostPathsReactiveJudge
execution_time_host = -1.0
case_file_prefix = build_case_file_prefix(problem_id, case_idx)
case_temp_dir = (
reusable_submission_container_pool.scratch_dir if reusable_submission_container_pool is not None else temp_dir
)

if problem_type == ProblemType.BATCH:
# Run the submission code and generate the output file
host_paths_run = setup_paths_batch_run(
host_paths_compile, case_temp_dir, input_str, f"{problem_id}_{case_idx:06d}_"
)
host_paths_run = setup_paths_batch_run(host_paths_compile, case_temp_dir, input_str, case_file_prefix)
if reusable_submission_container_pool is None:
run_volumes = get_batch_run_volumes(host_paths_run, temp_dir)
run_result = run_batch_run_container(
Expand Down Expand Up @@ -1286,7 +1293,7 @@ def case_iter_func(
host_paths_compile,
case_temp_dir,
input_str,
f"{problem_id}_{case_idx:06d}_",
case_file_prefix,
)
if reusable_submission_container_pool is None:
judge_volumes = get_reactive_judge_volumes(host_paths_judge, temp_dir, tool_dir)
Expand Down Expand Up @@ -1365,16 +1372,17 @@ def case_iter_func(
if not skip_local_visualization and problem_id not in ale_bench.constants.NO_LOCAL_VIS:
# Run the local visualization command in the Docker container
vis_temp_dir = case_temp_dir if reusable_tool_container_pool is not None else temp_dir
host_paths_vis = setup_paths_vis(host_paths_judge, vis_temp_dir, problem_id, f"{problem_id}_{case_idx:06d}_")
host_paths_vis = setup_paths_vis(host_paths_judge, vis_temp_dir, problem_id, case_file_prefix)
if reusable_tool_container_pool is None:
vis_volumes = get_vis_volumes(host_paths_vis, tool_dir)
run_vis_container(vis_command, vis_volumes)
else:
generated_file_path = (
generated_file_name = Path(
ale_bench.constants.LOCAL_VIS_SVG
if host_paths_vis.local_visualization_file.suffix == ".svg"
else ale_bench.constants.LOCAL_VIS_HTML
)
).name
generated_file_path = f"{ale_bench.constants.TMP_DIR}/{generated_file_name}"
reusable_vis_command = build_vis_command(
input_file=reusable_tool_container_pool.container_path(host_paths_vis.input_file),
output_file=reusable_tool_container_pool.container_path(host_paths_vis.output_file),
Expand Down
24 changes: 24 additions & 0 deletions tests/judge/test_reuse_containers.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,30 @@ def test_reuse_containers_matches_default_batch(
assert all(result.local_visualization is not None for result in default_results)
assert all(result.local_visualization is not None for result in reused_results)

def test_reuse_containers_handles_unsafe_problem_id_characters(
self,
inputs: dict[str, str],
ac_codes: dict[str, str],
ahc001_session: Session,
) -> None:
results = run_cases(
inputs=[inputs["ahc001"]],
code=ac_codes["ahc001"],
code_language=self.CODE_LANGUAGE,
judge_version=self.JUDGE_VERSION,
time_limit=5.0,
memory_limit=256 * 1024 * 1024,
problem_id=r"problem/name (variant); $(command)&",
problem_type=ProblemType.BATCH,
tool_dir=ahc001_session.tool_dir,
return_details=True,
skip_local_visualization=True,
num_workers=1,
reuse_containers=True,
)

assert [result.judge_result for result in results] == [JudgeResult.ACCEPTED]

def test_reuse_containers_matches_default_reactive(
self,
inputs: dict[str, str],
Expand Down
175 changes: 143 additions & 32 deletions tests/tool_wrappers/test_case_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
HostPathsVis,
build_batch_judge_command,
build_batch_run_command,
build_case_file_prefix,
build_compile_command,
build_reactive_judge_command,
build_vis_command,
Expand Down Expand Up @@ -454,21 +455,50 @@ def test_build_batch_run_command(
assert run_command == expected


def test_build_batch_run_command_custom_paths() -> None:
@pytest.mark.parametrize(
("input_file", "output_file", "profiles_file", "expected"),
[
pytest.param(
"/reuse/input.txt",
"/reuse/output.txt",
"/reuse/profiles.json",
(
"timeout 2.2 prlimit --cpu=2.1 "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o /reuse/profiles.json "
"./a.out < /reuse/input.txt > /reuse/output.txt; sync"
),
id="standard",
),
pytest.param(
"/reuse/problem files (input).txt",
"/reuse/problem files (output).txt",
"/reuse/problem files (profiles).json",
(
"timeout 2.2 prlimit --cpu=2.1 "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o '/reuse/problem files (profiles).json' "
"./a.out < '/reuse/problem files (input).txt' > '/reuse/problem files (output).txt'; sync"
),
id="unsafe",
),
],
)
def test_build_batch_run_command_custom_paths(
input_file: str,
output_file: str,
profiles_file: str,
expected: str,
) -> None:
run_command = build_batch_run_command(
CodeLanguage.CPP20,
JudgeVersion.V202301,
1.0,
input_file="/reuse/input.txt",
output_file="/reuse/output.txt",
profiles_file="/reuse/profiles.json",
)
assert run_command == (
"timeout 2.2 prlimit --cpu=2.1 "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o /reuse/profiles.json "
"./a.out < /reuse/input.txt > /reuse/output.txt; sync"
input_file=input_file,
output_file=output_file,
profiles_file=profiles_file,
)
assert run_command == expected


def test_setup_paths_batch_judge() -> None:
Expand Down Expand Up @@ -512,12 +542,29 @@ def test_build_batch_judge_command() -> None:
)


def test_build_batch_judge_command_custom_paths() -> None:
@pytest.mark.parametrize(
("input_file", "output_file", "expected"),
[
pytest.param(
"/reuse/input.txt",
"/reuse/output.txt",
f"{ale_bench.constants.TESTER_BIN} /reuse/input.txt /reuse/output.txt",
id="standard",
),
pytest.param(
"/reuse/problem files (input).txt",
"/reuse/problem files (output).txt",
f"{ale_bench.constants.TESTER_BIN} '/reuse/problem files (input).txt' '/reuse/problem files (output).txt'",
id="unsafe",
),
],
)
def test_build_batch_judge_command_custom_paths(input_file: str, output_file: str, expected: str) -> None:
judge_command = build_batch_judge_command(
input_file="/reuse/input.txt",
output_file="/reuse/output.txt",
input_file=input_file,
output_file=output_file,
)
assert judge_command == f"{ale_bench.constants.TESTER_BIN} /reuse/input.txt /reuse/output.txt"
assert judge_command == expected


@pytest.mark.parametrize(
Expand Down Expand Up @@ -837,22 +884,52 @@ def test_build_reactive_judge_command(
assert run_command == expected


def test_build_reactive_judge_command_custom_paths() -> None:
@pytest.mark.parametrize(
("input_file", "output_file", "profiles_file", "expected"),
[
pytest.param(
"/reuse/input.txt",
"/reuse/output.txt",
"/reuse/profiles.json",
(
"timeout 2.2 prlimit --cpu=2.1 "
f"{ale_bench.constants.TESTER_BIN} "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o /reuse/profiles.json "
"./a.out < /reuse/input.txt > /reuse/output.txt; sync"
),
id="standard",
),
pytest.param(
"/reuse/problem files (input).txt",
"/reuse/problem files (output).txt",
"/reuse/problem files (profiles).json",
(
"timeout 2.2 prlimit --cpu=2.1 "
f"{ale_bench.constants.TESTER_BIN} "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o '/reuse/problem files (profiles).json' "
"./a.out < '/reuse/problem files (input).txt' > '/reuse/problem files (output).txt'; sync"
),
id="unsafe",
),
],
)
def test_build_reactive_judge_command_custom_paths(
input_file: str,
output_file: str,
profiles_file: str,
expected: str,
) -> None:
run_command = build_reactive_judge_command(
CodeLanguage.CPP20,
JudgeVersion.V202301,
1.0,
input_file="/reuse/input.txt",
output_file="/reuse/output.txt",
profiles_file="/reuse/profiles.json",
)
assert run_command == (
"timeout 2.2 prlimit --cpu=2.1 "
f"{ale_bench.constants.TESTER_BIN} "
f'/usr/bin/time -f "{ale_bench.constants.TIME_OUTPUT_FORMAT}" '
"-o /reuse/profiles.json "
"./a.out < /reuse/input.txt > /reuse/output.txt; sync"
input_file=input_file,
output_file=output_file,
profiles_file=profiles_file,
)
assert run_command == expected


@pytest.mark.parametrize(
Expand Down Expand Up @@ -1020,9 +1097,29 @@ def test_build_vis_command() -> None:
)


def test_build_vis_command_custom_paths() -> None:
vis_command = build_vis_command(input_file="/reuse/input.txt", output_file="/reuse/output.txt")
assert vis_command == f"{ale_bench.constants.VIS_BIN} /reuse/input.txt /reuse/output.txt"
@pytest.mark.parametrize(
("input_file", "output_file", "expected"),
[
pytest.param(
"/reuse/input.txt",
"/reuse/output.txt",
f"{ale_bench.constants.VIS_BIN} /reuse/input.txt /reuse/output.txt",
id="standard",
),
pytest.param(
"/reuse/problem files (input).txt",
"/reuse/problem files (output).txt",
f"{ale_bench.constants.VIS_BIN} '/reuse/problem files (input).txt' '/reuse/problem files (output).txt'",
id="unsafe",
),
],
)
def test_build_vis_command_custom_paths(input_file: str, output_file: str, expected: str) -> None:
vis_command = build_vis_command(
input_file=input_file,
output_file=output_file,
)
assert vis_command == expected


def test_run_batch_judge_reusable_container() -> None:
Expand Down Expand Up @@ -1066,15 +1163,15 @@ def run(self, command: str, *, workdir: str = ale_bench.constants.WORK_DIR) -> t
pool,
f"vis {reusable_input_file} {reusable_output_file}",
reusable_local_visualization_file,
ale_bench.constants.LOCAL_VIS_HTML,
f"{ale_bench.constants.TMP_DIR}/vis.html",
)

assert pool.calls == [
(
"timeout 10 bash -c 'rm -f /workdir/vis.html; "
"timeout 10 bash -c 'rm -f /tmp/vis.html; "
f"vis {reusable_input_file} {reusable_output_file}; "
f"cp /workdir/vis.html {reusable_local_visualization_file}'",
ale_bench.constants.WORK_DIR,
f"cp /tmp/vis.html {reusable_local_visualization_file}'",
ale_bench.constants.TMP_DIR,
)
]

Expand Down Expand Up @@ -1531,3 +1628,17 @@ def test_parse_profiles(
expected: CaseResult | tuple[float, int],
) -> None:
assert parse_profiles(time_limit, memory_limit, profiles_content, execution_time_host, None, None, None) == expected


@pytest.mark.parametrize(
("problem_id", "expected_safe_problem_id"),
[
pytest.param("problem.alpha-1_beta", "problem.alpha-1_beta", id="safe-ascii"),
pytest.param("problem name (variant)", "problem_name_variant_", id="whitespace-and-parentheses"),
pytest.param(r"problem/path\segment", "problem_path_segment", id="path-separators"),
pytest.param("problem;$()|&<>`'\"*?name", "problem_name", id="shell-metacharacters"),
pytest.param("problem-問題", "problem-_", id="non-ascii"),
],
)
def test_build_case_file_prefix(problem_id: str, expected_safe_problem_id: str) -> None:
assert build_case_file_prefix(problem_id, 12) == f"{expected_safe_problem_id}_000012_"