From 8a9394c8726458b4abf02cef616a8ca1136490c1 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:11:07 +0530 Subject: [PATCH] fix: diagnose TruthBench generated-code sandbox crashes Scheduled CI intermittently reports a generated_code_sandbox regression whose only evidence is "generated code exited -11: " - a segfault in the sandbox subprocess with empty stderr and no indication of which script. - Run the sandbox with -X faulthandler so native crashes dump a traceback to stderr (already canary-scanned and redacted before reporting). - Pin OMP/OpenBLAS/MKL/Polars/Rayon thread pools to 1 and LANG=C.UTF-8 in the otherwise empty sandbox environment. - Prefix sandbox gate failures with domain/surface. --- benchmarks/truthbench/generated_code.py | 13 ++++++- benchmarks/truthbench/runner.py | 9 +++-- tests/truthbench/test_generated_code.py | 50 +++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py index 51ba48a2..4e2ac742 100644 --- a/benchmarks/truthbench/generated_code.py +++ b/benchmarks/truthbench/generated_code.py @@ -193,11 +193,22 @@ def leaks(label: str, payload: Any) -> None: "FRESHDATA_NO_NETWORK": "1", "HOME": str(workdir), "TMPDIR": str(workdir), + # The environment is otherwise empty, so native thread pools would + # size themselves from the host core count; pin them (and the + # locale) so runs are deterministic across machines. + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "POLARS_MAX_THREADS": "1", + "RAYON_NUM_THREADS": "1", + "LANG": "C.UTF-8", } interpreter = python or sys.executable try: proc = subprocess.run( # noqa: PLW1510 - exit code inspected below - [interpreter, "-I", str(harness_path)], + # -X faulthandler: a native crash (negative exit code) dumps the + # Python traceback to stderr, which is redacted and reported. + [interpreter, "-I", "-X", "faulthandler", str(harness_path)], cwd=workdir, env=env, capture_output=True, diff --git a/benchmarks/truthbench/runner.py b/benchmarks/truthbench/runner.py index 5356e857..2f6dba81 100644 --- a/benchmarks/truthbench/runner.py +++ b/benchmarks/truthbench/runner.py @@ -677,6 +677,7 @@ def run_release( parity_failures: list[str] = [] generated: list[str] = [] generated_results: list[GeneratedCodeResult] = [] + sandbox_failures: list[str] = [] audit_ids: set[str] = set() case_observed: dict[str, bool] = {} @@ -729,6 +730,10 @@ def run_release( generated.append(secondary.generated_code) outcome = verify_generated_code(secondary.generated_code, fixture) generated_results.append(outcome) + sandbox_failures.extend( + f"{domain}/{surface_name}: {failure}" + for failure in outcome.failures + ) observations, failures, parity_ledger = _parity_observations(parity, backends) parity_failures.extend( @@ -761,10 +766,6 @@ def run_release( cleaning_annotated = tuple(r for r in all_records if r.surface == "cleaning") parity_annotated = tuple(r for r in all_records if r.surface == "backend_parity") - sandbox_failures = [ - failure for outcome in generated_results for failure in outcome.failures - ] - def _sub_run(records: tuple[Any, ...]) -> RunResult: return RunResult( run_id=run_id, diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py index 5de6561d..29e90b9d 100644 --- a/tests/truthbench/test_generated_code.py +++ b/tests/truthbench/test_generated_code.py @@ -3,6 +3,10 @@ from __future__ import annotations +import json +import signal +import sys + import pandas as pd import pytest from benchmarks.truthbench import generated_code as gc @@ -125,3 +129,49 @@ def test_input_file_overwrite_is_reported(): result = verify_generated_code(code, _fixture()) assert not result.passed assert any("modified its input file" in f for f in result.failures) + + +def _fake_interpreter(tmp_path, body): + """An executable standing in for ``python`` that runs *body* instead.""" + script = tmp_path / "fake_python" + script.write_text(f"#!{sys.executable}\n{body}", encoding="utf-8") + script.chmod(0o755) + return str(script) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX shebang interpreter") +def test_sandbox_pins_native_threads_and_enables_faulthandler(tmp_path): + python = _fake_interpreter( + tmp_path, + "import json, os, sys\n" + "print(json.dumps({'argv': sys.argv[1:], 'env': dict(os.environ)}))\n", + ) + result = verify_generated_code(GOOD, _fixture(), python=python) + assert result.passed, result.failures + seen = json.loads(result.stdout) + assert seen["argv"][:3] == ["-I", "-X", "faulthandler"] + for var in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "POLARS_MAX_THREADS", + "RAYON_NUM_THREADS", + ): + assert seen["env"][var] == "1", var + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") +def test_native_crash_reports_signal_exit_and_faulthandler_dump(tmp_path): + # The flake seen in CI was a bare "generated code exited -11: " with no + # evidence; a crash must now carry the exit signal and the native dump. + python = _fake_interpreter( + tmp_path, + "import faulthandler, os, signal\n" + "faulthandler.enable()\n" + "os.kill(os.getpid(), signal.SIGSEGV)\n", + ) + result = verify_generated_code(GOOD, _fixture(), python=python) + assert not result.passed + [failure] = [f for f in result.failures if "exited" in f] + assert f"exited {-signal.SIGSEGV}" in failure + assert "Segmentation fault" in failure