fix: diagnose TruthBench generated-code sandbox crashes

benchmarks/truthbench/generated_code.py

@@ -193,11 +193,22 @@ def leaks(label: str, payload: Any) -> None:
             "FRESHDATA_NO_NETWORK": "1",
             "HOME": str(workdir),
             "TMPDIR": str(workdir),
+            # The environment is otherwise empty, so native thread pools would
+            # size themselves from the host core count; pin them (and the
+            # locale) so runs are deterministic across machines.
+            "OMP_NUM_THREADS": "1",
+            "OPENBLAS_NUM_THREADS": "1",
+            "MKL_NUM_THREADS": "1",
+            "POLARS_MAX_THREADS": "1",
+            "RAYON_NUM_THREADS": "1",
+            "LANG": "C.UTF-8",
         }
         interpreter = python or sys.executable
         try:
             proc = subprocess.run(  # noqa: PLW1510 - exit code inspected below
-                [interpreter, "-I", str(harness_path)],
+                # -X faulthandler: a native crash (negative exit code) dumps the
+                # Python traceback to stderr, which is redacted and reported.
+                [interpreter, "-I", "-X", "faulthandler", str(harness_path)],
                 cwd=workdir,
                 env=env,
                 capture_output=True,

benchmarks/truthbench/runner.py

@@ -677,6 +677,7 @@ def run_release(
     parity_failures: list[str] = []
     generated: list[str] = []
     generated_results: list[GeneratedCodeResult] = []
+    sandbox_failures: list[str] = []
     audit_ids: set[str] = set()
 
     case_observed: dict[str, bool] = {}
@@ -729,6 +730,10 @@ def run_release(
                     generated.append(secondary.generated_code)
                     outcome = verify_generated_code(secondary.generated_code, fixture)
                     generated_results.append(outcome)
+                    sandbox_failures.extend(
+                        f"{domain}/{surface_name}: {failure}"
+                        for failure in outcome.failures
+                    )
 
         observations, failures, parity_ledger = _parity_observations(parity, backends)
         parity_failures.extend(
@@ -761,10 +766,6 @@ def run_release(
     cleaning_annotated = tuple(r for r in all_records if r.surface == "cleaning")
     parity_annotated = tuple(r for r in all_records if r.surface == "backend_parity")
 
-    sandbox_failures = [
-        failure for outcome in generated_results for failure in outcome.failures
-    ]
-
     def _sub_run(records: tuple[Any, ...]) -> RunResult:
         return RunResult(
             run_id=run_id,

tests/truthbench/test_generated_code.py

@@ -3,6 +3,10 @@
 
 from __future__ import annotations
 
+import json
+import signal
+import sys
+
 import pandas as pd
 import pytest
 from benchmarks.truthbench import generated_code as gc
@@ -125,3 +129,49 @@ def test_input_file_overwrite_is_reported():
     result = verify_generated_code(code, _fixture())
     assert not result.passed
     assert any("modified its input file" in f for f in result.failures)
+
+
+def _fake_interpreter(tmp_path, body):
+    """An executable standing in for ``python`` that runs *body* instead."""
+    script = tmp_path / "fake_python"
+    script.write_text(f"#!{sys.executable}\n{body}", encoding="utf-8")
+    script.chmod(0o755)
+    return str(script)
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="POSIX shebang interpreter")
+def test_sandbox_pins_native_threads_and_enables_faulthandler(tmp_path):
+    python = _fake_interpreter(
+        tmp_path,
+        "import json, os, sys\n"
+        "print(json.dumps({'argv': sys.argv[1:], 'env': dict(os.environ)}))\n",
+    )
+    result = verify_generated_code(GOOD, _fixture(), python=python)
+    assert result.passed, result.failures
+    seen = json.loads(result.stdout)
+    assert seen["argv"][:3] == ["-I", "-X", "faulthandler"]
+    for var in (
+        "OMP_NUM_THREADS",
+        "OPENBLAS_NUM_THREADS",
+        "MKL_NUM_THREADS",
+        "POLARS_MAX_THREADS",
+        "RAYON_NUM_THREADS",
+    ):
+        assert seen["env"][var] == "1", var
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals")
+def test_native_crash_reports_signal_exit_and_faulthandler_dump(tmp_path):
+    # The flake seen in CI was a bare "generated code exited -11: " with no
+    # evidence; a crash must now carry the exit signal and the native dump.
+    python = _fake_interpreter(
+        tmp_path,
+        "import faulthandler, os, signal\n"
+        "faulthandler.enable()\n"
+        "os.kill(os.getpid(), signal.SIGSEGV)\n",
+    )
+    result = verify_generated_code(GOOD, _fixture(), python=python)
+    assert not result.passed
+    [failure] = [f for f in result.failures if "exited" in f]
+    assert f"exited {-signal.SIGSEGV}" in failure
+    assert "Segmentation fault" in failure