From 526b05efc0841331019f5cb988bf6470acd59bf1 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:59:45 +0530 Subject: [PATCH 1/2] ci: temporary campaign test matrix --- .github/workflows/campaign-matrix.yml | 333 ++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 .github/workflows/campaign-matrix.yml diff --git a/.github/workflows/campaign-matrix.yml b/.github/workflows/campaign-matrix.yml new file mode 100644 index 00000000..5c481ea1 --- /dev/null +++ b/.github/workflows/campaign-matrix.yml @@ -0,0 +1,333 @@ +name: Campaign matrix + +# Temporary test-only workflow: runs the fast test suite across operating +# systems and Python versions, builds the FreshCore extension on every OS, and +# checks each optional extra on Linux. It publishes nothing and writes nothing. + +on: + push: + branches: [campaign/matrix] + workflow_dispatch: + +concurrency: + group: campaign-matrix-${{ github.sha }} + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + suite: + name: suite (${{ matrix.os }}, py${{ matrix.python }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.9", "3.12", "3.13", "3.14"] + include: + - python: "3.9" + pandas: "pandas>=1.5,<2" + numpy: "numpy<2" + env: + PYTHONUTF8: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python }} + allow-prereleases: true + cache: pip + - name: Install + run: | + python -m pip install --upgrade pip + if pip install -e ".[dev,ml]" pytest-timeout; then + echo "EXTRA_INSTALLED=dev,ml" + else + echo "EXTRA_FALLBACK=dev" + echo "::warning::.[dev,ml] failed to install; falling back to .[dev]" + pip install -e ".[dev]" pytest-timeout + fi + if [ -n "${{ matrix.pandas }}" ]; then pip install "${{ matrix.pandas }}"; fi + if [ -n "${{ matrix.numpy }}" ]; then pip install "${{ matrix.numpy }}"; fi + python -c "import sys, platform, pandas, numpy; print('ENV', sys.version, platform.platform(), 'pandas', pandas.__version__, 'numpy', numpy.__version__)" + pip freeze > pip-freeze.txt + - name: Test (fast marker set) + run: | + python -m pytest -m "not online and not large" \ + -o addopts="--strict-markers -q -rfE" -p no:cacheprovider \ + --timeout 600 --junitxml=junit.xml + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-suite-${{ matrix.os }}-py${{ matrix.python }} + path: | + junit.xml + pip-freeze.txt + if-no-files-found: warn + + freshcore-native: + name: freshcore-native (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + RUSTUP_TOOLCHAIN: "1.98.1" + PYTHONUTF8: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + - name: Install Rust toolchain + run: | + rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal + rustc --version + cargo --version + - name: Install + # maturin develop installs into a virtualenv, so create one in the + # workspace and put it first on PATH for the remaining steps. + run: | + python -m venv .venv + if [ "$RUNNER_OS" = "Windows" ]; then + VENV_DIR="$(cygpath -w "$PWD/.venv")" + VENV_BIN="$(cygpath -w "$PWD/.venv/Scripts")" + VPY=".venv/Scripts/python.exe" + else + VENV_DIR="$PWD/.venv" + VENV_BIN="$PWD/.venv/bin" + VPY=".venv/bin/python" + fi + echo "VIRTUAL_ENV=$VENV_DIR" >> "$GITHUB_ENV" + echo "$VENV_BIN" >> "$GITHUB_PATH" + "$VPY" -m pip install --upgrade pip + "$VPY" -m pip install -e ".[dev,freshcore]" + - name: Show interpreter + run: | + which python + python -c "import sys; print(sys.executable, sys.prefix)" + - name: Rust unit tests + run: cargo test --locked --manifest-path crates/freshcore/Cargo.toml + - name: Build the extension + run: maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module + - name: FreshCore tests + run: | + python -c "import freshdata_freshcore; print(freshdata_freshcore.__file__)" + python -m pytest tests/test_execution -k freshcore -o addopts="--strict-markers" -q \ + -p no:cacheprovider --junitxml=junit.xml -rs + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-freshcore-${{ matrix.os }} + path: junit.xml + if-no-files-found: warn + + extras-linux: + name: extras-linux (${{ matrix.extra }}) + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + extra: [spark, airflow, dagster, dbt, kafka, flight, semantic, privacy, cleanlab, viz, notebook, domains, excel, entity-resolution, outofcore, all] + env: + EXTRA: ${{ matrix.extra }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + - name: Set up Java (spark) + if: matrix.extra == 'spark' || matrix.extra == 'all' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: temurin + java-version: "17" + - name: Resolve extra plan + run: | + case "$EXTRA" in + spark) + MODS="pyspark" + FD="from freshdata.execution._lazy import has_pyspark, require_pyspark; assert has_pyspark(); require_pyspark(); import freshdata.execution.backends._spark" + TESTS="tests/test_execution/test_spark_engine.py tests/test_execution/test_spark_engine_fixes.py" ;; + airflow) + MODS="airflow" + FD="from freshdata.integrations.airflow import FreshDataCleanOperator; print(FreshDataCleanOperator.__mro__)" + TESTS="tests/test_integrations/test_airflow.py tests/test_integrations/test_imports.py" ;; + dagster) + MODS="dagster" + FD="from freshdata.integrations.dagster import FreshDataResource, freshdata_asset_check; print(FreshDataResource)" + TESTS="tests/test_integrations/test_dagster.py tests/test_integrations/test_imports.py" ;; + dbt) + MODS="dbt.version sqlalchemy" + FD="from freshdata.integrations.dbt import FreshDataDbtTransform; print(FreshDataDbtTransform)" + TESTS="tests/test_integrations/test_dbt.py tests/test_integrations/test_dbt_tests_exporter.py tests/test_integrations/test_dbt_tests_exporter_yaml.py tests/test_integrations/test_dbt_transform_fail_audit.py tests/test_integrations/test_core.py" ;; + kafka) + MODS="kafka" + FD="from freshdata.streaming._connectors import kafka_batches; print(kafka_batches)" + TESTS="tests/test_streaming_cleaner.py tests/test_streaming_reports.py tests/test_streaming_cli.py" ;; + flight) + MODS="pyarrow.flight" + FD="from freshdata.streaming._connectors import flight_batches; print(flight_batches)" + TESTS="tests/test_streaming_cleaner.py tests/test_streaming_reports.py tests/test_streaming_cli.py" ;; + semantic) + MODS="onnxruntime tokenizers" + FD="from freshdata.models._lazy import has_semantic_extra; assert has_semantic_extra()" + TESTS="tests/test_semantic_extra_optional.py tests/test_models_runtime.py tests/test_models_download_robustness.py tests/test_models_registry.py tests/test_semantic_backends.py tests/test_cli_models.py" ;; + privacy) + MODS="presidio_analyzer presidio_anonymizer pyffx" + FD="import freshdata.enterprise.privacy as p; print(p.__name__)" + TESTS="tests/test_enterprise_privacy.py tests/test_privacy_adversarial.py tests/test_privacy_keyless_defaults.py tests/test_privacy_missing_and_labels.py tests/test_privacy_policy.py tests/test_privacy_policy_classification.py" ;; + cleanlab) + MODS="cleanlab sklearn" + FD="import freshdata.enterprise.cleaner as c; print(c.__name__)" + TESTS="tests/test_enterprise_cleaner.py" ;; + viz) + MODS="itables plotly great_tables" + FD="from freshdata.render import require; [require(p) for p in ('itables', 'plotly', 'great_tables')]" + TESTS="tests/test_render_html.py tests/test_peel_view.py tests/test_peel_notebook.py tests/test_report_rendering_fixes.py" ;; + notebook) + MODS="itables plotly great_tables anywidget IPython" + FD="from freshdata.render import require; [require(p) for p in ('itables', 'plotly', 'great_tables', 'anywidget', 'IPython')]; import freshdata.render.notebook" + TESTS="tests/test_render_html.py tests/test_peel_view.py tests/test_peel_notebook.py tests/test_report_rendering_fixes.py" ;; + domains) + MODS="yaml" + FD="import freshdata.domains as d; print(d.__name__)" + TESTS="tests/domains tests/test_domain_rules_transport_icpn.py" ;; + excel) + MODS="openpyxl" + FD="import freshdata as fd; assert callable(fd.clean_excel)" + TESTS="tests/test_clean_excel.py tests/test_csv_formula_sanitize.py" ;; + entity-resolution) + MODS="duckdb" + FD="import freshdata.enterprise.entity_resolution as er; print(er.__name__)" + TESTS="tests/test_enterprise_entity_resolution.py tests/test_enterprise_entity_resolution_dedupe.py tests/test_link.py tests/test_er_comparators.py tests/test_er_review_loop.py" ;; + outofcore) + MODS="polars duckdb pyarrow" + FD="from freshdata.execution._lazy import require_polars, require_duckdb, require_pyarrow; require_polars(); require_duckdb(); require_pyarrow()" + TESTS="tests/test_execution tests/test_polars_adapter.py" ;; + all) + MODS="polars pyarrow sklearn yaml cleanlab duckdb presidio_analyzer presidio_anonymizer pyffx itables plotly great_tables anywidget IPython openpyxl" + FD="import freshdata.enterprise.privacy, freshdata.enterprise.cleaner, freshdata.render.notebook, freshdata.domains" + TESTS="tests" ;; + *) echo "unknown extra $EXTRA"; exit 1 ;; + esac + { + echo "MODS=$MODS" + echo "FD_SMOKE=$FD" + echo "TESTS=$TESTS" + } >> "$GITHUB_ENV" + - name: Extra-only install (no dev) and import smoke + # Checks the extra is self-sufficient: the dev extra already ships some + # of these packages, which would hide a missing dependency. + continue-on-error: true + run: | + python -m venv /tmp/xonly + /tmp/xonly/bin/python -m pip install --upgrade pip + /tmp/xonly/bin/pip install -e ".[$EXTRA]" + /tmp/xonly/bin/python - <<'PY' + import importlib, os, sys + bad = [] + for m in os.environ["MODS"].split(): + try: + importlib.import_module(m) + print("EXTRA_ONLY_IMPORT_OK", m) + except Exception as exc: + print("EXTRA_ONLY_IMPORT_FAIL", m, type(exc).__name__, exc) + bad.append(m) + try: + exec(os.environ["FD_SMOKE"]) + print("EXTRA_ONLY_FD_SMOKE_OK") + except Exception as exc: + print("EXTRA_ONLY_FD_SMOKE_FAIL", type(exc).__name__, exc) + bad.append("freshdata-smoke") + sys.exit(1 if bad else 0) + PY + - name: Install (dev + extra) + run: | + python -m pip install --upgrade pip + if ! pip install -e ".[dev,$EXTRA]" pytest-timeout; then + echo "EXTRA_INSTALL_FAIL=$EXTRA" + echo "::error::pip install -e .[dev,$EXTRA] failed" + exit 1 + fi + echo "EXTRA_INSTALL_OK=$EXTRA" + pip freeze > pip-freeze.txt + - name: Import smoke + run: | + python - <<'PY' + import importlib, os, sys + bad = [] + for m in os.environ["MODS"].split(): + try: + mod = importlib.import_module(m) + print("IMPORT_OK", m, getattr(mod, "__version__", "")) + except Exception as exc: + print("IMPORT_FAIL", m, type(exc).__name__, exc) + bad.append(m) + try: + exec(os.environ["FD_SMOKE"]) + print("FD_SMOKE_OK") + except Exception as exc: + print("FD_SMOKE_FAIL", type(exc).__name__, exc) + bad.append("freshdata-smoke") + sys.exit(1 if bad else 0) + PY + - name: Related tests + run: | + # shellcheck disable=SC2086 + python -m pytest $TESTS -m "not online and not large" \ + -o addopts="--strict-markers -q -rs" -p no:cacheprovider \ + --timeout 600 --junitxml=junit.xml + - name: Count skips for missing modules + if: always() + run: | + python - <<'PY' + import os, re, xml.etree.ElementTree as ET + if not os.path.exists("junit.xml"): + print("NO_JUNIT"); raise SystemExit(0) + root = ET.parse("junit.xml").getroot() + pat = re.compile(r"No module named|could not import|not installed|unavailable|requires? .{0,40}install", re.I) + total = passed = failed = skipped = missing = 0 + reasons = {} + for tc in root.iter("testcase"): + total += 1 + kids = {c.tag: c for c in tc} + if "failure" in kids or "error" in kids: + failed += 1 + elif "skipped" in kids: + skipped += 1 + msg = kids["skipped"].get("message", "") or (kids["skipped"].text or "") + if pat.search(msg): + missing += 1 + reasons[msg[:160]] = reasons.get(msg[:160], 0) + 1 + else: + passed += 1 + line = (f"EXTRA_RESULT extra={os.environ['EXTRA']} total={total} passed={passed} " + f"failed={failed} skipped={skipped} skipped_missing_module={missing}") + print(line) + for r, n in sorted(reasons.items(), key=lambda kv: -kv[1]): + print(f"SKIP_MISSING [{n}] {r}") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as fh: + fh.write(line + "\n") + PY + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-extra-${{ matrix.extra }} + path: | + junit.xml + pip-freeze.txt + if-no-files-found: warn From c6a57488cbb1a0d7898a27101e375e8d581d5995 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:05:26 +0530 Subject: [PATCH 2/2] ci: validate the truthbench Windows sandbox fix (temporary, do not merge) --- benchmarks/truthbench/generated_code.py | 90 ++++++++++++--- benchmarks/truthbench/runner.py | 8 ++ tests/truthbench/test_generated_code.py | 128 ++++++++++++++++++++- tests/truthbench/test_runner_report_cli.py | 29 +++++ 4 files changed, 235 insertions(+), 20 deletions(-) diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py index 317a3dbd..58d7594f 100644 --- a/benchmarks/truthbench/generated_code.py +++ b/benchmarks/truthbench/generated_code.py @@ -12,9 +12,11 @@ from __future__ import annotations import ast +import os import subprocess import sys import tempfile +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -97,6 +99,11 @@ def _stderr_excerpt(stderr: str) -> str: #: and every crash is still recorded in ``GeneratedCodeResult.native_crashes``. _NATIVE_CRASH_RETRIES = 1 +#: The harness prints this line before anything else. A child that exits +#: without it never ran the harness (the interpreter failed to start), so none +#: of the post-run checks below observed the generated code. +_STARTED_MARKER = "truthbench-sandbox-child-started" + @dataclass(frozen=True) class GeneratedCodeResult: @@ -111,6 +118,48 @@ class GeneratedCodeResult: #: One redacted excerpt per signal exit, including crashes that a retry #: recovered from, so a flaky pass is never silent. native_crashes: tuple[str, ...] = () + #: Set when the sandbox child could not start the harness. The generated + #: code never ran, so this result says nothing about its behavior and the + #: caller must treat it as an infrastructure failure, not a verdict. + infrastructure_failure: str | None = None + + +def _sandbox_env( + workdir: Path, + *, + os_name: str | None = None, + environ: Mapping[str, str] | None = None, +) -> dict[str, str]: + """The deliberately minimal environment of the sandbox child.""" + + env = { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONNOUSERSITE": "1", + "FRESHDATA_NO_NETWORK": "1", + "HOME": str(workdir), + "TMPDIR": str(workdir), + # The environment is otherwise empty, so native thread pools would + # size themselves from the host core count; pin them (and the + # locale) so runs are deterministic across machines. + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "POLARS_MAX_THREADS": "1", + "RAYON_NUM_THREADS": "1", + "LANG": "C.UTF-8", + } + if (os_name if os_name is not None else os.name) == "nt": + # CPython <= 3.10 on Windows seeds hash randomization through + # CryptGenRandom, which cannot load its provider without SystemRoot: + # the child dies with "Fatal Python error: _Py_HashRandomization_Init: + # failed to get random numbers". 3.11+ uses BCryptGenRandom and starts + # without it. Windows variable names are case-insensitive, so one key + # covers SystemRoot and SYSTEMROOT. + source = os.environ if environ is None else environ + system_root = source.get("SystemRoot") or source.get("SYSTEMROOT") + if system_root: + env["SystemRoot"] = system_root + return env def _allowlist_failures(tree: ast.AST) -> list[str]: @@ -141,6 +190,10 @@ def _harness(code_path: str, input_name: str) -> str: return f""" import sys +# First output: proves the interpreter started and is running this harness. +sys.stdout.write({_STARTED_MARKER!r} + "\\n") +sys.stdout.flush() + # Pre-import the allowed libraries so their own internal use of stdlib # modules (pandas imports subprocess for locale probing) completes first... import pandas # noqa: F401 @@ -220,22 +273,7 @@ def leaks(label: str, payload: Any) -> None: harness_path.write_text( _harness(str(code_path), _INPUT_BASENAME), encoding="utf-8" ) - env = { - "PYTHONDONTWRITEBYTECODE": "1", - "PYTHONNOUSERSITE": "1", - "FRESHDATA_NO_NETWORK": "1", - "HOME": str(workdir), - "TMPDIR": str(workdir), - # The environment is otherwise empty, so native thread pools would - # size themselves from the host core count; pin them (and the - # locale) so runs are deterministic across machines. - "OMP_NUM_THREADS": "1", - "OPENBLAS_NUM_THREADS": "1", - "MKL_NUM_THREADS": "1", - "POLARS_MAX_THREADS": "1", - "RAYON_NUM_THREADS": "1", - "LANG": "C.UTF-8", - } + env = _sandbox_env(workdir) interpreter = python or sys.executable def redacted(text: str) -> str: @@ -270,8 +308,26 @@ def redacted(text: str) -> str: stages=(*stages, "execute"), native_crashes=tuple(native_crashes), ) + stdout, stderr = proc.stdout or "", proc.stderr or "" + started = f"{_STARTED_MARKER}\n" + if proc.returncode >= 0 and not stdout.startswith(started): + # Fail closed: the generated code never ran, so the overwrite and + # canary checks below would observe nothing; report no verdict. + reason = ( + "sandbox infrastructure failure: the child interpreter exited " + f"with code {proc.returncode} before starting the harness: " + f"{_stderr_excerpt(redacted(stderr))}" + ) + return GeneratedCodeResult( + False, + (*failures, reason), + stderr=redacted(stderr), + stages=tuple(stages), + native_crashes=tuple(native_crashes), + infrastructure_failure=reason, + ) + stdout = stdout[len(started) :] stages.append("execute") - stdout, stderr = proc.stdout, proc.stderr if proc.returncode != 0: failures.append( f"generated code exited {proc.returncode}: " diff --git a/benchmarks/truthbench/runner.py b/benchmarks/truthbench/runner.py index 0ad203d6..159a14b2 100644 --- a/benchmarks/truthbench/runner.py +++ b/benchmarks/truthbench/runner.py @@ -742,6 +742,14 @@ def run_release( file=sys.stderr, ) + if any(result.infrastructure_failure for result in generated_results): + # The sandbox child never ran the generated code, so the sandbox + # checks observed nothing; that is not a gate verdict. + raise TruthBenchRunError( + "generated-code sandbox could not run: " + + "; ".join(sandbox_failures) + ) + observations, failures, parity_ledger = _parity_observations(parity, backends) parity_failures.extend( failure for failure in failures if failure not in parity_failures diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py index 17c4b578..c82be195 100644 --- a/tests/truthbench/test_generated_code.py +++ b/tests/truthbench/test_generated_code.py @@ -47,6 +47,9 @@ def _fixture(): print("rows:", len(df)) """ +#: What a child that started the harness prints first. +_STARTED = gc._STARTED_MARKER + "\n" + def test_wellformed_code_passes_all_stages(): result = verify_generated_code(GOOD, _fixture()) @@ -90,14 +93,16 @@ def test_runtime_poison_blocks_banned_module_even_if_statically_allowed(monkeypa gc, "ALLOWED_IMPORTS", frozenset({*gc.ALLOWED_IMPORTS, "socket"}) ) result = verify_generated_code("import socket\n", _fixture()) + assert result.infrastructure_failure is None, result.failures assert not result.passed - assert any("exited" in f for f in result.failures) + assert any("generated code exited" in f for f in result.failures) def test_timeout_is_enforced(): result = verify_generated_code( "while True:\n pass\n", _fixture(), timeout=3.0 ) + assert result.infrastructure_failure is None, result.failures assert not result.passed assert any("timeout" in f for f in result.failures) @@ -116,6 +121,7 @@ def test_pii_canary_in_stdout_is_reported(): 'print(df["memo"].tolist())\n' ) result = verify_generated_code(code, _fixture()) + assert result.infrastructure_failure is None, result.failures assert not result.passed assert any("stdout leaked canary" in f for f in result.failures) assert "example.invalid" not in result.stdout # evidence itself is redacted @@ -128,6 +134,7 @@ def test_input_file_overwrite_is_reported(): 'df.head(1).to_csv("your_data.csv", index=False)\n' ) result = verify_generated_code(code, _fixture()) + assert result.infrastructure_failure is None, result.failures assert not result.passed assert any("modified its input file" in f for f in result.failures) @@ -145,6 +152,7 @@ def test_sandbox_pins_native_threads_and_enables_faulthandler(tmp_path): python = _fake_interpreter( tmp_path, "import json, os, sys\n" + f"print({gc._STARTED_MARKER!r})\n" "print(json.dumps({'argv': sys.argv[1:], 'env': dict(os.environ)}))\n", ) result = verify_generated_code(GOOD, _fixture(), python=python) @@ -212,7 +220,7 @@ def test_ordinary_failure_keeps_traceback_tail(monkeypatch): stderr = "noise\n" * 300 + "ValueError: bad column\n" def failed_child(args, **kwargs): - return subprocess.CompletedProcess(args, 1, "", stderr) + return subprocess.CompletedProcess(args, 1, _STARTED, stderr) monkeypatch.setattr(gc.subprocess, "run", failed_child) result = verify_generated_code(GOOD, _fixture()) @@ -234,7 +242,7 @@ def _scripted_children(monkeypatch, outcomes): def child(args, **kwargs): code, stderr = outcomes[min(len(calls), len(outcomes) - 1)] calls.append(code) - return subprocess.CompletedProcess(args, code, "", stderr) + return subprocess.CompletedProcess(args, code, _STARTED, stderr) monkeypatch.setattr(gc.subprocess, "run", child) return calls @@ -265,3 +273,117 @@ def test_ordinary_failure_is_not_retried(monkeypatch): assert calls == [1] assert not result.passed assert result.native_crashes == () + + +_POSIX_ENV_KEYS = { + "PYTHONDONTWRITEBYTECODE", + "PYTHONNOUSERSITE", + "FRESHDATA_NO_NETWORK", + "HOME", + "TMPDIR", + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "POLARS_MAX_THREADS", + "RAYON_NUM_THREADS", + "LANG", +} +_HOST_ENV = {"SYSTEMROOT": r"C:\Windows", "PATH": r"C:\bin", "USERPROFILE": r"C:\u"} + + +@pytest.mark.parametrize("key", ["SYSTEMROOT", "SystemRoot"]) +def test_sandbox_env_keeps_system_root_on_windows(tmp_path, key): + # CPython <= 3.10 on Windows cannot seed hash randomization without it. + host = {key: r"C:\Windows", "PATH": r"C:\bin", "USERPROFILE": r"C:\u"} + env = gc._sandbox_env(tmp_path, os_name="nt", environ=host) + assert env["SystemRoot"] == r"C:\Windows" + assert set(env) == _POSIX_ENV_KEYS | {"SystemRoot"} + + +def test_sandbox_env_on_posix_is_unchanged(tmp_path): + env = gc._sandbox_env(tmp_path, os_name="posix", environ=_HOST_ENV) + assert env == { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONNOUSERSITE": "1", + "FRESHDATA_NO_NETWORK": "1", + "HOME": str(tmp_path), + "TMPDIR": str(tmp_path), + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "POLARS_MAX_THREADS": "1", + "RAYON_NUM_THREADS": "1", + "LANG": "C.UTF-8", + } + + +def test_windows_child_receives_system_root(monkeypatch): + class _WindowsOs: + name = "nt" + environ = _HOST_ENV + + seen = {} + + def child(args, **kwargs): + seen.update(kwargs["env"]) + return subprocess.CompletedProcess(args, 0, _STARTED + "rows: 2\n", "") + + monkeypatch.setattr(gc, "os", _WindowsOs) + monkeypatch.setattr(gc.subprocess, "run", child) + result = verify_generated_code(GOOD, _fixture()) + assert result.passed, result.failures + assert seen["SystemRoot"] == r"C:\Windows" + assert "PATH" not in seen + assert result.stdout == "rows: 2\n" + + +_STARTUP_FATAL = ( + "Fatal Python error: _Py_HashRandomization_Init: failed to get random " + "numbers to initialize Python\nPython runtime state: preinitialized\n\n" +) + + +@pytest.mark.parametrize( + "code", + [ + GOOD, + 'import pandas as pd\nprint(pd.read_csv("your_data.csv")["memo"].tolist())\n', + 'import pandas as pd\npd.DataFrame().to_csv("your_data.csv")\n', + ], +) +def test_child_that_cannot_start_is_an_infrastructure_failure(monkeypatch, code): + # Seen on Windows + CPython 3.9: the child died at startup, and each case + # reported an ordinary "generated code exited 1", so the canary and + # overwrite checks passed vacuously. + calls = [] + + def child_never_started(args, **kwargs): + calls.append(args) + return subprocess.CompletedProcess(args, 1, "", _STARTUP_FATAL) + + monkeypatch.setattr(gc.subprocess, "run", child_never_started) + result = verify_generated_code(code, _fixture()) + assert not result.passed + assert result.infrastructure_failure + assert "_Py_HashRandomization_Init" in result.infrastructure_failure + assert result.failures == (result.infrastructure_failure,) + assert "execute" not in result.stages + assert not any("generated code exited" in f for f in result.failures) + assert len(calls) == 1 # a startup failure is not retried as a native crash + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX shebang interpreter") +def test_child_exiting_cleanly_without_running_the_harness_fails_closed(tmp_path): + python = _fake_interpreter(tmp_path, "import sys\nsys.exit(0)\n") + result = verify_generated_code(GOOD, _fixture(), python=python) + assert not result.passed + assert "exited with code 0 before starting the harness" in ( + result.infrastructure_failure or "" + ) + + +def test_normal_run_reports_generated_stdout_without_the_start_marker(): + result = verify_generated_code(GOOD, _fixture()) + assert result.passed, result.failures + assert result.infrastructure_failure is None + assert result.stdout == "rows: 2\n" diff --git a/tests/truthbench/test_runner_report_cli.py b/tests/truthbench/test_runner_report_cli.py index e60f0129..94d651f9 100644 --- a/tests/truthbench/test_runner_report_cli.py +++ b/tests/truthbench/test_runner_report_cli.py @@ -14,6 +14,7 @@ import pytest from benchmarks.truthbench import cli from benchmarks.truthbench import runner as runner_module +from benchmarks.truthbench.generated_code import GeneratedCodeResult from benchmarks.truthbench.minimize import minimize_failure from benchmarks.truthbench.models import GateResult, RunResult from benchmarks.truthbench.report import compare_to_baseline, write_artifacts @@ -109,6 +110,34 @@ def missing(name: str) -> str: run_release(domains=("finance",), write=False) +def test_sandbox_child_that_cannot_start_is_an_infrastructure_failure( + monkeypatch, tmp_path, capsys +): + # A sandbox child that dies at interpreter startup never ran the generated + # code; the run must fail as infrastructure (exit 2), not grade a gate the + # regression ratchet could wave through. + reason = "sandbox infrastructure failure: the child interpreter exited with code 1" + + def child_never_started(code, fixture, **_kwargs): + return GeneratedCodeResult( + False, (reason,), stages=("parse", "allowlist", "compile"), + infrastructure_failure=reason, + ) + + monkeypatch.setattr(runner_module, "verify_generated_code", child_never_started) + code = cli.main( + [ + "run", "--domains", "finance", "--backends", "pandas", + "--results-dir", str(tmp_path), "--check-regressions", + ] + ) + assert code == 2 + err = capsys.readouterr().err + assert "INFRASTRUCTURE FAILURE" in err + assert "sandbox could not run" in err + assert not (tmp_path / "latest.json").exists() + + def test_minimizer_never_removes_the_target_cell(): fixture = parity_fixture() target = next(c for c in fixture.cells if c.row_id == "par-01" and c.column == "name")