From 16a23b7ac19dee292e7b0676d5d51afa757c3b6e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 15:18:22 +0200 Subject: [PATCH 1/2] Revert "fix(harness): crux launchers kill their fixture servers on TERM/INT/KILL (#5806)" Operator 2026-10-21 13:02Z: Python is banned; revert #5806 as ruled. The re-land is the guard rewritten in bash (bashrs-checked) or Rust, with no python3 dependency. check_harness_server_cleanup.sh exits 'ENV - python3 missing' in the sovereign-ci container (#5844 shard 5) and its apr-stub row races on guard-tree (#5854). Co-Authored-By: Claude Sonnet 5.5 --- .../817-harness-server-cleanup.cmd | 1 - scripts/check_crux_ollama_in_lock.sh | 24 +--- scripts/check_crux_serve_code.sh | 11 +- scripts/check_harness_server_cleanup.sh | 104 ------------------ scripts/crux_inference_dogfood.sh | 11 +- 5 files changed, 4 insertions(+), 147 deletions(-) delete mode 100644 ci/explicit-test-commands.d/817-harness-server-cleanup.cmd delete mode 100755 scripts/check_harness_server_cleanup.sh diff --git a/ci/explicit-test-commands.d/817-harness-server-cleanup.cmd b/ci/explicit-test-commands.d/817-harness-server-cleanup.cmd deleted file mode 100644 index 18ede22b41..0000000000 --- a/ci/explicit-test-commands.d/817-harness-server-cleanup.cmd +++ /dev/null @@ -1 +0,0 @@ -bash scripts/check_harness_server_cleanup.sh diff --git a/scripts/check_crux_ollama_in_lock.sh b/scripts/check_crux_ollama_in_lock.sh index 019145f785..0b66f0231e 100755 --- a/scripts/check_crux_ollama_in_lock.sh +++ b/scripts/check_crux_ollama_in_lock.sh @@ -50,26 +50,12 @@ SRV_PIDS=() _cleanup() { local p for p in "${SRV_PIDS[@]}"; do kill "$p" 2>/dev/null; done - # the apr stub servers run inside the dogfood children, so they are not in SRV_PIDS: kill each recorded pid - # that is still one of our stub servers (never a recycled pid) - for f in "${TMP:-/nonexistent}"/*.lock.srv; do - [ -f "$f" ] && p=$(cat "$f") && grep -aq stub_server.py "/proc/$p/cmdline" 2>/dev/null && kill "$p" 2>/dev/null - done case "${TMP:-}" in /tmp/?*|/var/folders/?*) rm -rf -- "$TMP" || : ;; *) : ;; esac } trap _cleanup EXIT -# guard_server : kill that server when this harness dies by ANY signal, SIGKILL included (a runner -# cancelling a job kills the tree with KILL, which no trap sees; the servers then lived 2-3 days at ppid 1). -# The watcher is its own process, off the harness's fds so it never holds a CI step's output pipe open. -guard_server() { - local srv="$1" par=$$ - # the id is read after the fork has exec'd, so a recycled pid (another cmdline) is left alone - ( trap - EXIT INT TERM; sleep 1; id=$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null); while kill -0 "$par" 2> /dev/null && [ "$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null)" = "$id" ]; do sleep 2; done - [ "$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null)" = "$id" ] && kill "$srv" 2> /dev/null ) < /dev/null > /dev/null 2>&1 & -} PASS=0 FAIL=0 @@ -186,12 +172,6 @@ case "${1:-}" in printf 'You: \nGoodbye!\n' ;; serve) port=""; while [ $# -gt 0 ]; do [ "$1" = --port ] && port="$2"; shift; done - # record this pid (exec keeps it) so the harness can kill the server if it dies mid-row - stub=$$; printf "%s\n" "$stub" > "${STUB_LOCK}.srv" - # a harness killed with SIGKILL runs no trap: this watcher ends the server when the harness is gone - ( sleep 1; id=$(tr "\0" " " < "/proc/$stub/cmdline" 2> /dev/null) - while kill -0 "$HARNESS_PID" 2> /dev/null && [ "$(tr "\0" " " < "/proc/$stub/cmdline" 2> /dev/null)" = "$id" ]; do sleep 2; done - [ "$(tr "\0" " " < "/proc/$stub/cmdline" 2> /dev/null)" = "$id" ] && kill "$stub" 2> /dev/null ) < /dev/null > /dev/null 2>&1 & exec python3 "$(dirname "$0")/stub_server.py" apr "$port" "$STUB_LOCK" "$STUB_LOG" ;; *) echo "stub apr: unhandled '$*'" >&2; exit 1 ;; esac @@ -211,10 +191,10 @@ run_row() { while [ "${#ROW_Q[@]}" -ge "$ROW_JOBS" ]; do wait "${ROW_Q[0]}" 2>/dev/null || true; ROW_Q=("${ROW_Q[@]:1}"); done lock="$TMP/$name.lock"; log="$TMP/$name.log"; : > "$lock"; : > "$log" port=$(free_port) - python3 "$BIN/stub_server.py" ollama "$port" "$lock" "$log" & SRV_PIDS+=("$!"); guard_server "$!" + python3 "$BIN/stub_server.py" ollama "$port" "$lock" "$log" & SRV_PIDS+=("$!") for i in $(seq 1 50); do curl -sf "http://127.0.0.1:$port/api/version" >/dev/null 2>&1 && break; sleep 0.1; done ( - ( cd "$tree" && HARNESS_PID="$$" STUB_LOCK="$lock" STUB_LOG="$log" CRUX_GPU_LOCK="$lock" GPUQ_BIN="$gpuq" \ + ( cd "$tree" && STUB_LOCK="$lock" STUB_LOG="$log" CRUX_GPU_LOCK="$lock" GPUQ_BIN="$gpuq" \ GPUQ_DIR="$TMP/$name.q" OLLAMA_BIN="$BIN/ollama" OLLAMA_HOST="127.0.0.1:$port" \ DOGFOOD_ALLOW_UNPINNED=1 APR="$BIN/apr" \ timeout 600 bash scripts/crux_inference_dogfood.sh 0.0.0 --model "$MODEL" --engines apr,ollama \ diff --git a/scripts/check_crux_serve_code.sh b/scripts/check_crux_serve_code.sh index f541d9c0e7..d41109d8e9 100755 --- a/scripts/check_crux_serve_code.sh +++ b/scripts/check_crux_serve_code.sh @@ -54,15 +54,6 @@ _cleanup() { esac } trap _cleanup EXIT -# guard_server : kill that server when this harness dies by ANY signal, SIGKILL included (a runner -# cancelling a job kills the tree with KILL, which no trap sees; the servers then lived 2-3 days at ppid 1). -# The watcher is its own process, off the harness's fds so it never holds a CI step's output pipe open. -guard_server() { - local srv="$1" par=$$ - # the id is read after the fork has exec'd, so a recycled pid (another cmdline) is left alone - ( trap - EXIT INT TERM; sleep 1; id=$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null); while kill -0 "$par" 2> /dev/null && [ "$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null)" = "$id" ]; do sleep 2; done - [ "$(tr "\0" " " < "/proc/$srv/cmdline" 2> /dev/null)" = "$id" ] && kill "$srv" 2> /dev/null ) < /dev/null > /dev/null 2>&1 & -} FAILS=0 ROWS=0 @@ -75,7 +66,7 @@ start() { # start : (re)start the fixture server; sets URL local port i port=$(free_port) python3 "$FAKE" --port "$port" --index "$1" --fault "$2" > /dev/null 2>&1 & - SRV=$!; guard_server "$SRV" + SRV=$! URL="http://127.0.0.1:$port" for i in $(seq 1 50); do curl -sf "$URL/health" > /dev/null 2>&1 && return 0; sleep 0.1; done printf '%s: ENV - fixture server did not come up\n' "$PROG" >&2 diff --git a/scripts/check_harness_server_cleanup.sh b/scripts/check_harness_server_cleanup.sh deleted file mode 100755 index b219c29624..0000000000 --- a/scripts/check_harness_server_cleanup.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -# check_harness_server_cleanup.sh: a CRUX harness that dies leaves no fixture server behind. -# infra-66 found stub_server.py and crux_fake_serve.py servers 2-3 days old (ppid 1) on the intel -# clean-room runners. bash runs its EXIT trap on TERM but nothing runs on SIGKILL (a cancelled job), so -# each launcher starts a watcher per server (guard_server). Behavioural rows; every mutant strips the -# mechanism from a copy of the launcher and must leak (a mutant that changes nothing is a BROKE). -# F guard_server itself, extracted from each of the 3 launchers: harness KILLed -> server gone -# S the call sites: each launcher calls guard_server on every server it starts (mutant: call removed) -# R the real harness run, KILLed: check_crux_serve_code (fake serve) and check_crux_ollama_in_lock -# (ollama stub, and the apr stub that dogfood execs inside a child) -# TERM is not a row: bash runs the EXIT trap on TERM with no trap of ours, so there is no code to prove. -# INT is not driven: a background job of a non-interactive shell starts with SIGINT ignored. -# Exit: 0 every row behaved · 1 a row broke · 2 ENV. -set -uo pipefail -case "${1:-}" in -h|--help) printf 'usage: bash scripts/check_harness_server_cleanup.sh (no arguments)\n'; exit 0 ;; esac -ROOT=$(cd "$(dirname "$0")/.." && pwd) || exit 2 -for t in python3 flock curl pgrep; do command -v "$t" > /dev/null 2>&1 || { printf 'check_harness_server_cleanup: ENV - %s missing\n' "$t" >&2; exit 2; }; done -T=$(mktemp -d) || exit 2 -trap 'rm -rf -- "${T:?}"' EXIT -FAIL=0 -LAUNCHERS="check_crux_serve_code check_crux_ollama_in_lock crux_inference_dogfood" -ok() { printf ' ok %s\n' "$1"; } -broke() { printf ' BROKE %s\n' "$1"; FAIL=$((FAIL + 1)); } -# mutant : a copy of that must differ; ROOT is pinned to the real tree -mutant() { - local src="$1" out="$2"; shift 2 - sed -e "$@" -e "s|^ROOT=.*|ROOT=\"$ROOT\"|" "$src" > "$out" - if cmp -s "$src" "$out"; then broke "mutant of $(basename "$src") changed nothing (vacuous)"; return 1; fi -} -# gone : wait up to 20 s for it to exit; prints gone|leaked and reaps a leak -# descendants : every live descendant, so a row never picks another session's server -descendants() { local c; for c in $(pgrep -P "$1" 2> /dev/null); do echo "$c"; descendants "$c"; done; } -# cmdl : the cmdline with NULs as spaces (read into a variable, so no pipe feeds a grep -q) -cmdl() { { tr "\0" " " < "/proc/$1/cmdline"; } 2> /dev/null; } -# reap : KILL what is left of a row's own fixture servers (checked by cmdline, so a recycled pid is spared) -reap() { local p c; for p in "$@"; do c=$(cmdl "$p"); grep -aqE "stub_server.py|crux_fake_serve|^sleep 300" <<< "$c" && kill -KILL "$p" 2> /dev/null; done; return 0; } -gone() { local i; for i in $(seq 1 100); do kill -0 "$1" 2> /dev/null || { echo gone; return; }; sleep 0.2; done; kill -KILL "$1" 2> /dev/null; echo leaked; } -verdict() { #