diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index 53dafd909d..c33e72b56d 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -707,6 +707,11 @@ jobs: if: ${{ !cancelled() && steps.decide.outputs.proceed == 'true' }} run: | set -uo pipefail + # -e OFF, as in the perf041 step: the default shell is `bash -e {0}`, so the + # RED control's expected exit 101 aborted the step at `; r=$?` before its + # assertions ran (run 37161229337: GREEN passed, step failed 101, no ::error::). + # Every failure path below is guarded explicitly. + set +e t=api::openai_handlers::qwen35_serve_tests::gpu_every_serve_route_prefills_through_the_batched_prefill model="$HOME/models/Qwen3.5-0.8B-Q4_K_M.gguf" [ -f "$model" ] || { echo "::error::#4250 UNMEASURABLE on ${{ matrix.name }} (provisioning): $model is absent"; exit 1; } diff --git a/docs/roadmaps/entries/PMAT-4715.yaml b/docs/roadmaps/entries/PMAT-4715.yaml new file mode 100644 index 0000000000..398c99140b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4715.yaml @@ -0,0 +1,19 @@ +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index f65016f6ce..6d12cec68b 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24060,3 +24060,22 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:docs notes: 'Operator C197, 2026-10-01 12:40Z: re-date via the spec''s own procedure; check unchanged; not a waiver.' +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.'