Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/roadmaps/roadmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17783,3 +17783,22 @@ roadmap:
labels:
- kind:code
notes: null
- id: PMAT-3235
github_issue: null
item_type: task
title: 'examples-nightly: 6 of 22 red rows are an example built without its feature, not a broken example'
status: inprogress
priority: medium
assigned_to: null
created: 2026-09-14T20:05:00Z
updated: 2026-09-14T20:05:00Z
spec: docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md
acceptance_criteria: []
phases: []
subtasks: []
estimated_effort: null
labels:
- kind:code
notes: 'adds needs-feature beside needs-args/needs-hardware/needs-data; like
needs-hardware it is a SKIP that must be re-run with the feature before a release
verdict, never read as the example working'
57 changes: 51 additions & 6 deletions scripts/dogfood_examples.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
# a driver-less host must not be readable as "the CUDA example
# works". These rows are re-run on the CUDA host before the
# release verdict (G3.EX).
# needs-feature (Added 2026-09-14): 6 rows on run 34826552378 were classified
# `fail` whose output says the example is not built with the
# feature it needs. Not runnable in THIS configuration, not
# broken. Re-run WITH the feature before a release verdict.
# needs-data (Added 2026-09-12): 173 rows on the 0.67 train head (981
# example targets) were classified `fail` whose stderr says
# the example needs a FILE the repository does not ship.
Expand Down Expand Up @@ -76,6 +80,15 @@ SELFTEST=0
NEEDS_ARGS_RE='^(Usage|error: the following required arguments)'
NEEDS_HW_RE='(CUDA_ERROR_[A-Z_]+|no CUDA-capable device|CUDA driver version is insufficient|cuInit|libcuda\.so|libnvidia-ml|[Nn]o (suitable )?(graphics )?adapter|RequestAdapterError|NoAdapter|wgpu.*(device|adapter) (not|un)|Metal device (not|un))'
NEEDS_DATA_RE='No such file or directory|[Mm]odel not found|not found at |Failed to open |[Nn]o tokenizer|Download with:|does not exist|hf://'
# needs-feature (Added 2026-09-14): 6 rows on run 34826552378 were classified
# `fail` whose output says the example is not built with the feature it needs --
# either the example's own guard ("This example requires the 'compression'
# feature.") or cargo refusing the target outright ("requires the features:
# cuda"). Not runnable in THIS configuration; not broken. Like needs-hardware,
# these must be re-run WITH the feature before a release verdict -- a skip here
# must never be readable as "the compression example works".
NEEDS_FEATURE_RE="requires the '[^']+' feature|requires the features?:"


# ---------------------------------------------------------------------------
# helpers
Expand Down Expand Up @@ -193,6 +206,11 @@ classify() {
printf 'needs-data\t%s\n' "$(oneline "${line}")"
return 0
fi
line=$(grep -m1 -E "${NEEDS_FEATURE_RE}" "${log}" 2> /dev/null) || line=''
if [ -n "${line}" ]; then
printf 'needs-feature\t%s\n' "$(oneline "${line}")"
return 0
fi
line=$(grep -m1 -E '[^[:space:]]' "${log}" 2> /dev/null) || line=''
printf 'fail\t%s: %s\n' "${stage}" "$(oneline "${line:-no output}")"
}
Expand Down Expand Up @@ -399,6 +417,33 @@ selftest() {
st_expect 'class: needs_arg -> needs-args' "${out}" "${p}" needs_arg needs-args
st_expect 'class: nohw -> needs-hardware' "${out}" "${p}" nohw needs-hardware
st_expect 'class: nodata -> needs-data' "${out}" "${p}" nodata needs-data
st_expect 'class: nofeature -> needs-feature' "${out}" "${p}" nofeature needs-feature

# NEEDS_FEATURE_RE, both polarities. A skip class is the dangerous kind of
# addition -- it can only ever turn a `fail` into a non-failure -- so the
# regex ships the strings it must NOT match alongside the ones it must.
# "feature is required for this operation" is the near-miss that decides it.
local _m _s
for _s in "This example requires the '"'"'compression'"'"' feature." \
"Error: This example requires the '"'"'tensor'"'"' feature." \
"target \`gpu_info\` in package requires the features: cuda"; do
if grep -qE "${NEEDS_FEATURE_RE}" <<< "${_s}"; then
st_row PASS "re: matches -- $(oneline "${_s}")"
else
st_row FAIL "re: SHOULD match -- $(oneline "${_s}")"
fi
done
for _s in "thread '"'"'main'"'"' panicked at src/main.rs:11:39" \
"error[E0432]: unresolved import \`trueno_gpu::driver::PinnedBuffer\`" \
"Error: Os { code: 6, kind: Uncategorized }" \
"assertion failed: left == right" \
"feature is required for this operation"; do
if grep -qE "${NEEDS_FEATURE_RE}" <<< "${_s}"; then
st_row FAIL "re: MUST NOT match -- $(oneline "${_s}")"
else
st_row PASS "re: does not match -- $(oneline "${_s}")"
fi
done

# rc of the failing example is recorded, not flattened to 1.
if awk -F'\t' '$2 == "bad" && $4 == 3 { f = 1 } END { exit !f }' "${out}"; then
Expand All @@ -409,11 +454,11 @@ selftest() {
fi

# Every citing class cites a LINE, not an empty cell.
if awk -F'\t' '$3 == "needs-args" || $3 == "needs-hardware" || $3 == "needs-data" { if ($6 == "") bad = 1 } END { exit bad }' \
if awk -F'\t' '$3 == "needs-args" || $3 == "needs-hardware" || $3 == "needs-data" || $3 == "needs-feature" { if ($6 == "") bad = 1 } END { exit bad }' \
"${out}"; then
st_row PASS 'cite: the three skip classes cite a line'
st_row PASS 'cite: the four skip classes cite a line'
else
st_row FAIL 'cite: the three skip classes cite a line'
st_row FAIL 'cite: the four skip classes cite a line'
fi

# Trailer counts.
Expand All @@ -431,11 +476,11 @@ selftest() {
fi

# No unclassified row.
if awk -F'\t' '/^#/ { next } { if ($3 !~ /^(pass|fail|timeout|needs-args|needs-hardware|needs-data)$/) bad = 1 } END { exit bad }' \
if awk -F'\t' '/^#/ { next } { if ($3 !~ /^(pass|fail|timeout|needs-args|needs-hardware|needs-data|needs-feature)$/) bad = 1 } END { exit bad }' \
"${out}"; then
st_row PASS 'rows: every row carries one of the six classes'
st_row PASS 'rows: every row carries one of the seven classes'
else
st_row FAIL 'rows: every row carries one of the six classes'
st_row FAIL 'rows: every row carries one of the seven classes'
fi

st_mutation_row "${td}" "${ws}" "${p}"
Expand Down
10 changes: 10 additions & 0 deletions tests/fixtures/dogfood_examples/examples/nofeature.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//! Classification witness: `needs-feature`.
//!
//! The real rows this stands for, from examples-nightly run 34826552378:
//! aprender-db::compressed_kv This example requires the 'compression' feature.
//! aprender-rag::semantic_embeddings This example requires the 'embeddings' feature.
//! aprender-distribute::tensor_example Error: This example requires the 'tensor' feature.
fn main() {
eprintln!("Error: This example requires the 'compression' feature.");
std::process::exit(1);
}
Loading