feat(affected-set): no-reader paths give READERS, not ALL (ALL 18->14 of 20; Refs #5809)

.github/workflows/guards-nightly.yml

@@ -349,6 +349,27 @@ jobs:
           exec > >(tee -a "$RUNNER_TEMP/census.log") 2>&1
           setsid --wait bash scripts/census_ratchet.sh --self-test
           CENSUS_BASE_REF="${RATCHET_24H_BASE:?no 24 h comparand (#5182)}" setsid --wait bash scripts/census_ratchet.sh
+      # #5809 (FLOW-010 E5): the affected-set tool's case tables. LAB, never on a PR path (C324 rules 5/7):
+      # the tool is not consumed by any gate yet; a red opens or updates the one LAB ticket below, never a block.
+      - name: Affected-set tool and readers guards (FLOW-010 E5, #5809)
+        id: affected_set
+        if: ${{ !cancelled() }}
+        run: |
+          set -euo pipefail
+          exec > >(tee -a "$RUNNER_TEMP/affected.log") 2>&1
+          setsid --wait bash scripts/check_affected_set.sh
+          setsid --wait bash scripts/check_affected_readers.sh
+          setsid --wait bash scripts/check_affected_readers.sh --self-test
+      - name: A red affected-set night opens or updates the one LAB ticket (#5809)
+        if: failure() && steps.affected_set.outcome == 'failure'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          bash scripts/lab_ticket.sh --label lab-affected-set --title "LAB: nightly affected-set guards red (#5809)" \
+            --owner aprender-w5018 --log "$RUNNER_TEMP/affected.log" --event "$GITHUB_EVENT_NAME" --run-url "$RUN_URL"
       - name: A red census night opens or updates the one LAB ticket (#4709)
         if: failure() && steps.ratchet_census.outcome == 'failure'
         env:

scripts/affected_set.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# FLOW-010 E5: the diff -> the set of workspace packages whose tests it can change.
+#
+#   scripts/affected_set.sh [--filter] [--diff-from FILE] [BASE [HEAD]]
+#
+# Prints one package per line, or the single word ALL. With --filter it prints the
+# nextest filterset instead (`package(a) | package(b)`, or `all()` for ALL).
+# Fail-safe: anything this tool cannot place in a package is ALL.
+#   crates/<pkg>/...        the package + its reverse-dependency closure (cargo metadata)
+#   a CORE crate            ALL (its closure is the workspace; naming it would only hide that)
+#   Cargo.toml of a crate   ALL (a manifest edit changes features/deps, which no path can bound)
+#   ci/ (sections.yml, explicit-test-commands.d/, ...), Cargo.lock, root Cargo.toml, rust-toolchain.toml, .cargo/, .github/, Makefile,
+#   any other path outside a crate            ALL
+#   docs/, contracts/, root *.md              READERS only (FLOW-010 b): their run-time readers are all in READERS. Exceptions, ALL:
+#                           contracts binding.yaml, contracts model-families and the BUILD_CONTRACTS files (compiled into a crate)
+#   .claude/, ci/shard-step-costs.tsv
+#                           NO-READER list (#5809): the only run-time readers of these paths (grep of crates/**/*.rs: contracts-cli,
+#                           orchestrate, core, mcp, apr-cli) are all in READERS, so the set is READERS alone. Alone or
+#                           beside crate .rs files; any other path in the diff still decides ALL.
+# Any non-.rs file under crates/<pkg>/ (README.md, *.json, *.toml, ...) is ALL: other crates' tests read them
+# (aprender-core readme_contract.rs reads every crates/*/README.md; aprender-contracts reads aprender-serve/kernel-registry.json).
+# An empty diff, or a diff git cannot compute (bad base: exit 128), is ALL: nothing was
+# measured, so nothing may be skipped.
+# git diff runs with --no-renames: a moved file is its old path AND its new path (a rename
+# out of a crate would otherwise show only the destination and the source crate would be skipped).
+# --diff-from reads the changed paths from FILE (tests). AFFECTED_META / AFFECTED_ROOT
+# replace `cargo metadata` and the checkout root (tests; synthetic graphs).
+set -euo pipefail
+# CORE = ALL on touch. apr-cli is here because other crates' tests read its sources (aprender-core readme_contract.rs reads
+# apr-cli/src/commands_enum.rs; spec_checklist_*, toyota_principles_tests). Reader edges are not in cargo metadata, so a
+# crate whose source another crate reads at run time must be listed (aprender-quant: aprender-serve/capability.rs reads
+# aprender-quant/src/ggml_type.rs). Any crates/*/build.rs is ALL: contracts/build_helper.rs reads crates/*/build.rs.
+CORE="${AFFECTED_CORE:-aprender-core aprender-compute aprender-contracts apr-cli aprender-quant}"
+# READERS = crates whose tests read OTHER crates' files at run time (CARGO_MANIFEST_DIR/.., crates/, ../<crate>). CORE ones are ALL on touch, but that does not put them in a non-ALL set; every non-ALL set includes them, because the touched crate may be what they read. scripts/check_affected_readers.sh
+# goes RED on a reader this list (or CORE) lacks.
+READERS="${AFFECTED_READERS:-apr-cli apr-format aprender-build-sha aprender-cbtop aprender-cgp aprender-ci-tools aprender-compute aprender-compute-xtask aprender-contracts aprender-contracts-cli aprender-contracts-macros aprender-contrastive-data aprender-core aprender-data aprender-db aprender-decide aprender-distribute aprender-explain aprender-gpu aprender-graph aprender-mcp aprender-orchestrate aprender-present-cli aprender-present-core aprender-present-layout aprender-present-lib aprender-present-terminal aprender-present-widgets aprender-present-yaml aprender-profile aprender-ptx-debug aprender-qa-certify aprender-qa-cli aprender-qa-gen aprender-qa-report aprender-qa-runner aprender-quant aprender-rag aprender-rag-cli aprender-registry aprender-review-experiment aprender-rollup aprender-serve aprender-shell aprender-simulate aprender-test-cli aprender-test-lib aprender-train aprender-train-bench aprender-train-distill aprender-train-inspect aprender-train-lora aprender-train-shell aprender-tsp aprender-update aprender-verify-ml aprender-viz aprender-viz-ttop aprender-zram aprender-zram-cli aprender-zram-generator facades}"
+filter=0 from="" bad=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --filter) filter=1; shift ;;
+    --diff-from) [ $# -ge 2 ] || { bad=1; break; }; from="$2"; shift 2 ;;
+    *) break ;;
+  esac
+done
+emit() { # $@ = package names, or ALL
+  if [ "${1:-}" = ALL ]; then [ "$filter" = 1 ] && echo 'all()' || echo ALL; return; fi
+  if [ "$filter" = 1 ]; then printf '%s\n' "$@" | sed 's/.*/package(&)/' | paste -sd'|' | sed 's/|/ | /g'; else printf '%s\n' "$@"; fi
+}
+[ "$bad" = 0 ] || { emit ALL; exit 0; }   # --diff-from with no FILE
+case "$from" in *..*) echo "affected_set: refusing a --diff-from with \"..\": $from" >&2; emit ALL; exit 0 ;; esac   # bashrs SEC010: refuse the path (never read it), fail closed with ALL
+if [ -n "$from" ]; then files="$(cat "$from")" || files=""
+else files="$(git diff --name-only --no-renames "${1:-origin/main}...${2:-HEAD}")" || files=""; fi
+[ -n "$files" ] || { emit ALL; exit 0; }
+if [ -n "${AFFECTED_META:-}" ]; then meta="$(cat "$AFFECTED_META")" || meta=""
+else meta="$(cargo metadata --no-deps --format-version 1 --offline 2>/dev/null || cargo metadata --no-deps --format-version 1)" || meta=""; fi
+root="${AFFECTED_ROOT:-$(git rev-parse --show-toplevel)}" || root=""
+[ -n "$meta" ] && [ -n "$root" ] || { emit ALL; exit 0; }   # no metadata, no checkout root: nothing measured
+dirs="$(jq -r --arg root "$root/" '.packages[] | [.name, (.manifest_path | sub($root;"") | sub("/?Cargo.toml$";""))] | @tsv' <<<"$meta")" || { emit ALL; exit 0; }
+[ -n "$dirs" ] || { emit ALL; exit 0; }
+# Root contracts/ files that build.rs or include_str!/include! turn into compiled code or constants (aprender-serve build.rs generates
+# arch/tensor-name code from three of them; apr-cli and mcp embed others). A change reaches dependents of the compiling crate, so these are ALL.
+# check_affected_readers.sh derives this list from the tree and is RED on a file it omits.
+BUILD_CONTRACTS="${AFFECTED_BUILD_CONTRACTS:-apr-mcp-tool-schemas-v1.yaml apr-model-capability-v1.yaml arch-constraints-v1.yaml architecture-requirements-v1.yaml kernel-fusion-v1.yaml layer-parity-v1.yaml softmax-kernel-v1.yaml tensor-names-v1.yaml thinking-budgets-v1.yaml}"
+touched="" skipped=0
+while IFS= read -r f; do
+  case "$f" in */*) ;; *.md) skipped=1; continue ;; esac   # root *.md: its run-time readers are in READERS
+  case "$f" in contracts/*) case " $BUILD_CONTRACTS " in *" ${f#contracts/} "*) emit ALL; exit 0 ;; esac ;; esac
+  case "$f" in
+    '') continue ;;
+    contracts/binding.yaml | contracts/*/binding.yaml | contracts/model-families/* | contracts/*/model-families/*) emit ALL; exit 0 ;;   # build inputs of core/compute/contracts/simulate/train: their generated code reaches dependents
+    contracts/* | docs/*) skipped=1; continue ;;   # FLOW-010(b): run-time readers are in READERS
+    .claude/* | ci/shard-step-costs.tsv) skipped=1; continue ;;
+    crates/*/build.rs) emit ALL; exit 0 ;;
+    crates/*.rs) ;;
+    crates/*) emit ALL; exit 0 ;;
+  esac
+  # longest manifest dir that prefixes the path
+  pkg="$(awk -F'\t' -v f="$f" 'index(f, $2 "/") == 1 && length($2) > best { best = length($2); n = $1 } END { print n }' <<<"$dirs")"
+  [ -n "$pkg" ] || { emit ALL; exit 0; }
+  case " $CORE " in *" $pkg "*) emit ALL; exit 0 ;; esac
+  touched="$touched"$'\n'"$pkg"
+done <<<"$files"
+touched="$(printf '%s\n' "$touched" | grep -v '^$' | sort -u || true)"
+[ -n "$touched" ] || [ "$skipped" = 1 ] || { emit ALL; exit 0; }   # nothing placed in a package and nothing on the no-reader list: nothing may be skipped
+# The closure, then aprender-core: its falsification_spec_v10 test scans every crate's src, so ANY non-ALL set includes it (and the READERS).
+res="$(jq -r --argjson t "$(printf '%s\n' "$touched" | jq -R . | jq -s .)" --arg rd "$READERS" '
+  (.packages | map({name, deps: [.dependencies[].name]})) as $p
+  | def grow($s): ($s + [$p|.[] | select(any(.deps[]; . as $d | $s | index($d))) | .name] | unique);
+    (.packages | map(.name)) as $m
+    | (($t | until(grow(.) == .; grow(.))) + ($rd | split(" ")) | unique) | .[] | select(. as $n | $m | index($n))' <<<"$meta")" || { emit ALL; exit 0; }
+[ -n "$res" ] || { emit ALL; exit 0; }
+mapfile -t out <<<"$res"
+emit "${out[@]}"

scripts/check_affected_readers.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# FLOW-010 E5: a crate whose Rust can reach OTHER crates' files at run time is a reader edge that cargo metadata
+# cannot see. scripts/affected_set.sh must list it in READERS (the only list that puts a crate in a non-ALL set),
+# or a change to the crate it reads would skip its tests. RED on a crate with any path-escape form and no entry.
+# Over-approximates on purpose: a false positive only adds a crate to READERS (more tests run); a miss skips tests.
+# Flagged in any .rs: a string holding `..`, parent(), ancestors(), current_dir, a "crates/" string or "crates",
+# CARGO_WORKSPACE_DIR, workspace_path_or_skip, read_dir(...) naming MANIFEST/crates/root/workspace, format!(...)
+# naming MANIFEST_DIR or crates/. Comment-only lines and files under a target/ path component are skipped.
+# Usage: check_affected_readers.sh [crates-dir] | --self-test | --list
+set -euo pipefail
+SET="${AFFECTED_SET_SH:-scripts/affected_set.sh}"
+# the exact file:line exemptions (a crate-wide exemption hides a later reader in the same crate)
+ALLOW="${AFFECTED_READERS_ALLOW:-crates/aprender-bench-tokenizer/src/lib.rs:50}"   # joins ../.. only for <root>/tokenizer.json and <root>/models/
+PAT='"\.\."|"[^"]*(\.\./|/\.\.)[^"]*"|\bparent\(\)|ancestors\(\)|current_dir|"[^"]*crates/|"crates"|CARGO_WORKSPACE_DIR|workspace_path_or_skip|read_dir\([^)]*(MANIFEST|crates|root|workspace)|format!\([^)]*(MANIFEST_DIR|crates/)'
+listed() { sed -n 's/^READERS="${AFFECTED_READERS:-\(.*\)}"$/\1/p' "$SET" | tr ' ' '\n' | grep -v '^$' | sort -u; }
+hits() { # $1 = crates dir; prints crate names
+  { grep -rEn --include='*.rs' "$PAT" "$1" 2>/dev/null || true; } \
+    | grep -vE '^[^:]*/target/' | grep -vE '^[^:]+:[0-9]+:[[:space:]]*//' \
+    | awk -F: -v allow=" $ALLOW " '{ k = $1 ":" $2; if (index(allow, " " k " ")) next; n = split($1, a, "/"); print a[2] }' | sort -u
+}
+build_listed() { sed -n 's/^BUILD_CONTRACTS="\${AFFECTED_BUILD_CONTRACTS:-\(.*\)}"$/\1/p' "$SET" | tr ' ' '\n' | grep -v '^$' | sort -u; }
+# Root contracts/<file> that a build.rs names (any quoted *-vN.yaml) or an include*! macro embeds: compiled into a crate, so ALL.
+build_inputs() { # $1 = crates dir; root contracts dir = ${CONTRACTS_DIR:-contracts}
+  { grep -rhE --include=build.rs '[A-Za-z0-9_-]+\.yaml"' "$1" 2>/dev/null | grep -vE '^\s*//' | grep -oE '[A-Za-z0-9_-]+\.yaml"' | tr -d '"'
+    grep -rhE --include='*.rs' 'include(_str|_bytes)?!' "$1" 2>/dev/null | grep -vE '^\s*//' | grep -oE 'contracts/[A-Za-z0-9_-]+\.yaml' | sed 's#contracts/##'
+  } | sort -u | while read -r f; do [ -f "${CONTRACTS_DIR:-contracts}/$f" ] && echo "$f"; done | grep -v 'binding\.yaml$' || true
+}
+check() {
+  local bad="" l c; l="$(listed)"
+  for c in $(hits "$1"); do grep -qx "$c" <<<"$l" || bad="$bad $c"; done
+  local bl; bl="$(build_listed)"
+  for c in $(build_inputs "$1"); do grep -qx "$c" <<<"$bl" || bad="$bad contracts/$c(BUILD_CONTRACTS)"; done
+  if [ -n "$bad" ]; then echo "RED: crates can reach other crates' files but are not in READERS (scripts/affected_set.sh):$bad"; return 1; fi
+  echo "ok: every crate with a path-escape form is in READERS"
+}
+if [ "${1:-}" = "--list" ]; then hits crates; exit 0; fi
+if [ "${1:-}" = "--self-test" ]; then
+  t=$(mktemp -d); trap 'rm -rf -- "${t:?}"' EXIT
+  SET="$(pwd)/$SET"; cd "$t"
+  fail=0
+  fresh() { rm -rf -- "${t:?}/crates"; mkdir -p "$t/crates/$1/src"; }   # one unlisted crate holding $2
+  expect_red() { # name crate body
+    fresh "$2"; printf '%s\n' "fn f() { $3 }" >"$t/crates/$2/src/x.rs"
+    if (check crates >/dev/null 2>&1); then echo "FAIL self-test: not caught: $1"; fail=1; fi
+  }
+  expect_red 'join("crates")' zz-a 'let _ = root.join("crates");'
+  expect_red 'MANIFEST_DIR.parent().parent()' zz-b 'let _ = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent();'
+  expect_red 'root.join("crates/aprender-core")' zz-c 'let _ = root.join("crates/aprender-core");'
+  expect_red 'format! with ../../crates' zz-d 'let _ = format!("{}/../../crates/x", env!("CARGO_MANIFEST_DIR"));'
+  expect_red 'ancestors().nth(2)' zz-e 'let _ = here.ancestors().nth(2);'
+  expect_red 'read_dir("../")' zz-f 'let _ = std::fs::read_dir("../");'
+  expect_red 'current_dir().join("..")' zz-g 'let _ = std::env::current_dir().unwrap().join("..");'
+  expect_red 'relative read_to_string' zz-h 'let _ = std::fs::read_to_string("../../crates/x/src/lib.rs");'
+  expect_red 'MANIFEST_DIR.parent().join(apr-cli)' zz-i 'let _ = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().join("apr-cli");'
+  expect_red 'workspace_path_or_skip!' zz-j 'let _ = workspace_path_or_skip!("crates/x/..");'
+  expect_red 'read_dir(workspace_root)' zz-k 'let _ = std::fs::read_dir(workspace_root);'
+  expect_red 'CARGO_WORKSPACE_DIR' zz-l 'let _ = env!("CARGO_WORKSPACE_DIR");'
+  expect_red 'format! with MANIFEST_DIR' zz-m 'let _ = format!("{}/x", env!("CARGO_MANIFEST_DIR"));'
+  expect_red 'a *contracts*/*target* crate name' zz-contracts-target-new 'let _ = root.join("crates");'
+  expect_red 'a line that mentions target/contracts' zz-n 'let _ = root.join("crates"); /* target contracts docs/ */'
+  # the bench-tokenizer exemption is the one line, not the crate
+  fresh aprender-bench-tokenizer; printf 'fn f() {}\nfn g() { let _ = root.join("crates"); }\n' >"$t/crates/aprender-bench-tokenizer/src/lib.rs"
+  if (AFFECTED_READERS_ALLOW="crates/aprender-bench-tokenizer/src/lib.rs:1" check crates >/dev/null 2>&1); then echo "FAIL self-test: an allow entry exempted a different line"; fail=1; fi
+  # a listed reader, a comment-only mention and a target/ file are green
+  fresh aprender-serve; echo 'fn f() { let _ = root.join("crates"); }' >"$t/crates/aprender-serve/src/y.rs"
+  (check crates >/dev/null 2>&1) || { echo "FAIL self-test: a listed reader was flagged"; fail=1; }
+  fresh zz-o; echo '// root.join("crates")' >"$t/crates/zz-o/src/y.rs"
+  (check crates >/dev/null 2>&1) || { echo "FAIL self-test: a comment-only line was flagged"; fail=1; }
+  [ "$fail" -eq 0 ] && echo "ok: self-test"
+  exit "$fail"
+fi
+check "${1:-crates}"