diff --git a/README.md b/README.md index 89199e4..4e1af59 100644 --- a/README.md +++ b/README.md @@ -722,6 +722,22 @@ weight of its own; a missing file is reported, never created: with its file and each nested file with its own budget. Apply refuses a saved nested edit if that file is no longer named in the current config. The two-session evidence floor, remembered rejections, and `apply` as the only writer are unchanged. +- **Reach.** Every in-repo path counts, reads included, so in a repo where most sessions + cross directories a nested file may see too few sessions to ever corroborate a gap. The + run summary and `backpass status` show, per nested file, how many sampled sessions + worked only under its directory, and warn below `minGapEvidence`, when no add, rewrite, + or remove can reach the file: + + ```text + nested apps/api/AGENTS.md · 1 of 78 sampled session(s) worked only under apps/api/ · budget ... + below minGapEvidence (2); no add, rewrite, or remove edit can reach this file + ``` + + `status` reads the count recorded at the last attribution (`analyze`, `propose`, or a + full run), prints `not yet measured` before one, and reports it under + `nestedAttribution` in `--json`; the proposal's `nested` entries carry + `sampledSessions`, `minGapEvidence`, and `belowMinGapEvidence`. The count is + report-only and never gates a run. Each nested directory keeps the pointer model above: name its canonical file, and a sibling `AGENTS.md` or `CLAUDE.md` that only imports it needs nothing, while a sibling @@ -738,7 +754,7 @@ exactly the files a run trains, so neither trains a nested file. | `backpass analyze` | calculate loss: the tier-1 pass over pending transcripts | | `backpass propose` | aggregate gradients + gradient descent: the tier-2 pass from cached evidence | | `backpass apply` | review and write the accepted edits | -| `backpass status` | cache state, failed transcripts, budget bars, and cross-surface overlaps | +| `backpass status` | cache state, failed transcripts, budget bars, nested reach, and cross-surface overlaps | | `backpass init` | initialize the selected scope's config and state | Run `backpass --help` for the full flag list. diff --git a/src/commands/analyze.js b/src/commands/analyze.js index d4ffea5..ca3cec1 100644 --- a/src/commands/analyze.js +++ b/src/commands/analyze.js @@ -8,8 +8,10 @@ import { renderAlsoLoaded, reportNestedMemoryFiles, resolveNestedMemoryFiles, + nestedReach, nestedSurfaceHash, workedUnder, + writeNestedReach, } from "../nested.js"; import { transcriptIdentity } from "../transcript.js"; import { loadProjectSkills, resolveOverflowTarget, skillDescriptionTokens } from "../skills.js"; @@ -127,6 +129,7 @@ async function runAnalysisCore(ctx) { /** * The sessions each nested memory file learns from: those that worked under its * directory (`src/nested.js`). Attribution is computed only when a nested file is named. + * Each file's share of the sample is recorded for `status` (`nestedReach`), report-only. */ export async function nestedCorpora(ctx, weights, transcripts, attribution = null) { if (!weights.length) return { corpora: [], attribution: null }; @@ -137,6 +140,9 @@ export async function nestedCorpora(ctx, weights, transcripts, attribution = nul workedUnder(placed.get(transcriptIdentity(transcript)), weight.dir), ), })); + if (ctx.config?.state) { + writeNestedReach(ctx.config.state, nestedReach(corpora, transcripts.length, ctx.config.minGapEvidence)); + } return { corpora, attribution: placed }; } diff --git a/src/commands/propose.js b/src/commands/propose.js index 67639d2..1e1ab5f 100644 --- a/src/commands/propose.js +++ b/src/commands/propose.js @@ -14,7 +14,7 @@ import { UserError, color, info, json, out, terminalSafe } from "../logger.js"; import { budgetBar, formatTokens } from "../tokens.js"; import { emitProgress } from "../progress.js"; import { nestedCorpora, primaryMemoryFile } from "./analyze.js"; -import { mergeNestedProposals, nestedContext, routingFor } from "../nested.js"; +import { mergeNestedProposals, nestedContext, nestedReachWarning, routingFor } from "../nested.js"; import { printUsage } from "./usage.js"; import { closeRemoteDiscovery, discoverForRun } from "./scan.js"; import { capTranscripts } from "../sample.js"; @@ -196,7 +196,12 @@ async function runProposalCore(ctx, precomputed) { }); accountForConsolidationUsage(rootProposal, summary); - const proposal = passes.length ? mergeNestedProposals(rootProposal, passes) : rootProposal; + const proposal = passes.length + ? mergeNestedProposals(rootProposal, passes, { + sampledSessions: transcripts.length, + minGapEvidence: config.minGapEvidence, + }) + : rootProposal; config.state.writeProposal(proposal); return { proposal, summary, memoryFile: file }; } @@ -263,13 +268,20 @@ export function printProposal(proposal, { applied = false, analysisUsage = [] } `${proposal.stats.gapClusters} gap clusters`, ); for (const nested of proposal.nested || []) { - const head = ` nested ${nested.memoryFile.path} · ${nested.sessions} session(s) under ${nested.dir}/`; + const sessions = + nested.sampledSessions != null + ? `${nested.sessions} of ${nested.sampledSessions} sampled session(s) worked only under ${nested.dir}/` + : `${nested.sessions} session(s) under ${nested.dir}/`; + const head = ` nested ${nested.memoryFile.path} · ${sessions}`; out( nested.skipped ? `${head} · ${color.dim(nested.skipped)}` : `${head} · budget ${budgetBar(nested.budget)} ${formatTokens(nested.budget.current)} -> ` + `${formatTokens(nested.budget.projected)} / ${formatTokens(nested.budget.capTokens)} tok`, ); + // Report-only: the count explains an empty result, it never gates one. + const warning = nestedReachWarning(nested); + if (warning) out(` ${color.yellow(warning)}`); } out(""); diff --git a/src/commands/status.js b/src/commands/status.js index 91cfb0b..851b0c4 100644 --- a/src/commands/status.js +++ b/src/commands/status.js @@ -4,7 +4,7 @@ import path from "node:path"; import { userClaudeSkillsDir } from "../config.js"; import { color, json, out } from "../logger.js"; import { resolveMemoryFiles } from "../memory.js"; -import { nestedBudget, resolveNestedMemoryFiles } from "../nested.js"; +import { nestedBudget, nestedReachWarning, readNestedReach, resolveNestedMemoryFiles } from "../nested.js"; import { loadProjectSkills, resolveOverflowTarget, @@ -60,7 +60,8 @@ export async function cmdStatus(ctx) { }; }); // Nested memory files are weights of their own, each under its own budget. - for (const weight of resolveNestedMemoryFiles(repo.root, config)) { + const nestedWeights = resolveNestedMemoryFiles(repo.root, config); + for (const weight of nestedWeights) { budgets.push({ path: weight.path, label: weight.path, @@ -73,10 +74,34 @@ export async function cmdStatus(ctx) { }); } + // How many sampled sessions each nested file could learn from at the last attribution, + // read back from state rather than re-discovered: `status` stays instant. Report-only. + const reach = nestedWeights.length ? readNestedReach(state) : null; + const nestedAttribution = nestedWeights.length + ? { + measuredAt: reach?.measuredAt ?? null, + files: nestedWeights + .filter((weight) => weight.file) + .map((weight) => { + const entry = reach?.files.find((candidate) => candidate.path === weight.path); + return { + path: weight.path, + dir: weight.dir, + measured: Boolean(entry), + sessions: entry?.sessions ?? null, + sampledSessions: entry?.sampledSessions ?? null, + minGapEvidence: entry?.minGapEvidence ?? null, + belowMinGapEvidence: entry?.belowMinGapEvidence ?? null, + }; + }), + } + : null; + if (ctx.flags.json) { json({ repo: repo.name, budgets, + ...(nestedAttribution ? { nestedAttribution } : {}), crossSurfaceDuplicates: duplicates, evidence: counts, scanCacheEntries: Object.keys(cache.entries).length, @@ -122,6 +147,24 @@ export async function cmdStatus(ctx) { ), ); } + if (nestedAttribution?.files.length) { + out(""); + const when = nestedAttribution.measuredAt + ? `sampled sessions at the last attribution, ${nestedAttribution.measuredAt.slice(0, 16).replace("T", " ")}; ` + : ""; + out(color.dim(`NESTED EVIDENCE REACH (${when}report-only)`)); + for (const entry of nestedAttribution.files) { + if (!entry.measured) { + out(` ${entry.path}: ${color.dim("not yet measured - run `backpass` or `backpass analyze`")}`); + continue; + } + out( + ` ${entry.path}: ${entry.sessions} of ${entry.sampledSessions} sampled session(s) worked only under ${entry.dir}/`, + ); + const warning = nestedReachWarning(entry); + if (warning) out(` ${color.yellow(warning)}`); + } + } if (duplicates.length) { out(""); out(color.dim("CROSS-SURFACE (report-only)")); diff --git a/src/nested.js b/src/nested.js index 1b188c4..4f88753 100644 --- a/src/nested.js +++ b/src/nested.js @@ -383,9 +383,10 @@ export function owningFile(sessionIds, weights, attribution) { * each nested file's edits renumbered after it and labeled with their file. A nested * file's image joins `targetFiles`, so apply refuses it if it changed since, the same * freshness contract every non-root target has; `nested` carries each file's own budget, - * which apply re-checks against the accepted subset. + * which apply re-checks against the accepted subset. With `sampledSessions` and + * `minGapEvidence`, each entry also carries its report-only reach (`nestedReach`). */ -export function mergeNestedProposals(root, passes) { +export function mergeNestedProposals(root, passes, { sampledSessions = null, minGapEvidence = null } = {}) { const edits = [...root.edits]; const targetFiles = [...(root.targetFiles || [])]; const usage = [...(root.usage || [])]; @@ -409,6 +410,13 @@ export function mergeNestedProposals(root, passes) { memoryFile: { path: weight.path, hash: weight.file.hash, tokens: weight.file.tokens }, dir: weight.dir, sessions: pass.transcripts.length, + ...(sampledSessions !== null && minGapEvidence !== null + ? { + sampledSessions, + minGapEvidence, + belowMinGapEvidence: pass.transcripts.length < minGapEvidence, + } + : {}), budget: proposal?.budget ?? budgetStatus(weight.file.text, null, pass.cap), maxEdits: proposal?.config?.maxEditsPerRun ?? null, edits: ids, @@ -434,3 +442,44 @@ export function routingFor(weights, attribution, rootPath, weight) { ownerOf: (sessionIds) => owningFile(sessionIds, weights, attribution), }; } + +/** + * How many sampled sessions each nested file can learn from: those whose every in-repo + * work path stays under its directory (`workedUnder`). Below `minGapEvidence` no gap can + * be corroborated for that file, since corroboration is bounded to the sample, so no add, + * rewrite, or remove can reach it - and a run that proposes nothing there cannot be told + * apart from a file with no gaps. Measured from deterministic attribution, never from a + * model; persisted in `.backpass/nested/reach.json` at each attribution so `status` can + * show it without discovering. Report-only: nothing reads it back to gate. + */ +export function nestedReach(corpora, sampledSessions, minGapEvidence) { + return corpora.map(({ weight, transcripts }) => ({ + path: weight.path, + dir: weight.dir, + sessions: transcripts.length, + sampledSessions, + minGapEvidence, + belowMinGapEvidence: transcripts.length < minGapEvidence, + })); +} + +function reachPath(state) { + return path.join(state.root, "nested", "reach.json"); +} + +export function writeNestedReach(state, reach) { + state.writeJsonFile(reachPath(state), { measuredAt: new Date().toISOString(), files: reach }); +} + +/** The last measured reach, or null when no run has attributed sessions yet. */ +export function readNestedReach(state) { + const saved = state.readJsonFile(reachPath(state), null); + return saved && Array.isArray(saved.files) ? saved : null; +} + +/** The floor warning for one file's reach, or null at or above `minGapEvidence`. */ +export function nestedReachWarning(entry) { + return entry.belowMinGapEvidence + ? `below minGapEvidence (${entry.minGapEvidence}); no add, rewrite, or remove edit can reach this file` + : null; +} diff --git a/test/nested-cli.test.js b/test/nested-cli.test.js index 8ca2dfb..8729688 100644 --- a/test/nested-cli.test.js +++ b/test/nested-cli.test.js @@ -310,7 +310,16 @@ test("a named nested file learns only from its subtree, and each lesson lands in assert.deepEqual(proposal.nested[0].edits, ["e2"]); assert.ok(proposal.targetFiles.some((target) => target.file === "apps/api/AGENTS.md")); assert.match(result.output, /e2 ADD\s+apps\/api\/AGENTS\.md: /); - assert.match(result.output, /nested apps\/api\/AGENTS\.md · 2 session\(s\) under apps\/api\/ · budget /); + // At the floor: the count is shown, with no warning. + assert.match( + result.output, + /nested apps\/api\/AGENTS\.md · 2 of 4 sampled session\(s\) worked only under apps\/api\/ · budget /, + ); + assert.doesNotMatch(result.output, /below minGapEvidence/); + assert.deepEqual( + [proposal.nested[0].sampledSessions, proposal.nested[0].minGapEvidence, proposal.nested[0].belowMinGapEvidence], + [4, 2, false], + ); // The nested file's state is its own; the root's evidence dir holds only root evidence. const nestedState = path.join(dir, ".backpass", "nested"); @@ -329,6 +338,68 @@ test("a named nested file learns only from its subtree, and each lesson lands in assert.match(applied.output, /wrote apps\/api\/AGENTS\.md \(e2\)/); }); +function status(dir, args = []) { + const result = spawnSync(process.execPath, [CLI, "status", ...args], { + cwd: dir, + encoding: "utf8", + env: { ...process.env, NO_COLOR: "1", CI: "1" }, + }); + return { ...result, output: `${result.stdout}${result.stderr}` }; +} + +test("a nested file whose sampled sessions fall below minGapEvidence is named in the run summary and status", () => { + const dir = makeMonorepo({ config: { nestedMemoryFiles: ["apps/api/AGENTS.md"] } }); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-nested-sparse-home-")); + writeSession(home, "api-one", dir, { quote: API_QUOTE, file: "apps/api/src/routes/orders.ts" }); + writeSession(home, "web-one", dir, { quote: WEB_QUOTE, file: "apps/web/src/checkout.tsx" }); + writeSession(home, "web-two", dir, { quote: WEB_QUOTE, file: "apps/web/src/checkout.tsx" }); + + const before = status(dir); + assert.equal(before.status, 0, before.output); + assert.match(before.output, /apps\/api\/AGENTS\.md: not yet measured - run `backpass` or `backpass analyze`/); + + const result = run(dir, home, []); + assert.equal(result.status, 0, result.output); + const warning = "below minGapEvidence (2); no add, rewrite, or remove edit can reach this file"; + assert.match( + result.stdout, + /nested apps\/api\/AGENTS\.md · 1 of 3 sampled session\(s\) worked only under apps\/api\//, + ); + assert.ok(result.stdout.includes(`\n ${warning}\n`), result.stdout); + const proposal = readJson(path.join(dir, ".backpass", "proposal.json")); + assert.deepEqual( + [proposal.nested[0].sessions, proposal.nested[0].sampledSessions, proposal.nested[0].belowMinGapEvidence], + [1, 3, true], + ); + // Report-only: the root edit the web sessions earned is still proposed. + assert.deepEqual( + proposal.edits.map((edit) => edit.file), + ["AGENTS.md"], + ); + + const text = status(dir); + assert.equal(text.status, 0, text.output); + assert.match(text.stdout, /NESTED EVIDENCE REACH \(sampled sessions at the last attribution, [^;]+; report-only\)/); + assert.match(text.stdout, /apps\/api\/AGENTS\.md: 1 of 3 sampled session\(s\) worked only under apps\/api\//); + assert.ok(text.stdout.includes(`\n ${warning}\n`), text.stdout); + + const structured = status(dir, ["--json"]); + assert.equal(structured.status, 0, structured.output); + const { nestedAttribution } = JSON.parse(structured.stdout); + assert.ok(nestedAttribution.measuredAt); + assert.deepEqual(nestedAttribution.files, [ + { + path: "apps/api/AGENTS.md", + dir: "apps/api", + measured: true, + sessions: 1, + sampledSessions: 3, + minGapEvidence: 2, + belowMinGapEvidence: true, + }, + ]); +}); + test("a gap spanning API and web is proposed only in root, not again from its API subset", () => { const dir = makeMonorepo({ config: { nestedMemoryFiles: ["apps/api/AGENTS.md"] } }); const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-nested-shared-home-")); diff --git a/test/status.test.js b/test/status.test.js index 0a8aebe..0821f61 100644 --- a/test/status.test.js +++ b/test/status.test.js @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { cmdStatus } from "../src/commands/status.js"; import { loadConfig } from "../src/config.js"; +import { nestedReach, writeNestedReach } from "../src/nested.js"; import { State } from "../src/state.js"; import { estimateTokens } from "../src/tokens.js"; import { makeRepo } from "./helpers/staging.js"; @@ -11,11 +12,12 @@ const MEMORY = "# Rules\n\n- Keep changes focused.\n"; const DESCRIPTION = "Load before changing database queries."; const SKILL = `---\nname: database\ndescription: ${DESCRIPTION}\n---\n\n# Database\n\nRead the schema first.\n`; -async function captureStatus(repo, json) { +async function captureStatus(repo, json, overrides = {}) { const config = loadConfig(repo.root, { memoryFiles: ["AGENTS.md"], skillsDir: ".agents/skills", budgetTokens: estimateTokens(MEMORY), + ...overrides, }); config.state = new State(repo.root).ensure(); config.agents = { @@ -119,3 +121,59 @@ test("status neither counts nor reports thematic breaks as instructions (#192)", assert.match(text, /· 3 instructions/); assert.doesNotMatch(text, /restates demo/); }); + +test("status shows each nested file's sampled-session reach and warns only below minGapEvidence", async () => { + const repo = makeRepo({ + "AGENTS.md": MEMORY, + "apps/api/AGENTS.md": "# API\n\n- Handlers live in src/routes.\n", + "apps/web/AGENTS.md": "# Web\n\n- Components live in src/ui.\n", + "apps/docs/AGENTS.md": "# Docs\n\n- Pages live in src/pages.\n", + }); + const nested = { nestedMemoryFiles: ["apps/api/AGENTS.md", "apps/web/AGENTS.md", "apps/docs/AGENTS.md"] }; + const state = new State(repo.root).ensure(); + const corpus = (n) => Array.from({ length: n }, () => ({})); + writeNestedReach( + state, + nestedReach( + [ + { weight: { path: "apps/api/AGENTS.md", dir: "apps/api" }, transcripts: corpus(1) }, + { weight: { path: "apps/web/AGENTS.md", dir: "apps/web" }, transcripts: corpus(2) }, + ], + 78, + 2, + ), + ); + + const text = (await captureStatus(repo, false, nested)).join("\n"); + assert.match(text, /NESTED EVIDENCE REACH \(sampled sessions at the last attribution, [^;]+; report-only\)/); + assert.match( + text, + /apps\/api\/AGENTS\.md: 1 of 78 sampled session\(s\) worked only under apps\/api\/\n {4}below minGapEvidence \(2\); no add, rewrite, or remove edit can reach this file/, + ); + assert.match( + text, + /apps\/web\/AGENTS\.md: 2 of 78 sampled session\(s\) worked only under apps\/web\/\n(?! {4}below)/, + ); + assert.equal((text.match(/below minGapEvidence/g) || []).length, 1); + assert.match(text, /apps\/docs\/AGENTS\.md: not yet measured - run `backpass` or `backpass analyze`/); + + const structured = JSON.parse((await captureStatus(repo, true, nested)).join("\n")); + assert.deepEqual( + structured.nestedAttribution.files.map((entry) => [ + entry.path, + entry.measured, + entry.sessions, + entry.sampledSessions, + entry.minGapEvidence, + entry.belowMinGapEvidence, + ]), + [ + ["apps/api/AGENTS.md", true, 1, 78, 2, true], + ["apps/web/AGENTS.md", true, 2, 78, 2, false], + ["apps/docs/AGENTS.md", false, null, null, null, null], + ], + ); + + const plain = JSON.parse((await captureStatus(repo, true)).join("\n")); + assert.equal(plain.nestedAttribution, undefined); +});