Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,22 @@ weight of its own; a missing file is reported, never created:
with its file and each nested file with its own budget. Apply refuses a saved nested
edit if that file is no longer named in the current config. The two-session evidence
floor, remembered rejections, and `apply` as the only writer are unchanged.
- **Reach.** Every in-repo path counts, reads included, so in a repo where most sessions
cross directories a nested file may see too few sessions to ever corroborate a gap. The
run summary and `backpass status` show, per nested file, how many sampled sessions
worked only under its directory, and warn below `minGapEvidence`, when no add, rewrite,
or remove can reach the file:

```text
nested apps/api/AGENTS.md · 1 of 78 sampled session(s) worked only under apps/api/ · budget ...
below minGapEvidence (2); no add, rewrite, or remove edit can reach this file
```

`status` reads the count recorded at the last attribution (`analyze`, `propose`, or a
full run), prints `not yet measured` before one, and reports it under
`nestedAttribution` in `--json`; the proposal's `nested` entries carry
`sampledSessions`, `minGapEvidence`, and `belowMinGapEvidence`. The count is
report-only and never gates a run.

Each nested directory keeps the pointer model above: name its canonical file, and a
sibling `AGENTS.md` or `CLAUDE.md` that only imports it needs nothing, while a sibling
Expand All @@ -738,7 +754,7 @@ exactly the files a run trains, so neither trains a nested file.
| `backpass analyze` | calculate loss: the tier-1 pass over pending transcripts |
| `backpass propose` | aggregate gradients + gradient descent: the tier-2 pass from cached evidence |
| `backpass apply` | review and write the accepted edits |
| `backpass status` | cache state, failed transcripts, budget bars, and cross-surface overlaps |
| `backpass status` | cache state, failed transcripts, budget bars, nested reach, and cross-surface overlaps |
| `backpass init` | initialize the selected scope's config and state |

Run `backpass --help` for the full flag list.
Expand Down
6 changes: 6 additions & 0 deletions src/commands/analyze.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import {
renderAlsoLoaded,
reportNestedMemoryFiles,
resolveNestedMemoryFiles,
nestedReach,
nestedSurfaceHash,
workedUnder,
writeNestedReach,
} from "../nested.js";
import { transcriptIdentity } from "../transcript.js";
import { loadProjectSkills, resolveOverflowTarget, skillDescriptionTokens } from "../skills.js";
Expand Down Expand Up @@ -127,6 +129,7 @@ async function runAnalysisCore(ctx) {
/**
* The sessions each nested memory file learns from: those that worked under its
* directory (`src/nested.js`). Attribution is computed only when a nested file is named.
* Each file's share of the sample is recorded for `status` (`nestedReach`), report-only.
*/
export async function nestedCorpora(ctx, weights, transcripts, attribution = null) {
if (!weights.length) return { corpora: [], attribution: null };
Expand All @@ -137,6 +140,9 @@ export async function nestedCorpora(ctx, weights, transcripts, attribution = nul
workedUnder(placed.get(transcriptIdentity(transcript)), weight.dir),
),
}));
if (ctx.config?.state) {
writeNestedReach(ctx.config.state, nestedReach(corpora, transcripts.length, ctx.config.minGapEvidence));
}
return { corpora, attribution: placed };
}

Expand Down
18 changes: 15 additions & 3 deletions src/commands/propose.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { UserError, color, info, json, out, terminalSafe } from "../logger.js";
import { budgetBar, formatTokens } from "../tokens.js";
import { emitProgress } from "../progress.js";
import { nestedCorpora, primaryMemoryFile } from "./analyze.js";
import { mergeNestedProposals, nestedContext, routingFor } from "../nested.js";
import { mergeNestedProposals, nestedContext, nestedReachWarning, routingFor } from "../nested.js";
import { printUsage } from "./usage.js";
import { closeRemoteDiscovery, discoverForRun } from "./scan.js";
import { capTranscripts } from "../sample.js";
Expand Down Expand Up @@ -196,7 +196,12 @@ async function runProposalCore(ctx, precomputed) {
});

accountForConsolidationUsage(rootProposal, summary);
const proposal = passes.length ? mergeNestedProposals(rootProposal, passes) : rootProposal;
const proposal = passes.length
? mergeNestedProposals(rootProposal, passes, {
sampledSessions: transcripts.length,
minGapEvidence: config.minGapEvidence,
})
: rootProposal;
config.state.writeProposal(proposal);
return { proposal, summary, memoryFile: file };
}
Expand Down Expand Up @@ -263,13 +268,20 @@ export function printProposal(proposal, { applied = false, analysisUsage = [] }
`${proposal.stats.gapClusters} gap clusters`,
);
for (const nested of proposal.nested || []) {
const head = ` nested ${nested.memoryFile.path} · ${nested.sessions} session(s) under ${nested.dir}/`;
const sessions =
nested.sampledSessions != null
? `${nested.sessions} of ${nested.sampledSessions} sampled session(s) worked only under ${nested.dir}/`
: `${nested.sessions} session(s) under ${nested.dir}/`;
const head = ` nested ${nested.memoryFile.path} · ${sessions}`;
out(
nested.skipped
? `${head} · ${color.dim(nested.skipped)}`
: `${head} · budget ${budgetBar(nested.budget)} ${formatTokens(nested.budget.current)} -> ` +
`${formatTokens(nested.budget.projected)} / ${formatTokens(nested.budget.capTokens)} tok`,
);
// Report-only: the count explains an empty result, it never gates one.
const warning = nestedReachWarning(nested);
if (warning) out(` ${color.yellow(warning)}`);
}
out("");

Expand Down
47 changes: 45 additions & 2 deletions src/commands/status.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import path from "node:path";
import { userClaudeSkillsDir } from "../config.js";
import { color, json, out } from "../logger.js";
import { resolveMemoryFiles } from "../memory.js";
import { nestedBudget, resolveNestedMemoryFiles } from "../nested.js";
import { nestedBudget, nestedReachWarning, readNestedReach, resolveNestedMemoryFiles } from "../nested.js";
import {
loadProjectSkills,
resolveOverflowTarget,
Expand Down Expand Up @@ -60,7 +60,8 @@ export async function cmdStatus(ctx) {
};
});
// Nested memory files are weights of their own, each under its own budget.
for (const weight of resolveNestedMemoryFiles(repo.root, config)) {
const nestedWeights = resolveNestedMemoryFiles(repo.root, config);
for (const weight of nestedWeights) {
budgets.push({
path: weight.path,
label: weight.path,
Expand All @@ -73,10 +74,34 @@ export async function cmdStatus(ctx) {
});
}

// How many sampled sessions each nested file could learn from at the last attribution,
// read back from state rather than re-discovered: `status` stays instant. Report-only.
const reach = nestedWeights.length ? readNestedReach(state) : null;
const nestedAttribution = nestedWeights.length
? {
measuredAt: reach?.measuredAt ?? null,
files: nestedWeights
.filter((weight) => weight.file)
.map((weight) => {
const entry = reach?.files.find((candidate) => candidate.path === weight.path);
return {
path: weight.path,
dir: weight.dir,
measured: Boolean(entry),
sessions: entry?.sessions ?? null,
sampledSessions: entry?.sampledSessions ?? null,
minGapEvidence: entry?.minGapEvidence ?? null,
belowMinGapEvidence: entry?.belowMinGapEvidence ?? null,
};
}),
}
: null;

if (ctx.flags.json) {
json({
repo: repo.name,
budgets,
...(nestedAttribution ? { nestedAttribution } : {}),
crossSurfaceDuplicates: duplicates,
evidence: counts,
scanCacheEntries: Object.keys(cache.entries).length,
Expand Down Expand Up @@ -122,6 +147,24 @@ export async function cmdStatus(ctx) {
),
);
}
if (nestedAttribution?.files.length) {
out("");
const when = nestedAttribution.measuredAt
? `sampled sessions at the last attribution, ${nestedAttribution.measuredAt.slice(0, 16).replace("T", " ")}; `
: "";
out(color.dim(`NESTED EVIDENCE REACH (${when}report-only)`));
for (const entry of nestedAttribution.files) {
if (!entry.measured) {
out(` ${entry.path}: ${color.dim("not yet measured - run `backpass` or `backpass analyze`")}`);
continue;
}
out(
` ${entry.path}: ${entry.sessions} of ${entry.sampledSessions} sampled session(s) worked only under ${entry.dir}/`,
);
const warning = nestedReachWarning(entry);
if (warning) out(` ${color.yellow(warning)}`);
}
}
if (duplicates.length) {
out("");
out(color.dim("CROSS-SURFACE (report-only)"));
Expand Down
53 changes: 51 additions & 2 deletions src/nested.js
Original file line number Diff line number Diff line change
Expand Up @@ -383,9 +383,10 @@ export function owningFile(sessionIds, weights, attribution) {
* each nested file's edits renumbered after it and labeled with their file. A nested
* file's image joins `targetFiles`, so apply refuses it if it changed since, the same
* freshness contract every non-root target has; `nested` carries each file's own budget,
* which apply re-checks against the accepted subset.
* which apply re-checks against the accepted subset. With `sampledSessions` and
* `minGapEvidence`, each entry also carries its report-only reach (`nestedReach`).
*/
export function mergeNestedProposals(root, passes) {
export function mergeNestedProposals(root, passes, { sampledSessions = null, minGapEvidence = null } = {}) {
const edits = [...root.edits];
const targetFiles = [...(root.targetFiles || [])];
const usage = [...(root.usage || [])];
Expand All @@ -409,6 +410,13 @@ export function mergeNestedProposals(root, passes) {
memoryFile: { path: weight.path, hash: weight.file.hash, tokens: weight.file.tokens },
dir: weight.dir,
sessions: pass.transcripts.length,
...(sampledSessions !== null && minGapEvidence !== null
? {
sampledSessions,
minGapEvidence,
belowMinGapEvidence: pass.transcripts.length < minGapEvidence,
}
: {}),
budget: proposal?.budget ?? budgetStatus(weight.file.text, null, pass.cap),
maxEdits: proposal?.config?.maxEditsPerRun ?? null,
edits: ids,
Expand All @@ -434,3 +442,44 @@ export function routingFor(weights, attribution, rootPath, weight) {
ownerOf: (sessionIds) => owningFile(sessionIds, weights, attribution),
};
}

/**
* How many sampled sessions each nested file can learn from: those whose every in-repo
* work path stays under its directory (`workedUnder`). Below `minGapEvidence` no gap can
* be corroborated for that file, since corroboration is bounded to the sample, so no add,
* rewrite, or remove can reach it - and a run that proposes nothing there cannot be told
* apart from a file with no gaps. Measured from deterministic attribution, never from a
* model; persisted in `.backpass/nested/reach.json` at each attribution so `status` can
* show it without discovering. Report-only: nothing reads it back to gate.
*/
export function nestedReach(corpora, sampledSessions, minGapEvidence) {
return corpora.map(({ weight, transcripts }) => ({
path: weight.path,
dir: weight.dir,
sessions: transcripts.length,
sampledSessions,
minGapEvidence,
belowMinGapEvidence: transcripts.length < minGapEvidence,
}));
}

function reachPath(state) {
return path.join(state.root, "nested", "reach.json");
}

export function writeNestedReach(state, reach) {
state.writeJsonFile(reachPath(state), { measuredAt: new Date().toISOString(), files: reach });
}

/** The last measured reach, or null when no run has attributed sessions yet. */
export function readNestedReach(state) {
const saved = state.readJsonFile(reachPath(state), null);
return saved && Array.isArray(saved.files) ? saved : null;
}

/** The floor warning for one file's reach, or null at or above `minGapEvidence`. */
export function nestedReachWarning(entry) {
return entry.belowMinGapEvidence
? `below minGapEvidence (${entry.minGapEvidence}); no add, rewrite, or remove edit can reach this file`
: null;
}
73 changes: 72 additions & 1 deletion test/nested-cli.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,16 @@ test("a named nested file learns only from its subtree, and each lesson lands in
assert.deepEqual(proposal.nested[0].edits, ["e2"]);
assert.ok(proposal.targetFiles.some((target) => target.file === "apps/api/AGENTS.md"));
assert.match(result.output, /e2 ADD\s+apps\/api\/AGENTS\.md: /);
assert.match(result.output, /nested apps\/api\/AGENTS\.md · 2 session\(s\) under apps\/api\/ · budget /);
// At the floor: the count is shown, with no warning.
assert.match(
result.output,
/nested apps\/api\/AGENTS\.md · 2 of 4 sampled session\(s\) worked only under apps\/api\/ · budget /,
);
assert.doesNotMatch(result.output, /below minGapEvidence/);
assert.deepEqual(
[proposal.nested[0].sampledSessions, proposal.nested[0].minGapEvidence, proposal.nested[0].belowMinGapEvidence],
[4, 2, false],
);

// The nested file's state is its own; the root's evidence dir holds only root evidence.
const nestedState = path.join(dir, ".backpass", "nested");
Expand All @@ -329,6 +338,68 @@ test("a named nested file learns only from its subtree, and each lesson lands in
assert.match(applied.output, /wrote apps\/api\/AGENTS\.md \(e2\)/);
});

function status(dir, args = []) {
const result = spawnSync(process.execPath, [CLI, "status", ...args], {
cwd: dir,
encoding: "utf8",
env: { ...process.env, NO_COLOR: "1", CI: "1" },
});
return { ...result, output: `${result.stdout}${result.stderr}` };
}

test("a nested file whose sampled sessions fall below minGapEvidence is named in the run summary and status", () => {
const dir = makeMonorepo({ config: { nestedMemoryFiles: ["apps/api/AGENTS.md"] } });
const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-nested-sparse-home-"));
writeSession(home, "api-one", dir, { quote: API_QUOTE, file: "apps/api/src/routes/orders.ts" });
writeSession(home, "web-one", dir, { quote: WEB_QUOTE, file: "apps/web/src/checkout.tsx" });
writeSession(home, "web-two", dir, { quote: WEB_QUOTE, file: "apps/web/src/checkout.tsx" });

const before = status(dir);
assert.equal(before.status, 0, before.output);
assert.match(before.output, /apps\/api\/AGENTS\.md: not yet measured - run `backpass` or `backpass analyze`/);

const result = run(dir, home, []);
assert.equal(result.status, 0, result.output);
const warning = "below minGapEvidence (2); no add, rewrite, or remove edit can reach this file";
assert.match(
result.stdout,
/nested apps\/api\/AGENTS\.md · 1 of 3 sampled session\(s\) worked only under apps\/api\//,
);
assert.ok(result.stdout.includes(`\n ${warning}\n`), result.stdout);
const proposal = readJson(path.join(dir, ".backpass", "proposal.json"));
assert.deepEqual(
[proposal.nested[0].sessions, proposal.nested[0].sampledSessions, proposal.nested[0].belowMinGapEvidence],
[1, 3, true],
);
// Report-only: the root edit the web sessions earned is still proposed.
assert.deepEqual(
proposal.edits.map((edit) => edit.file),
["AGENTS.md"],
);

const text = status(dir);
assert.equal(text.status, 0, text.output);
assert.match(text.stdout, /NESTED EVIDENCE REACH \(sampled sessions at the last attribution, [^;]+; report-only\)/);
assert.match(text.stdout, /apps\/api\/AGENTS\.md: 1 of 3 sampled session\(s\) worked only under apps\/api\//);
assert.ok(text.stdout.includes(`\n ${warning}\n`), text.stdout);

const structured = status(dir, ["--json"]);
assert.equal(structured.status, 0, structured.output);
const { nestedAttribution } = JSON.parse(structured.stdout);
assert.ok(nestedAttribution.measuredAt);
assert.deepEqual(nestedAttribution.files, [
{
path: "apps/api/AGENTS.md",
dir: "apps/api",
measured: true,
sessions: 1,
sampledSessions: 3,
minGapEvidence: 2,
belowMinGapEvidence: true,
},
]);
});

test("a gap spanning API and web is proposed only in root, not again from its API subset", () => {
const dir = makeMonorepo({ config: { nestedMemoryFiles: ["apps/api/AGENTS.md"] } });
const home = fs.mkdtempSync(path.join(os.tmpdir(), "backpass-nested-shared-home-"));
Expand Down
Loading
Loading