From 01e2d58ae3cb22499b93d1a057ef3c1e35e527fd Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 18 Oct 2026 20:36:11 +0200 Subject: [PATCH 01/30] feat(question): a decision post can carry the number the chat reply gave it, so the log and its links never point at another decision --- src/genesis-tools-mcp/lib/server.ts | 3 +- .../lib/tools/question-post.ts | 6 +++ src/question/commands/ask.ts | 1 + src/question/lib/decisions/items.ts | 12 ++++++ src/question/lib/decisions/schema.ts | 9 ++++ src/question/lib/decisions/store.test.ts | 42 +++++++++++++++++++ src/question/lib/decisions/store.ts | 9 +++- 7 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/genesis-tools-mcp/lib/server.ts b/src/genesis-tools-mcp/lib/server.ts index cd62eb9cbe..1a00d68766 100644 --- a/src/genesis-tools-mcp/lib/server.ts +++ b/src/genesis-tools-mcp/lib/server.ts @@ -107,7 +107,8 @@ export function serverInstructions({ askViaQuestionTool, inboxState }: Instructi "one list.\n" + 'DECISIONS AND TODOS: a `question_post` item with `type: "decision"` or `type: "todo"` is not a form. ' + "It is numbered in this session's decision log (❓ DECISION N, TODO N; numbers never reused) and the " + - "result is the markdown section to paste into your reply. " + + "result is the markdown section to paste into your reply. When your reply already numbered it, pass that " + + "`number`, so the log and the chat agree. " + decisionNudge({ state: inboxState, askViaQuestionTool }) + "Record " + "progress (acknowledged, implemented, commit refs, verdict, comments, a copy of a chat answer) with " + diff --git a/src/genesis-tools-mcp/lib/tools/question-post.ts b/src/genesis-tools-mcp/lib/tools/question-post.ts index 6d106d488d..f42d923533 100644 --- a/src/genesis-tools-mcp/lib/tools/question-post.ts +++ b/src/genesis-tools-mcp/lib/tools/question-post.ts @@ -346,6 +346,12 @@ const ITEM_SCHEMA = { "screenshots for this item (any type): copied into the question store and appended to " + "promptMarkdown as images, so the card shows previews", }, + number: { + type: "integer", + description: + "decision/todo: the number this item already has in your reply (❓ DECISION 24), so the log and the " + + "chat agree and a later link to it opens the right one. Omit it to take the next free number.", + }, supersedes: { type: "string", description: diff --git a/src/question/commands/ask.ts b/src/question/commands/ask.ts index e79e42170e..59491d0177 100644 --- a/src/question/commands/ask.ts +++ b/src/question/commands/ask.ts @@ -97,6 +97,7 @@ const ASK_JSON_HELP = ` for decision/todo: who acts on it, "human" (default for a decision), "agent" or a model name reevaluateWhen decision/todo: a condition that reopens it, e.g. "after the PR merges" refs decision: [{ path, line?, endLine?, sha? }]; the first ref's lines become the excerpt + number decision/todo: the number your chat reply already shows, so the log matches it supersedes decision/todo: id of an open or drafted item this one replaces (it keeps its id and number; the old text stays as a version: ${toolCommand("question show", "", "--versions")}) attachments any item: screenshots, [{ "type": "image", "path": "/abs/shot.png", "label"?: "After" }]; diff --git a/src/question/lib/decisions/items.ts b/src/question/lib/decisions/items.ts index 4cb159c111..0c259f7005 100644 --- a/src/question/lib/decisions/items.ts +++ b/src/question/lib/decisions/items.ts @@ -29,6 +29,8 @@ export interface QuestionItemInput extends CreateAskItemInput { blocking?: boolean; /** Decision/todo: the id of an open or drafted item this post replaces; the old text is kept as a version. */ supersedes?: string; + /** Decision/todo: the number the item already has in the chat reply, so the log uses the same one. */ + number?: number; /** * Screenshots for any item type. `prepareQuestionItems` copies them into the question store, appends them to * `promptMarkdown` as markdown images and removes this field, so neither store ever sees it. @@ -59,6 +61,7 @@ const ITEM_KEYS = new Set([ "refs", "blocking", "supersedes", + "number", "attachments", ]); @@ -130,6 +133,14 @@ export function validateQuestionItems(value: unknown, help: string): QuestionIte ) { problems.push(`${at}: supersedes must be a decision or todo id like d_3_`); } + + if (record.number !== undefined && record.type !== "decision" && record.type !== "todo") { + problems.push(`${at}: number applies to decision and todo items`); + } + + if (record.number !== undefined && (!Number.isInteger(record.number) || Number(record.number) < 1)) { + problems.push(`${at}: number must be a positive whole number, the one your reply shows`); + } }); if (problems.length > 0) { @@ -200,6 +211,7 @@ export function decisionPayload(items: QuestionItemInput[], hint: DecisionSessio ...(item.for ? { for: item.for } : {}), ...(item.reevaluateWhen ? { reevaluateWhen: item.reevaluateWhen } : {}), ...(item.supersedes ? { supersedes: item.supersedes } : {}), + ...(item.number === undefined ? {} : { number: item.number }), ...decisionTransclusion(item.transclusion), })), }; diff --git a/src/question/lib/decisions/schema.ts b/src/question/lib/decisions/schema.ts index 8ed50ea87e..4d22b41362 100644 --- a/src/question/lib/decisions/schema.ts +++ b/src/question/lib/decisions/schema.ts @@ -47,6 +47,15 @@ export const postedDecisionSchema = z.object({ .enum(DECISION_KINDS) .optional() .describe("decision (default): a numbered ❓ DECISION N. todo: a numbered TODO N, options optional."), + number: z + .number() + .int() + .positive() + .optional() + .describe( + "The number this item already has in your reply (❓ DECISION 24), so the log and the chat agree. Omit it " + + "to take the next free number. Refused when this session already used it for this kind." + ), prompt: z.string().min(1).describe("The question, self-contained."), options: z.array(z.string()).default([]).describe("The answer options in order: a), b), c)."), excerpt: z.string().optional().describe("Context text. Defaults to up to 40 lines of the first ref."), diff --git a/src/question/lib/decisions/store.test.ts b/src/question/lib/decisions/store.test.ts index db71652821..9fc47df536 100644 --- a/src/question/lib/decisions/store.test.ts +++ b/src/question/lib/decisions/store.test.ts @@ -21,6 +21,7 @@ import { answerGitWithoutSpawning } from "@genesiscz/utils/test/no-git"; import { z } from "zod"; import { DeliveryUnknownError, deliverToSession, resolveDeliveryTarget } from "./deliver"; import { livePaneTargets, noPaneTargets } from "./deliver.fixtures"; +import { decisionPayload, validateQuestionItems } from "./items"; import { decisionLine, decisionsMarkdown, @@ -111,6 +112,47 @@ describe("decision store", () => { expect(first[0]?.sessionId).toBe("abc"); }); + test("a post keeps the number its chat reply gave, and refuses one the session already used", async () => { + const dir = mkdtempSync(join(tmpdir(), "decisions-")); + const file = join(dir, "decisions.jsonl"); + const events = join(dir, "events.jsonl"); + const chat = await postDecisions(file, events, { + sessionId: "abc", + decisions: [{ number: 24, prompt: "close the alternative?", options: ["a", "b"] }], + }); + const next = await postDecisions(file, events, { + sessionId: "abc", + decisions: [{ prompt: "next", options: ["a"] }], + }); + + expect(chat[0]?.number).toBe(24); + expect(chat[0]?.id).toBe("d_24_abc"); + expect(next[0]?.number).toBe(25); + await expect( + postDecisions(file, events, { + sessionId: "abc", + decisions: [{ number: 24, prompt: "again", options: ["a"] }], + }) + ).rejects.toThrow("DECISION 24 is already in session abc"); + expect(readDecisions(file)).toHaveLength(2); + }); + + test("a question_post item carries its chat number to the store, and only a decision or todo may", () => { + const items = validateQuestionItems( + [{ type: "decision", number: 24, promptMarkdown: "close it?", choices: ["a", "b"] }], + "help" + ); + const payload = postDecisionsInputSchema.parse(decisionPayload(items, { sessionId: "abc" })); + + expect(payload.decisions[0]?.number).toBe(24); + expect(() => validateQuestionItems([{ number: 3, promptMarkdown: "a form" }], "help")).toThrow( + "number applies to decision and todo items" + ); + expect(() => + validateQuestionItems([{ type: "decision", number: 0, promptMarkdown: "x", choices: ["a"] }], "help") + ).toThrow("number must be a positive whole number"); + }); + test("a post allocates its number only after another writer releases the lock", async () => { const dir = mkdtempSync(join(tmpdir(), "decisions-")); const file = join(dir, "decisions.jsonl"); diff --git a/src/question/lib/decisions/store.ts b/src/question/lib/decisions/store.ts index eccae5391f..c7d8830637 100644 --- a/src/question/lib/decisions/store.ts +++ b/src/question/lib/decisions/store.ts @@ -344,7 +344,14 @@ export async function postDecisions( } const used = numbers[kind]; - const number = used.size === 0 ? 1 : Math.max(...used) + 1; + if (decision.number !== undefined && used.has(decision.number)) { + throw new Error( + `${kind === "todo" ? "TODO" : "DECISION"} ${decision.number} is already in session ${sessionId}; ` + + "post a later number, or revise the earlier one with supersedes" + ); + } + + const number = decision.number ?? (used.size === 0 ? 1 : Math.max(...used) + 1); const row: DecisionRecord = { id: `${ID_PREFIX[kind]}_${number}_${sessionId}`, sessionId, From c7f1a92bbcb9c18fe01b9fc1d3aa8655928db675 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 18 Oct 2026 20:55:13 +0200 Subject: [PATCH 02/30] feat(skills, babysit): tmp-reclaim --own-session reclaims the running session's own SwiftPM build output, and briefs give every reply item a severity --- .claude/skills/babysit/scripts/tmp-reclaim.ts | 320 +++++++++++++++++- 1 file changed, 317 insertions(+), 3 deletions(-) diff --git a/.claude/skills/babysit/scripts/tmp-reclaim.ts b/.claude/skills/babysit/scripts/tmp-reclaim.ts index 1ecf70f4bd..17762822d7 100644 --- a/.claude/skills/babysit/scripts/tmp-reclaim.ts +++ b/.claude/skills/babysit/scripts/tmp-reclaim.ts @@ -23,7 +23,18 @@ * - its tree could not be scanned within the time budget. * * bun tmp-reclaim.ts [--run ] [--min-age-hours 6] [--live-hours 12] [--protect ...] + * bun tmp-reclaim.ts --own-session [--min-age-minutes 10] [--run ] * bun tmp-reclaim.ts --apply --plan [--run ] + * + * `--own-session` is for the session that runs it, which the live-session guard otherwise always shields + * (2026-10-18: a lead's own `/tmp/cc/GenesisTools//` held 14 GB of SwiftPM scratch builds while the disk was + * full). It lists BUILD OUTPUT only, inside `/tmp/cc///`: a SwiftPM triple folder + * (`arm64-apple-macosx`, `x86_64-apple-macosx`), a SwiftPM scratch folder (`.build` or a `--scratch-path`) that holds + * nothing but SwiftPM's own entries, and a Swift or clang module cache (only `.pcm`, `.swiftmodule`, `.timestamp`, + * `.modulevalidation` and `.lock` files). Notes, logs, reply files, patches, briefs, worktrees' sources and the + * `removals/` folder are never listed. Every other guard stays: no open file, socket, FIFO or `.git` inside, same + * owner and device, not changed in the last --min-age-minutes (default 10, because the session is live), and on + * apply the same inode and the same build-output class again. */ import { existsSync, @@ -55,8 +66,42 @@ interface Plan { createdAt: number; minAgeHours: number; candidates: Candidate[]; + /** Set by an `--own-session` dry run: the apply re-checks each entry as that session's build output. */ + ownSession?: { id: string; dirs: string[]; minAgeMinutes: number }; } +export type BuildOutputKind = "swiftpm-triple" | "swiftpm-scratch" | "module-cache"; + +const SWIFTPM_TRIPLES = new Set(["arm64-apple-macosx", "x86_64-apple-macosx"]); +/** What SwiftPM itself writes into a scratch path; a folder holding anything else is not pure build output. */ +const SWIFTPM_SCRATCH_ENTRIES = new Set([ + ...SWIFTPM_TRIPLES, + ".lock", + "artifacts", + "build.db", + "checkouts", + "debug", + "debug.yaml", + "index-build", + "manifest.db", + "manifest.db-shm", + "manifest.db-wal", + "plugin-tools.yaml", + "plugins", + "prebuilts", + "registry", + "release", + "release.yaml", + "repositories", + "workspace-state.json", +]); +const MODULE_CACHE_FILES = [".pcm", ".swiftmodule", ".timestamp", ".modulevalidation", ".lock"]; +const MODULE_CACHE_MARKERS = [".pcm", ".swiftmodule", "modules.timestamp"]; +/** Never walked in an own-session scan: dependency trees, repositories and moved-aside files a user may restore. */ +const OWN_SESSION_SKIP = new Set(["node_modules", ".git", "removals"]); +const OWN_SESSION_MAX_DEPTH = 12; +const SESSION_UUID = /^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i; + const PLAN_MAX_AGE_MS = 30 * 60_000; const SCAN_BUDGET_MS = 20_000; const SYSTEM_PREFIXES = [ @@ -295,6 +340,215 @@ function judge(path: string, protection: Protection, open: string[]): Candidate return { path, ino: st.ino, bytes: tree.bytes, newest: tree.newest }; } +function childNames(path: string): string[] | null { + try { + return readdirSync(path); + } catch { + return null; + } +} + +/** True when every file of the tree is a module-cache file and at least one is a compiled module. */ +function isModuleCache(path: string): boolean { + const deadline = Date.now() + SCAN_BUDGET_MS; + const stack = [path]; + let modules = 0; + + while (stack.length) { + if (Date.now() > deadline) { + return false; + } + + const p = stack.pop() as string; + const st = lstatSync(p, { throwIfNoEntry: false }); + + if (!st || st.isSymbolicLink()) { + return false; + } + + if (st.isDirectory()) { + const names = childNames(p); + + if (!names) { + return false; + } + + stack.push(...names.map((name) => join(p, name))); + continue; + } + + if (!st.isFile() || !MODULE_CACHE_FILES.some((ext) => p.endsWith(ext))) { + return false; + } + + if (p.endsWith(".pcm") || p.endsWith(".swiftmodule")) { + modules++; + } + } + + return modules > 0; +} + +/** Which kind of regenerable build output a folder is, or null when it is anything else. */ +export function buildOutputKind(path: string): BuildOutputKind | null { + const st = lstatSync(path, { throwIfNoEntry: false }); + + if (!st?.isDirectory() || st.isSymbolicLink()) { + return null; + } + + const name = basename(path); + + if (SWIFTPM_TRIPLES.has(name)) { + return "swiftpm-triple"; + } + + const names = childNames(path) ?? []; + + if ( + names.length > 0 && + names.every((child) => SWIFTPM_SCRATCH_ENTRIES.has(child)) && + (name === ".build" || names.includes("workspace-state.json") || names.some((n) => SWIFTPM_TRIPLES.has(n))) + ) { + return "swiftpm-scratch"; + } + + if (names.some((child) => MODULE_CACHE_MARKERS.some((m) => child.endsWith(m))) && isModuleCache(path)) { + return "module-cache"; + } + + return null; +} + +/** The scratch folders of one session: every `//` that is a real folder. */ +export function ownSessionDirs(id: string, ccRoot: string): string[] { + if (!SESSION_UUID.test(id)) { + throw new Error(`--own-session needs the full session uuid, got ${SafeJSON.stringify(id)}`); + } + + return (childNames(ccRoot) ?? []) + .map((project) => join(ccRoot, project, id)) + .filter((dir) => { + const st = lstatSync(dir, { throwIfNoEntry: false }); + return Boolean(st?.isDirectory() && !st.isSymbolicLink()); + }); +} + +/** + * The build output inside one session folder. A SwiftPM scratch folder whose tree the guards refuse (its + * `checkouts/` are git clones) is not listed whole; its triple folders below it are. + */ +export function ownBuildOutput(sessionDir: string): string[] { + const out: string[] = []; + const rootDev = lstatSync(sessionDir).dev; + const stack: Array<{ dir: string; depth: number }> = [{ dir: sessionDir, depth: 0 }]; + + while (stack.length) { + const { dir, depth } = stack.pop() as { dir: string; depth: number }; + + for (const name of childNames(dir) ?? []) { + const path = join(dir, name); + const st = lstatSync(path, { throwIfNoEntry: false }); + + if (!st?.isDirectory() || st.isSymbolicLink() || OWN_SESSION_SKIP.has(name)) { + continue; + } + + const kind = buildOutputKind(path); + + if (kind === "swiftpm-scratch" && "refuse" in scan(path, rootDev)) { + stack.push({ dir: path, depth: depth + 1 }); + continue; + } + + if (kind) { + out.push(path); + continue; + } + + if (depth < OWN_SESSION_MAX_DEPTH) { + stack.push({ dir: path, depth: depth + 1 }); + } + } + } + + return out.sort(); +} + +/** The guards for one entry of an own-session plan; the live-session guard is the one it leaves out. */ +export function judgeOwn( + path: string, + options: { sessionDirs: string[]; open: string[]; minAgeMinutes: number } +): Candidate | string { + const st = lstatSync(path, { throwIfNoEntry: false }); + + if (!st) { + return "gone"; + } + + if (st.isSymbolicLink()) { + return "symlink"; + } + + if (st.uid !== uid) { + return "other-user"; + } + + const real = realpathSync(path); + const sessionDir = options.sessionDirs + .filter((dir) => existsSync(dir)) + .map((dir) => realpathSync(dir)) + .find((dir) => real.startsWith(`${dir}/`)); + + if (!sessionDir) { + return "outside-session"; + } + + if ( + relative(sessionDir, real) + .split("/") + .some((part) => OWN_SESSION_SKIP.has(part)) + ) { + return "skipped-folder"; + } + + const kind = buildOutputKind(path); + + if (!kind) { + return "not-build-output"; + } + + if (options.open.some((o) => o === path || o.startsWith(`${path}/`) || o === real || o.startsWith(`${real}/`))) { + return "open-file"; + } + + const tree = scan(path, lstatSync(sessionDir).dev); + + if ("refuse" in tree) { + return tree.refuse; + } + + if (Date.now() - tree.newest < options.minAgeMinutes * 60_000) { + return "recent"; + } + + return { path, ino: st.ino, bytes: tree.bytes, newest: tree.newest }; +} + +/** The session folder an own-session entry sits in (canonical paths only): its quarantine goes there, on one volume. */ +function ownRoot(path: string, sessionDirs: string[]): string | null { + if (typeof path !== "string" || resolve(path) !== path || realpathSync(path) !== path) { + return null; + } + + return ( + sessionDirs + .filter((dir) => existsSync(dir)) + .map((dir) => realpathSync(dir)) + .find((dir) => path.startsWith(`${dir}/`)) ?? null + ); +} + /** Only canonical paths (realpath equal, so no symlink anywhere in them) with the planner's shapes may be reclaimed. */ function candidateRoot(path: string): string | null { if (typeof path !== "string" || resolve(path) !== path) { @@ -348,7 +602,62 @@ function entries(): string[] { const gb = (bytes: number) => `${(bytes / 1e9).toFixed(2)} GB`; +function ownSessionDryRun(id: string): void { + const minAgeMinutes = Number(flag("--min-age-minutes") ?? 10); + + if (!Number.isFinite(minAgeMinutes) || minAgeMinutes < 1) { + console.error("✖ --min-age-minutes must be at least 1"); + process.exit(2); + } + + const dirs = ownSessionDirs(id, join(roots[0] ?? "/tmp", "cc")); + + if (dirs.length === 0) { + console.error(`✖ no /tmp/cc//${id} folder`); + process.exit(2); + } + + const open = openPaths(); + const candidates: Candidate[] = []; + const skipped: Record = {}; + + for (const path of dirs.flatMap((dir) => ownBuildOutput(dir))) { + const verdict = judgeOwn(path, { sessionDirs: dirs, open, minAgeMinutes }); + + if (typeof verdict === "string") { + skipped[verdict] = (skipped[verdict] ?? 0) + 1; + } else { + candidates.push(verdict); + } + } + + candidates.sort((a, b) => b.bytes - a.bytes); + + for (const c of candidates.slice(0, 40)) { + const age = ((Date.now() - c.newest) / 60_000).toFixed(0); + console.log(`${(c.bytes / 1e6).toFixed(0).padStart(7)} MB ${age.padStart(6)} min ${c.path}`); + } + + const total = candidates.reduce((sum, c) => sum + c.bytes, 0); + const planDir = runDir ? join(runDir, "logs") : join(roots[0] ?? "/tmp", "cc"); + mkdirSync(planDir, { recursive: true }); + const planPath = join(planDir, `tmp-reclaim-plan-${Date.now()}.json`); + const plan: Plan = { createdAt: Date.now(), minAgeHours: 0, candidates, ownSession: { id, dirs, minAgeMinutes } }; + writeFileSync(planPath, SafeJSON.stringify(plan, null, 2)); + console.log( + `${candidates.length} build-output candidates of session ${id}, ${gb(total)}; skipped ${SafeJSON.stringify(skipped)}` + ); + console.log(`plan: ${planPath}\napply it within 30 minutes: bun tmp-reclaim.ts --apply --plan ${planPath}`); + process.exit(0); +} + function main(): void { + const ownSession = flag("--own-session"); + + if (!has("--apply") && ownSession) { + ownSessionDryRun(ownSession); + } + if (!has("--apply")) { const protection = protectedEntries(); const open = openPaths(); @@ -409,7 +718,10 @@ function main(): void { // The dry run decided with its own threshold; the re-check uses the same one. minAgeHours = validAge(plan.minAgeHours, "plan.minAgeHours"); - const protection = protectedEntries(); + const own = plan.ownSession; + // An own-session plan is re-checked by judgeOwn, which needs no live-session list (that scan reads every + // agent's transcript folders). + const protection = own ? null : protectedEntries(); const open = openPaths(); let freed = 0; let removed = 0; @@ -421,7 +733,7 @@ function main(): void { for (const planned of plan.candidates) { let root: string | null = null; try { - root = candidateRoot(planned.path); + root = own ? ownRoot(planned.path, own.dirs) : candidateRoot(planned.path); } catch { // A vanished or unreadable path is no longer a candidate either. } @@ -431,7 +743,9 @@ function main(): void { continue; } - const now = judge(planned.path, protection, open); + const now = own + ? judgeOwn(planned.path, { sessionDirs: own.dirs, open, minAgeMinutes: own.minAgeMinutes }) + : judge(planned.path, protection ?? protectedEntries(), open); if (typeof now === "string" || now.ino !== planned.ino) { const why = typeof now === "string" ? now : "replaced"; From aa7bc0f4bee21c6dbc47611ed39dd2181af2b763 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 18 Oct 2026 20:46:58 +0200 Subject: [PATCH 03/30] docs(skills, babysit): Martin's 2026-10-18 rules: round 3 fixes only cheap or same-class Medium and Low, confirmation only for self-spawned workers, Swift suite before ready, tmp-reclaim without asking --- .claude/skills/babysit/SKILL.md | 155 ++++++++++++------ .../babysit/references/brief-self-fetch.md | 5 + .../babysit/references/brief-template.md | 9 +- .claude/skills/babysit/scripts/policy.ts | 12 +- .../skills/babysit/scripts/scripts.test.ts | 135 ++++++++++++++- 5 files changed, 256 insertions(+), 60 deletions(-) diff --git a/.claude/skills/babysit/SKILL.md b/.claude/skills/babysit/SKILL.md index 59441a3552..7fdcbb36fb 100644 --- a/.claude/skills/babysit/SKILL.md +++ b/.claude/skills/babysit/SKILL.md @@ -29,6 +29,21 @@ Invoke, with the Skill tool, before any other step: These skills own their mechanics; this file only says how they fit together. When they disagree with this file, they win for their own mechanics, and you say so in the report. +## 0a. Start of every run: the checklist + +Do all of these before the first dispatch, in this order: + +1. The skills of step 0 are loaded. +2. 🛑 **Disk alarm:** `bun .claude/skills/babysit/scripts/df-watch.ts --below-gb 5` with Bash `run_in_background`. It + exits, and so wakes you, only when free space drops below 5 GB. When it fires, run step 7c's `tmp-reclaim.ts` at + once and start the alarm again; never wait for the user (2026-10-18: the disk sat full three times while the lead + waited for the user, although this script and Martin's standing permission existed). +3. The run card (step 1) and the mode (step 2). +4. The plan confirmation for the workers you spawn yourself (step 2). +5. The owner check of every PR (step 3.1). +6. Decisions: every ❓ DECISION of the run goes into the decision log under its chat number and into the chat, and + gets a `decision` event in the run ledger (review-loop, "Decisions during a loop"). A later mention is a link. + ## 1. The run card: what to configure Write it to `/tmp/cc///babysit/run.json` at the start; the scripts read it. @@ -71,9 +86,12 @@ above 10 GB after all worker worktrees, else `unified`. cherry-picks onto the PR branch in a scratch worktree, and a file that unified changed later than the PR branch conflicts (#511's import list did). Use it only when disk is short. -🛑 **Confirmation period (from 2026-10-14 until Martin says he no longer needs it):** before you dispatch, show the -plan with AskUserQuestion and wait: the PRs, the worker per PR, the mode with the free-disk numbers, `post`, and -the stop point. After the confirmation period, state the plan in one line and proceed. +🛑 **Plan confirmation for the workers you spawn yourself** (Martin, 2026-10-18; it narrows the confirmation period +that started 2026-10-14). Before you dispatch a worker that YOU decided to spawn, show the plan with +AskUserQuestion and wait: the PRs, the worker per PR, the mode with the free-disk numbers, `post`, and the stop +point. A session the user assigns to you (he names it, or another session hands it over on his word) needs no +confirmation: state the plan in one line and proceed. (2026-10-18: the lead dispatched self-spawned workers with no +confirmation.) ## 3. Prepare each PR @@ -90,30 +108,29 @@ the stop point. After the confirmation period, state the plan in one line and pr the hidden channels, `expand` every thread (the L1 table can scramble files). Keep each thread's `threadId`. 4. Read the source at every cited line before you brief: a stale or wrong finding is briefed as such, so the worker does not "fix" it. -5. **Round policy** (Martin, 2026-10-17 12:16; it replaces "fix every finding in every round", because eve opened - new Medium and Low findings on every push of #536 for 28 rounds). The executable rule is `scripts/policy.ts`; - `fill-brief.ts` derives it from the round, GitHub's changed-file count and `--dedicated-review`. Both brief - templates and judging use that generated policy. Count the PR's fix rounds; a follow-up PR starts at round 1. - - A PR with more than 100 changed files: rounds 1 to 3 fix every valid finding, Medium and Low included. From - round 4, fix only Critical and High. - - A PR with 100 changed files or fewer: rounds 1 and 2 fix every valid finding. From round 3, fix only Critical - and High. - - In the rounds that fix only Critical and High, a Medium or Low finding gets a reply (valid or not, and why). - It is not fixed in that PR, and its thread stays OPEN: resolve only a thread that was acted on (fixed, or - rejected with a cited file:line). Record a policy deferral with the reply transport's supported - `"verdict": "belongs-elsewhere"` and explain the policy in the reply; do not add unsupported schema fields. `integrate.ts` keeps it in the summary but partitions it into a separate - `post-replies.ts --no-resolve` batch, never the batch that resolves threads (Martin, 2026-10-17 12:45). - - **A valid security finding is always fixed**, at any severity and in any round (Martin, 2026-10-17 12:45: - "security valid always"). Read the reviewer's category (`🔒 Security`), not only its severity. +5. **Round policy**: review-loop's "Round policy" section is the source; this is its short form (Martin, + 2026-10-18). The executable rule is `scripts/policy.ts`; `fill-brief.ts` derives it from the round, GitHub's + changed-file count and `--dedicated-review`, and both brief templates and judging use that generated policy. + Count the fix rounds of the PR; a follow-up PR starts again at round 1. + - Rounds 1 and 2 fix every valid finding, Medium and Low included, whatever the PR's size. No deferral, and every + file a valid fix needs is in scope. A `no-change` or rejection needs a cited file:line that proves the finding + wrong. + - Round 3 and later fix every valid Critical and High. A Medium or Low is fixed only when it is cheap (about 20 + changed lines) or the same class as an earlier round's fix; every other one gets a reply (valid or not, and why + it waits) and its thread stays OPEN: resolve only a thread that was acted on. Record such a deferral with the + reply transport's `"verdict": "belongs-elsewhere"` and explain the policy in the reply; `integrate.ts` keeps it + in the summary but posts it in a separate `post-replies.ts --no-resolve` batch, never the batch that resolves + threads (Martin, 2026-10-17 12:45). (2026-10-18: a lead fixed a Low in round 3 with neither reason.) + - **A valid security finding is always fixed**, at any severity and in any round. Read the reviewer's category + (`🔒 Security`), not only its severity. - **A dedicated code review the user asks for** (an independent worker review such as `review-brief.ts` on Astra - or Sol) is fixed in full: every valid finding, Medium and Low included, whatever round the PR is in. Fix it - yourself or steer the PR's fixing worker, then judge it as any round (Martin, 2026-10-17 13:05). - - Inside the fix-everything rounds, a deferral is not accepted ("nothing is a follow up or out of scope", Martin, - 2026-10-17), and every file a valid fix needs is in scope. A `no-change` or rejection needs a cited file:line - that proves the finding wrong. + or Sol) is fixed in full, whatever round the PR is in. Fix it yourself or steer the PR's fixing worker, then + judge it as any round (Martin, 2026-10-17 13:05). - **Merge bar:** when the last round's Critical and High findings are fixed and judged, merge without waiting - for eve to review that last fix. The PR still needs at least one CodeRabbit review, green CI and a reply on - every thread. + for eve to review that last fix (eve's degraded pass counts as a review). A PR over about 50 changed files + still needs at least one CodeRabbit review, asked through review-loop's `scripts/cr-queue.ts`; a smaller PR may + land on eve's review. Green CI, and a reply on every Critical, Major, High, Security, refused and deferred + thread (review-loop step 5a). 6. Scope: `git fetch origin `, then `bun .claude/skills/babysit/scripts/pr-scope.ts --compare HEAD`. It lists the PR's files, the test file of each one (a sibling `.test.ts`, or a neighbour test that imports it), and whether unified differs from the PR branch per file. That list is the starting context, not a scope limit. @@ -212,7 +229,7 @@ Three Monitors, all with `timeout_ms: 1800000`, each re-armed as the very next t approving it. Two-way proof: once per new worker, send a ping on the bus (`--to grok_pr` or `--to codex_`) and see the -reply arrive (Sol answered in 20 s, Grok in 59 s). During the confirmation period do it for every worker. +reply arrive (Sol answered in 20 s, Grok in 59 s). Do it for every worker you spawn yourself. Waiting on a Codex turn: `bun .claude/skills/babysit/scripts/wait-turn.ts --run --worker --thread --pr --round ` with Bash `run_in_background`; the harness wakes the lead when the turn ends, and the worker's @@ -249,11 +266,18 @@ PR: VERIFIED, VERIFIED WITH CAVEATS, or REFUTED, with evidence. Judge against the same derived round policy from `scripts/policy.ts` that `fill-brief.ts` put in the brief. Every required finding is fixed in its PR before the push; a valid security finding and every valid finding in a -dedicated user review are always required, including Medium and Low. Other Medium/Low findings deferred by the -ordinary later-round policy get explained replies and remain OPEN. Verify that integrate partitions those replies -into its no-resolve batch. If a required fix needs a file outside the original diff, change it in this PR and explain -it in the reply; scope never excuses a required fix. A Swift change the worker could not run is not judged until the -lead ran its tests: `swift-typecheck.ts` proves it compiles, and #501 round 2 typechecked clean yet broke a test. +dedicated user review are always required. In rounds 1 and 2 that is every valid finding, and a deferral is not +accepted. Medium/Low findings deferred by the later-round policy get explained replies and remain OPEN; verify that +integrate partitions those replies into its no-resolve batch. If a required fix needs a file outside the original +diff, change it in this PR and explain it in the reply; scope never excuses a required fix. A Swift change the worker +could not run is not judged until the lead ran its tests through `swift-slot.ts`: `swift-typecheck.ts` proves it +compiles, and #501 round 2 typechecked clean yet broke a test. + +🛑 **Swift and "ready".** CI runs no Swift. A PR that changes Swift is prepared for merge only after a local run of the +FULL Swift suite of every package it changes, at the PR's final head, with the result written into the PR (the +commands, the head SHA, the test count, the exit code; a round summary's "Also in this push" line or a PR comment). +Add `src/macos/GenesisTools` (`--disable-build-manifest-caching`) when GenesisKit changed. A filtered judge run is +not the suite (Martin, 2026-10-18; that night Swift PRs were called ready before their suites reran). ## 7. Correct, integrate, push @@ -305,11 +329,17 @@ Off unless the user approves it in their own words for this run ("drive the PRs only for PRs the run owns: never another session's PR, never the integration branch. Ask once, early, and keep running review rounds while the question is open: a missing answer never blocks a round, it only means no merge. -A PR lands when all hold: at least 3 review rounds done; every finding required by the derived policy is fixed -and judged (including every valid security finding and all dedicated-review findings); at least one CodeRabbit -review; a reply on every thread; CI green on the landing tree; its base is master or a parent the run merges first -(a PR stacked on another session's branch waits for that branch). Merge without waiting for eve to review the last -judged fix. Policy-deferred Medium/Low threads may remain OPEN. Then the review-loop landing recipe, which +A PR lands when all hold: review-loop step 5a ("Before you call a PR ready") passes; at least 3 review rounds done; +every finding required by the derived policy is fixed and judged (every valid security finding and all +dedicated-review findings included), so no critical or high finding is open; eve reviewed the head before the last +fix (the merge bar, step 3.5); a PR over about 50 files has a CodeRabbit review; a reply on every thread; CI green +on the landing tree; its base is master or a parent the run merges first (a PR stacked on another session's branch +waits for that branch). Merge without waiting for eve to review the last judged fix. Policy-deferred Medium/Low +threads may remain OPEN. In an unattended land-it job a red CI, an open +Critical or High, a conflict or a failing test is the next task, never a reason to park the PR with a note (Martin, +2026-10-05: "you just resolve the issues and iterate until ur able to drive it to master"); keep the guardrails +(`tools github merge` only, no bypassed check, no paid CodeRabbit usage). Then the review-loop landing recipe (its +file-set check for a recommitted stacked branch included), which `bun .claude/skills/babysit/scripts/land.ts --run --pr --merge` runs as one command (without `--merge` it only prepares and prints the squash; every git call is an argument array, because a hand-typed `--force-with-lease=$B:c1a…` in zsh read `:c` as a modifier and pushed without the lease): @@ -330,7 +360,8 @@ only prepares and prints the squash; every git call is an argument array, becaus 3. `git fetch origin ` in the main checkout (`tools github merge` compares against the local remote ref and refuses a "moved" branch otherwise), then `tools github merge --rebase `. Stacks merge bottom-up; the tool retargets dependents. -4. Log a `note` event with the merge SHA, and list every PR that did not land with its reason in the report. +4. Check `gh pr view --json state`: a restacked merge can leave the PR OPEN (#501, 2026-10-15). Log a `note` + event with the merge SHA, and list every PR that did not land with its reason in the report. A PR that conflicts with master, or whose parent was just squash-merged, is restacked first: `bun .claude/skills/babysit/scripts/restack.ts --wt --onto origin/master [--upstream ]`. It runs @@ -344,11 +375,14 @@ code: rerun CI and judge it before the merge; eve need not re-review the last ju CodeRabbit review and a reply on every thread. When the restack carries a fix round too, push it with `integrate.ts … --restacked-from `: the lease is that old head, and the gates check only the PR's own commits. -## 7c. Disk: watch it, reclaim /tmp safely +## 7c. Disk: watch it, reclaim /tmp yourself -A night of worktrees, Swift builds and test runs fills a disk. Start `bun .claude/skills/babysit/scripts/df-watch.ts ---below-gb 5` with Bash `run_in_background` at the start of a run: it exits (and wakes you) only when free space drops -below the threshold. Then, and only with the user's standing permission for /tmp cleanup: +A night of worktrees, Swift builds and test runs fills a disk. `df-watch.ts --below-gb 5` runs from the start of +every run (step 0a.2). When it fires, or whenever free disk is low, reclaim at once with the steps below. Do not wait +for the user and do not ask: Martin gave the standing permission on 2026-10-18 (an agent may reclaim its own +session scratch and its own build output, and `tmp-reclaim.ts` is the sanctioned way to delete in /tmp, not `rm`). +On 2026-10-18 the disk sat full three times while the lead waited for him, and the lead's own session folder held +14 GB, most of it SwiftPM scratch builds. 1. `bun .claude/skills/babysit/scripts/tmp-reclaim.ts --run ` (dry run): lists the candidates in /tmp, $TMPDIR and each `/tmp/cc//` folder, writes a plan file, deletes nothing. It skips live sessions (any agent @@ -359,8 +393,25 @@ below the threshold. Then, and only with the user's standing permission for /tmp inode, still old, still closed), moved into a scratch folder in the same root and deleted there. The result is a `note` in the run ledger. -Never delete a worktree this way (it refuses `.git` anyway): worktrees go through step 8's release. Machine caches -outside /tmp (uv, SwiftPM, a repo's `.build`) are a separate, user-approved decision, not this script's. +Limits you must know: + +- The plain dry run never picks a LIVE session's folder, and that includes your own. For your own session run a + second dry run with `--own-session ` (then `--apply --plan ` as above): it lists BUILD + OUTPUT only inside `/tmp/cc///`, even though the session is live. That is SwiftPM triple + folders (`arm64-apple-macosx`, `x86_64-apple-macosx`), SwiftPM scratch folders that hold only SwiftPM's own + entries, and Swift or clang module caches (only `.pcm`, `.swiftmodule` and their stamp files). It never lists + notes, logs, reply files, patches, briefs, `node_modules` or `removals/`, and it keeps the other guards (no open + file, socket, FIFO or `.git` inside; not changed in the last `--min-age-minutes`, default 10; the same inode and + the same build-output class again at apply). Measured 2026-10-18 on this run's lead session: 38 candidates, + 10.55 GB. Worker and scratch worktrees still go through step 8's release, and review clones through review-loop's + `scripts/cleanup.ts`. +- Never delete a worktree this way (it refuses `.git` anyway): worktrees go through step 8's release. +- Machine caches outside /tmp (uv, SwiftPM, a repo's `.build`) are a separate, user-approved decision, not this + script's. +- 🛑 Never copy a SwiftPM `.build/` folder for a bench or an A/B: build and measure in the checkout's real `.build`, + and for a before and after use two worktrees that already exist (Martin, 2026-10-07; four copies cost 3.5 GB on a + full disk, 2026-10-06, when the session folder had grown to 3.8 GB). +- Check `du -sh /tmp/cc//` before the final report. ## 8. Clean up after every round, automatically @@ -397,16 +448,22 @@ everything that arrived after a merge; it exits 1 while any of those needs work. follow-up commit on a branch from master, with a reply on the thread naming it. Never write "every thread answered" or "clean at merge" without a fresh exit 0, or a list of what is still open. -A CodeRabbit thread that WE resolved without a reply is not a gap: post-replies resolves a fixed CodeRabbit thread -silently on purpose, because a thread reply spends CodeRabbit's hourly chat budget, and the round summary comment -carries the answer. The audit counts those apart ("resolved by us with the answer in the round summary"). Answer by -hand only a thread the reviewer resolved itself with no answer anywhere (2026-10-14: five #515 threads were answered -by hand for nothing). +Thread replies follow review-loop's "Thread replies by severity" (Martin, 2026-10-18): every Critical, Major, High +and Security thread gets our reply on the thread, from any reviewer, CodeRabbit included (every reply item carries +the required `severity`, and post-replies posts the CodeRabbit ones of those severities); so does every refusal and +deferral. A fixed Medium or Low CodeRabbit +thread that WE resolved without a reply is not a gap and needs no backfill: post-replies resolves it silently on +purpose, because a thread reply spends CodeRabbit's hourly chat budget, and the round summary comment carries the +answer. The audit counts those apart ("resolved by us with the answer in the round summary"). Answer by hand only a +thread the reviewer resolved itself with no answer anywhere (2026-10-14: five #515 threads were answered by hand for +nothing). (2026-10-18: Critical and High CodeRabbit threads were resolved with no reply.) + +Before the word "ready" goes into any report, run review-loop step 5a ("Before you call a PR ready") for each PR. ## 10. Bugs in the harness: fix them, after the user confirms A defect in handoff-to, agents-talk, `tools codex`, `tools grok`, `tools claude worker` or `tools agents` that broke a step, cost real time, or reported something false: describe it in one line with the proposed fix and ask the user -(AskUserQuestion) before you change anything. On yes: fix on unified with a test, then cherry-pick onto the open PR +(AskUserQuestion, and post it to the decision log under its chat number, step 0a.6) before you change anything. On yes: fix on unified with a test, then cherry-pick onto the open PR that owns that code (on 2026-10-14 that was #509 for Codex). Do not nitpick: wording, cosmetics and one-off quirks go into the report's "what to change" list, not into a fix. diff --git a/.claude/skills/babysit/references/brief-self-fetch.md b/.claude/skills/babysit/references/brief-self-fetch.md index b4b42f4633..db65f26c83 100644 --- a/.claude/skills/babysit/references/brief-self-fetch.md +++ b/.claude/skills/babysit/references/brief-self-fetch.md @@ -37,6 +37,10 @@ reruns your checks, and pushes. Quality is judged, not speed. (name the commit), or `no-change` (only when a cited file:line proves the finding wrong). For valid findings deferred by the derived policy, use `belongs-elsewhere` (the reply schema's deferral verdict), explain the policy, and keep the thread OPEN. Do not add fields the installed reply schema does not support. + Give every item its `severity`, the reviewer's label lower-cased: `critical`, `major`, `high`, `security`, + `medium`, `minor` or `low` (a security finding is `security` at any level; CodeRabbit's Trivial or Nitpick is + `low`). The file is refused without it, and critical, major, high and security threads get a reply on the + thread. 2. Fix the valid findings selected by the derived round policy at the root. Respect each module's documented contracts and its callers. Add or extend a test that fails without the fix, in an existing test file next to the code. Do not create a new test file unless the code has none. @@ -68,6 +72,7 @@ finding, including rejected ones: "where": ":", "title": "", "context": "", + "severity": "", "verdict": "fixed", "commits": [""], "codeBefore": { "lang": "ts", "code": "<1 to 8 lines>" }, diff --git a/.claude/skills/babysit/references/brief-template.md b/.claude/skills/babysit/references/brief-template.md index ddcf7e3296..daffff3fda 100644 --- a/.claude/skills/babysit/references/brief-template.md +++ b/.claude/skills/babysit/references/brief-template.md @@ -54,9 +54,11 @@ Codex variant: 1. For each finding, read the current source at the cited lines BEFORE you judge it. A finding is a claim with a timestamp, not a fact. One verdict per finding: `fixed`, `rejected` (cite the file:line that disproves it), - `already-done`, or `no-change` (only when a cited file:line proves the finding wrong). A valid finding deferred - by the derived policy uses `belongs-elsewhere` (the reply schema's deferral verdict), an explained reply, and an - OPEN thread. Do not add fields the installed reply schema does not support. + `already-done`, or `no-change` (only when a cited file:line proves the finding wrong). Give every item its + `severity`, the reviewer's label lower-cased (`critical`, `major`, `high`, `security`, `medium`, `minor`, `low`; + security at any level is `security`, CodeRabbit's Trivial or Nitpick is `low`); the file is refused without it. + A valid finding deferred by the derived policy uses `belongs-elsewhere` (the reply schema's deferral verdict), an + explained reply, and an OPEN thread. Do not add fields the installed reply schema does not support. 2. Fix the valid findings selected by the derived round policy at the root, and respect the module's documented contracts (what a function promises to throw or return) and its callers. Add or extend a test that fails without the fix, in the test files listed above. Do NOT create a new test file. @@ -85,6 +87,7 @@ Codex variant: "where": ":", "title": "", "context": "", + "severity": "", "verdict": "fixed", "commits": [""], "codeBefore": { "lang": "ts", "code": "<1 to 8 lines before>" }, diff --git a/.claude/skills/babysit/scripts/policy.ts b/.claude/skills/babysit/scripts/policy.ts index d81b2543f7..d5160e2e28 100644 --- a/.claude/skills/babysit/scripts/policy.ts +++ b/.claude/skills/babysit/scripts/policy.ts @@ -1,14 +1,20 @@ -/** One policy for generated worker briefs, judging and reply disposition. Dedicated user reviews are fixed in full. */ +/** + * One policy for generated worker briefs, judging and reply disposition (review-loop "Round policy", Martin, + * 2026-10-18): rounds 1 and 2 fix every valid finding whatever the PR's size; later rounds fix Critical, High and + * security, plus a Medium or Low that is cheap or of an earlier fix's class. Dedicated user reviews are fixed in full. + */ export function roundPolicy(options: { round: number; changedFiles: number; dedicatedReview?: boolean }) { const { round, changedFiles, dedicatedReview = false } = options; if (!Number.isSafeInteger(round) || round < 1 || !Number.isSafeInteger(changedFiles) || changedFiles < 0) { throw new Error("round and changed-file count must be finite integers (round >= 1, files >= 0)"); } - const throughRound = changedFiles > 100 ? 3 : 2; + const throughRound = 2; const fixAll = dedicatedReview || round <= throughRound; const text = [ `Round ${round} (${changedFiles} changed files${dedicatedReview ? ", dedicated user review" : ""}): ` + - (fixAll ? "fix EVERY valid finding, critical, high, medium and low." : "fix Critical and High findings."), + (fixAll + ? "fix EVERY valid finding, critical, high, medium and low." + : "fix Critical and High findings, and a valid Medium or Low only when the fix is cheap (about 20 changed lines) or the same class as an earlier round's fix."), "A valid security finding is always fixed, at any severity and in any round; use the reviewer's Security category.", fixAll ? "Fix every valid finding in this PR, including every file it needs; no deferral is accepted." diff --git a/.claude/skills/babysit/scripts/scripts.test.ts b/.claude/skills/babysit/scripts/scripts.test.ts index 1e91f603ed..7ac377fea8 100644 --- a/.claude/skills/babysit/scripts/scripts.test.ts +++ b/.claude/skills/babysit/scripts/scripts.test.ts @@ -30,7 +30,7 @@ import { reviewBrief } from "./review-brief"; import { loadRun, runPaths } from "./run"; import { trySwiftSlot } from "./swift-slot"; import { parseSwiftTargets, typecheckPackage } from "./swift-typecheck"; -import { openPaths } from "./tmp-reclaim"; +import { buildOutputKind, judgeOwn, openPaths, ownBuildOutput, ownSessionDirs } from "./tmp-reclaim"; import { conflictedFiles, higherVersionSide, @@ -1450,7 +1450,7 @@ describe("round 5 carried findings", () => { }); }); -test("generated worker briefs follow the size, round and dedicated-review policy", () => { +test("generated worker briefs follow the round and dedicated-review policy", () => { const dir = fs.mkdtempSync(join(tmpdir(), "babysit-policy-")); const preload = join(dir, "github-fixture.ts"); fs.writeFileSync(join(dir, "run.json"), SafeJSON.stringify({ repo: "fixture/repo", main: dir })); @@ -1475,7 +1475,7 @@ test("generated worker briefs follow the size, round and dedicated-review policy [63, 1, false, true], [63, 2, false, true], [63, 3, false, false], - [120, 3, false, true], + [120, 3, false, false], [120, 4, false, false], [120, 32, true, true], ] as const) { @@ -1513,6 +1513,7 @@ test("generated worker briefs follow the size, round and dedicated-review policy expect(brief).toContain("A valid security finding is always fixed"); expect(brief).toContain("Change every file the fix needs in this PR"); if (!all) { + expect(brief).toContain("about 20 changed lines"); expect(brief).toContain("deferred"); expect(brief).toContain("OPEN"); expect(brief).not.toContain("never defer"); @@ -4151,7 +4152,7 @@ describe("round 22 external Swift products", () => { }); try { for (const [manifest, expected] of [ - ["src/macos/GenesisTools/Package.swift", ["GenesisTools", "GenesisToolsTests"]], + ["src/macos/GenesisTools/Package.swift", ["GenesisToolsHost", "GenesisTools", "GenesisToolsTests"]], ["src/macos/GenesisWidgetPreview/Package.swift", ["GenesisWidgetPreview"]], ] as const) { const text = fs.readFileSync(join(import.meta.dir, "../../../..", manifest), "utf8"); @@ -4973,7 +4974,9 @@ describe("round 32 policy and landing routing", () => { true ); expect(roundPolicy({ round: 3, changedFiles: 100 }).fixAll).toBe(false); - expect(roundPolicy({ round: 3, changedFiles: 101 }).fixAll).toBe(true); + // One rule for every size since 2026-10-18: a PR over 100 files no longer gets a third fix-everything round. + expect(roundPolicy({ round: 3, changedFiles: 101 }).fixAll).toBe(false); + expect(roundPolicy({ round: 2, changedFiles: 101 }).fixAll).toBe(true); for (const template of ["brief-self-fetch.md", "brief-template.md"]) { const text = fs.readFileSync(join(import.meta.dir, "../references", template), "utf8"); expect(text).toContain("{{ROUND_POLICY}}"); @@ -5150,3 +5153,125 @@ describe("round 32 policy and landing routing", () => { } ); }); + +describe("tmp-reclaim --own-session lists only the live session's build output", () => { + const session = "1b4001ba-d479-40f2-9e25-b13cb2ef5a13"; + + function plantSession(): { cc: string; dir: string; nonBuild: string[] } { + const cc = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "babysit-own-"))); + const dir = join(cc, "Project", session); + const file = (path: string, text = "x") => { + fs.mkdirSync(dirname(join(dir, path)), { recursive: true }); + fs.writeFileSync(join(dir, path), text); + }; + // Build output: a scratch path of SwiftPM entries only, a `.build` whose checkouts are git clones (so only its + // triple folder may go), and a module cache. + file("pluggable/b-hub/arm64-apple-macosx/debug/App.o"); + file("pluggable/b-hub/workspace-state.json", "{}"); + file("pluggable/b-hub/build.db"); + file("pkg/.build/arm64-apple-macosx/debug/Kit.o"); + file("pkg/.build/workspace-state.json", "{}"); + file("pkg/.build/checkouts/dep/.git/HEAD", "ref: refs/heads/main\n"); + file("pluggable/mc/ABC/Foundation-1.pcm"); + file("pluggable/mc/Swift-2.swiftmodule"); + file("pluggable/mc/modules.timestamp"); + // Everything below must never be listed, nor any folder above it. + const nonBuild = [ + "briefs/520-r1.md", + "logs/run.log", + "524-r2-replies.json", + "fix.patch", + "pluggable/b-notes/notes.md", + "mc-mixed/readme.txt", + "removals/old/arm64-apple-macosx/debug/Old.o", + "worktrees/w/node_modules/dep/.build/arm64-apple-macosx/x.o", + ]; + for (const path of nonBuild) { + file(path); + } + file("pluggable/b-notes/arm64-apple-macosx/debug/Notes.o"); + file("pluggable/b-notes/workspace-state.json", "{}"); + file("mc-mixed/Foundation-1.pcm"); + fs.symlinkSync(join(dir, "pluggable/b-hub/arm64-apple-macosx"), join(dir, "linked-triple")); + return { cc, dir, nonBuild: nonBuild.map((path) => join(dir, path)) }; + } + + test("the dry-run list holds build output and never a planted non-build file", () => { + const { dir, nonBuild } = plantSession(); + const listed = ownBuildOutput(dir); + expect(listed).toEqual( + [ + "pkg/.build/arm64-apple-macosx", + "pluggable/b-hub", + "pluggable/b-notes/arm64-apple-macosx", + "pluggable/mc", + ].map((path) => join(dir, path)) + ); + for (const path of nonBuild) { + expect(listed.filter((candidate) => path === candidate || path.startsWith(`${candidate}/`))).toEqual([]); + } + expect(buildOutputKind(join(dir, "pluggable/b-notes"))).toBeNull(); + expect(buildOutputKind(join(dir, "mc-mixed"))).toBeNull(); + expect(buildOutputKind(join(dir, "linked-triple"))).toBeNull(); + expect(buildOutputKind(join(dir, "pluggable/mc"))).toBe("module-cache"); + }); + + test("judgeOwn keeps the guards: recent, open, FIFO, outside the session, not build output", () => { + const { cc, dir } = plantSession(); + // A change time can sit a millisecond ahead of Date.now(); with a zero age limit that still reads as recent. + Bun.sleepSync(20); + const options = { sessionDirs: [dir], open: [] as string[], minAgeMinutes: 0 }; + const hub = join(dir, "pluggable/b-hub"); + const verdict = judgeOwn(hub, options); + expect(typeof verdict === "string" ? verdict : verdict.path).toBe(hub); + expect(judgeOwn(hub, { ...options, minAgeMinutes: 10 })).toBe("recent"); + expect(judgeOwn(hub, { ...options, open: [join(hub, "build.db")] })).toBe("open-file"); + expect(judgeOwn(join(dir, "pluggable/b-notes"), options)).toBe("not-build-output"); + expect(judgeOwn(join(dir, "removals/old/arm64-apple-macosx"), options)).toBe("skipped-folder"); + expect(judgeOwn(join(dir, "linked-triple"), options)).toBe("symlink"); + const outside = join(cc, "Other", "arm64-apple-macosx"); + fs.mkdirSync(outside, { recursive: true }); + expect(judgeOwn(outside, options)).toBe("outside-session"); + expect( + Bun.spawnSync(["mkfifo", join(hub, "arm64-apple-macosx/debug/pipe")], { env: process.env }).exitCode + ).toBe(0); + expect(judgeOwn(hub, options)).toBe("socket"); + }); + + test("the session is found by its full uuid only", () => { + const { cc, dir } = plantSession(); + expect(ownSessionDirs(session, cc)).toEqual([dir]); + expect(() => ownSessionDirs("1b4001ba", cc)).toThrow("full session uuid"); + }); + + test("apply removes the planned build output and refuses an entry that is no longer build output", () => { + const { dir } = plantSession(); + const hub = join(dir, "pluggable/b-hub"); + const notesTriple = join(dir, "pluggable/b-notes/arm64-apple-macosx"); + const candidate = (path: string) => ({ path, ino: fs.lstatSync(path).ino, bytes: 1, newest: 0 }); + const planned = [candidate(hub), candidate(notesTriple)]; + // After the dry run someone put a note into the triple folder's place: the apply must refuse it. + fs.rmSync(notesTriple, { recursive: true }); + fs.mkdirSync(notesTriple); + fs.writeFileSync(join(notesTriple, "kept.md"), "a note"); + const plan = join(dir, "..", "plan.json"); + fs.writeFileSync( + plan, + SafeJSON.stringify({ + createdAt: Date.now(), + minAgeHours: 0, + candidates: planned, + ownSession: { id: session, dirs: [dir], minAgeMinutes: 0 }, + }) + ); + const result = Bun.spawnSync( + [process.execPath, join(import.meta.dir, "tmp-reclaim.ts"), "--apply", "--plan", plan], + { stdout: "pipe", stderr: "pipe", env: process.env } + ); + expect(result.stderr.toString()).toBe(""); + expect(result.stdout.toString()).toContain("removed 1 of 2 planned entries"); + expect(fs.existsSync(hub)).toBe(false); + expect(fs.readFileSync(join(notesTriple, "kept.md"), "utf8")).toBe("a note"); + expect(fs.existsSync(join(dir, "pluggable/b-notes/notes.md"))).toBe(true); + }); +}); From ce3c083a83a83bc4a3ffe812f59a4711cf008141 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 21 Oct 2026 22:06:09 +0200 Subject: [PATCH 04/30] fix(question): a decision the agent got answered in chat moves to acknowledged or implemented without a send, unless a queued delivery holds it --- src/question/lib/decisions/store.test.ts | 23 +++++++++++++++++++++++ src/question/lib/decisions/store.ts | 10 ++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/question/lib/decisions/store.test.ts b/src/question/lib/decisions/store.test.ts index 9fc47df536..dff1dd4bf9 100644 --- a/src/question/lib/decisions/store.test.ts +++ b/src/question/lib/decisions/store.test.ts @@ -1675,6 +1675,29 @@ test("an answer reserved by a queued delivery cannot be changed under its queue } expect(readDecisions(file)[0].option).toBe("a"); await updateDecision(file, events, row.id, { comment: "a note is still fine" }); + await expect(updateDecision(file, events, row.id, { state: "implemented" })).rejects.toThrow( + "reserved by a queued delivery" + ); +}); + +test("an answer the agent got in chat moves to acknowledged and implemented without a send", async () => { + const dir = mkdtempSync(join(tmpdir(), "decision-chat-answer-")); + const file = join(dir, "decisions.jsonl"); + const events = join(dir, "events.jsonl"); + const [first, second] = await postDecisions(file, events, { + sessionId: "test-session", + decisions: [ + { prompt: "First?", options: ["yes"] }, + { prompt: "Second?", options: ["no"] }, + ], + }); + for (const row of [first, second]) { + await updateDecision(file, events, row.id, { state: "answered", answer: "yes" }); + } + + expect((await updateDecision(file, events, first.id, { state: "acknowledged" })).state).toBe("acknowledged"); + expect((await updateDecision(file, events, first.id, { state: "implemented" })).state).toBe("implemented"); + expect((await updateDecision(file, events, second.id, { state: "implemented" })).state).toBe("implemented"); }); test("decision revisions preserve the source turn of each posted version", async () => { diff --git a/src/question/lib/decisions/store.ts b/src/question/lib/decisions/store.ts index c7d8830637..8e7948c7fa 100644 --- a/src/question/lib/decisions/store.ts +++ b/src/question/lib/decisions/store.ts @@ -180,8 +180,10 @@ const NEXT: Record> = { // Saving a draft again replaces it: the user edits a draft more than once before sending. // Clearing the last pick puts it back to open; Dismiss drops it without an answer. drafted: ["answered", "drafted", "open", "dismissed"], - answered: ["sent"], - sent: ["acknowledged"], + // An agent that got the answer in chat records its progress without a send (patched() refuses it while a + // queued delivery holds the row). + answered: ["sent", "acknowledged", "implemented"], + sent: ["acknowledged", "implemented"], acknowledged: ["implemented"], implemented: [], dismissed: [], @@ -512,6 +514,10 @@ function patched(row: DecisionRecord, patch: DecisionPatch, ts: string): Decisio throw new Error(`cannot move ${row.id} from ${row.state} to ${patch.state}`); } + if (row.state === "answered" && patch.state && patch.state !== "sent" && row.delivery?.queueId) { + throw new Error(`${row.id} is reserved by a queued delivery; wait for its consumer acknowledgement`); + } + // A queued answer is matched by revision on ACK, and these fields do not bump it; the ACK must not confirm new text. const content = [patch.answer, patch.option, patch.draft, patch.draftOption].some((value) => value !== undefined); if (row.delivery?.queueId && content) { From dab09f19fa3af6bee52eaa51fcf866ac175e3042 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 21 Oct 2026 23:55:36 +0200 Subject: [PATCH 05/30] fix(question): an item that carries the post's wait, or a source, names where the key belongs --- src/question/commands/ask.test.ts | 7 +++++++ src/question/lib/decisions/items.ts | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/src/question/commands/ask.test.ts b/src/question/commands/ask.test.ts index 1b8c390749..98578724ba 100644 --- a/src/question/commands/ask.test.ts +++ b/src/question/commands/ask.test.ts @@ -69,6 +69,13 @@ describe("parseItems", () => { expect(() => parseItems({ json })).toThrow(/Run tools question ask --help/); }); + test("a post-level wait or a source inside an item names where it belongs", () => { + const json = '[{"type":"decision","promptMarkdown":"Keep?","wait":true,"source":"a.ts"}]'; + + expect(() => parseItems({ json })).toThrow(/unknown key "wait" \(did you mean the post's own "wait"/); + expect(() => parseItems({ json })).toThrow(/unknown key "source" \(did you mean refs\?\)/); + }); + test("an item without promptMarkdown is refused, also inside a question_post payload", () => { expect(() => parseItems({ json: '{"items":[{"type":"todo","title":"Rerun"}]}' })).toThrow( /item 1: promptMarkdown must be a non-empty string/ diff --git a/src/question/lib/decisions/items.ts b/src/question/lib/decisions/items.ts index 0c259f7005..87833764f1 100644 --- a/src/question/lib/decisions/items.ts +++ b/src/question/lib/decisions/items.ts @@ -73,6 +73,10 @@ const KEY_HINTS: Record = { options: "choices", images: "attachments", screenshots: "attachments", + // `source` and `wait` inside an item both failed a post on 2026-10-21; `wait` belongs to the post itself. + source: "refs", + sources: "refs", + wait: 'the post\'s own "wait", next to "items"', }; const ITEM_TYPES = new Set(["question", "decision", "todo"]); From 57fb24b270100acc1a62fb14555eb855adcac569 Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 22 Oct 2026 16:54:56 +0200 Subject: [PATCH 06/30] feat(tools): every failed tool run lands in a day's failures log with its command, message, stderr and the agent, session and model that ran it, from runTool and the entries with parsers of their own --- src/ask/index.ts | 4 +- src/cursor/index.ts | 5 + src/jenkins-mcp/index.ts | 3 + src/macos-resources/index.tsx | 4 + src/utils/cli/commander.test.ts | 25 +- src/utils/cli/commander.ts | 9 +- src/utils/cli/failures/caller.ts | 147 ++++++++++ src/utils/cli/failures/failures.test.ts | 150 +++++++++++ src/utils/cli/failures/record.ts | 140 ++++++++++ src/utils/cli/failures/watch.ts | 345 ++++++++++++++++++++++++ src/utils/cli/run-record.ts | 4 + 11 files changed, 830 insertions(+), 6 deletions(-) create mode 100644 src/utils/cli/failures/caller.ts create mode 100644 src/utils/cli/failures/failures.test.ts create mode 100644 src/utils/cli/failures/record.ts create mode 100644 src/utils/cli/failures/watch.ts diff --git a/src/ask/index.ts b/src/ask/index.ts index 3b29751e6a..d188806dfd 100644 --- a/src/ask/index.ts +++ b/src/ask/index.ts @@ -18,6 +18,7 @@ import { expandAtMentions } from "@ask/utils/at-mentions"; import { webSearchTool } from "@ask/utils/websearch"; import * as p from "@clack/prompts"; import { transcriptionManager } from "@genesiscz/utils/ai/transcription/TranscriptionManager"; +import { watchToolRun } from "@genesiscz/utils/cli/failures/watch"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; import { logger, out } from "@genesiscz/utils/logger"; import { input } from "@genesiscz/utils/prompts/clack"; @@ -845,7 +846,8 @@ class ASKTool { } } -// Run the tool +// Run the tool. Its argument parser is its own (ask/utils/cli), so it records its failures itself, as runTool does. +watchToolRun({ tool: "ask" }); const askTool = new ASKTool(); askTool.main().catch((err) => { logger.error(`Unexpected error: ${err}`); diff --git a/src/cursor/index.ts b/src/cursor/index.ts index ef2e68254e..352f3f04d4 100644 --- a/src/cursor/index.ts +++ b/src/cursor/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun +import { noteUsageError, watchToolRun } from "@genesiscz/utils/cli/failures/watch"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; import { out } from "@genesiscz/utils/logger"; import { handleReadmeFlag } from "@genesiscz/utils/readme"; @@ -8,12 +9,15 @@ import { parseCursorArgs } from "./lib/args"; import { streamCursorAgent } from "./lib/stream-agent"; handleReadmeFlag(import.meta.url); +// The parser below is stricter than commander on purpose (lib/args.ts), so this entry records its failures itself. +watchToolRun({ tool: "cursor" }); // ─── CLI argument parsing ────────────────────────────────────────────────── const parsed = parseCursorArgs(process.argv.slice(2), process.cwd()); if (!parsed.ok) { + noteUsageError(parsed.message); out.error(pc.red(parsed.message)); out.error(pc.dim(`Run ${toolCommand("cursor")} --help for the options.`)); process.exit(1); @@ -42,6 +46,7 @@ Each option also takes its value as --option=value. A question that starts with } if (!question) { + noteUsageError("No question provided."); out.error(pc.red("No question provided.")); out.error(pc.dim(`Usage: ${toolCommand("cursor")} "which service creates the reservation?"`)); process.exit(1); diff --git a/src/jenkins-mcp/index.ts b/src/jenkins-mcp/index.ts index 5620efd6e2..2b527de874 100644 --- a/src/jenkins-mcp/index.ts +++ b/src/jenkins-mcp/index.ts @@ -1,5 +1,8 @@ #!/usr/bin/env bun +import { watchToolRun } from "@genesiscz/utils/cli/failures/watch"; + // Kept so `tools jenkins-mcp` and MCP configs that launch it keep working. The code lives in src/jenkins. +watchToolRun({ tool: "jenkins-mcp" }); const { runEntry } = await import("../jenkins/lib/mcp/entry"); await runEntry(process.argv.slice(2)); diff --git a/src/macos-resources/index.tsx b/src/macos-resources/index.tsx index 548d19d08c..ce166ae813 100644 --- a/src/macos-resources/index.tsx +++ b/src/macos-resources/index.tsx @@ -2,6 +2,7 @@ import "@genesiscz/utils/ink/react-production"; import { parseArgs } from "node:util"; +import { noteUsageError, watchToolRun } from "@genesiscz/utils/cli/failures/watch"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; import { requireInteractiveTty } from "@genesiscz/utils/ink/lib/tty-guard"; import { logger, out } from "@genesiscz/utils/logger"; @@ -602,12 +603,15 @@ const App: React.FC<{ options: AppOptions }> = ({ options }) => { export default App; if (import.meta.main) { + // An Ink app with a node:util parser: it records its failures itself, as runTool does for commander tools. + watchToolRun({ tool: "macos-resources" }); let options: AppOptions | "help"; try { options = parseCliOptions(process.argv.slice(2)); } catch (err) { const message = err instanceof Error ? err.message : String(err); + noteUsageError(message); logger.error(message); process.exit(1); } diff --git a/src/utils/cli/commander.test.ts b/src/utils/cli/commander.test.ts index b9030f8c09..2958aa6427 100644 --- a/src/utils/cli/commander.test.ts +++ b/src/utils/cli/commander.test.ts @@ -1,8 +1,11 @@ import { afterEach, describe, expect, it, spyOn } from "bun:test"; -import { readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; +import { readFailures, type ToolFailure } from "@genesiscz/utils/cli/failures/record"; import { env } from "@genesiscz/utils/env"; import { setConsoleLevel } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { Command } from "commander"; import { addGlobalVerboseOption, @@ -163,9 +166,13 @@ describe("addGlobalVerboseOption trace gate", () => { // runTool: a real run (a child process, because runTool refuses process.argv under the test runner) and a caller // that passes its own argv. describe("runTool error boundary", () => { - async function runFixture(...args: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }> { + // Each run gets its own home, so the failure record it writes can be read back. + async function runFixture( + ...args: string[] + ): Promise<{ exitCode: number; stdout: string; stderr: string; failures: ToolFailure[] }> { + const home = mkdtempSync(join(tmpdir(), "gt-run-tool-")); const proc = Bun.spawn(["bun", "run", BOUNDARY_FIXTURE, ...args], { - env: process.env, + env: { ...process.env, GENESIS_TOOLS_HOME: home }, stdout: "pipe", stderr: "pipe", }); @@ -175,7 +182,10 @@ describe("runTool error boundary", () => { proc.exited, ]); - return { exitCode, stdout, stderr }; + const failures = [ + ...readFailures({ fromDay: "0000-00-00", toDay: "9999-99-99", dir: genesisToolsDirIn(home, "logs") }), + ]; + return { exitCode, stdout, stderr, failures }; } it("reports an action error from a real run as one ERROR line and exits 1", async () => { @@ -185,6 +195,10 @@ describe("runTool error boundary", () => { expect(run.stderr.match(/^ERROR: boom$/gm)?.length).toBe(1); expect(run.stderr).not.toContain("run-tool-boundary.ts"); expect(run.stdout).toBe(""); + expect(run.failures.map((failure) => [failure.kind, failure.tool, failure.command, failure.message])).toEqual([ + ["error", "boundary", "boundary boom", "boom"], + ]); + expect(run.failures[0].stack).toContain("Error: boom"); }); it("leaves a Commander error to Commander's own exit path", async () => { @@ -193,6 +207,9 @@ describe("runTool error boundary", () => { expect(run.exitCode).toBe(1); expect(run.stderr).toContain("unknown command 'nope'"); expect(run.stderr).not.toContain("ERROR:"); + expect(run.failures.map((failure) => [failure.kind, failure.argv, failure.message, failure.exitCode])).toEqual([ + ["usage", ["nope"], "error: unknown command 'nope'", 1], + ]); }); it("hands an action error back to a caller that passes its own argv", async () => { diff --git a/src/utils/cli/commander.ts b/src/utils/cli/commander.ts index da943888d0..838cfe8583 100644 --- a/src/utils/cli/commander.ts +++ b/src/utils/cli/commander.ts @@ -6,8 +6,9 @@ import { consoleFloorFor } from "@genesiscz/utils/logging/tool-policy"; import { type Command, CommanderError } from "commander"; import { setCurrentCommand } from "./current-command"; import { enhanceHelp, markRequiredOptionsDeep, setSuggestCommandProgram, showHelpAfterErrorDeep } from "./executor"; +import { commandPathFromArgv, noteToolError, watchToolRun } from "./failures/watch"; import { observeInterrupts } from "./interrupt"; -import { recordRunOnExit, setRunLabel } from "./run-record"; +import { currentRunLabel, recordRunOnExit, setRunLabel } from "./run-record"; export type Verbosity = 0 | 1 | 2 | 3; @@ -207,6 +208,7 @@ function launchedByTestRunner(): boolean { export function reportUnhandledToolError(error: unknown): void { const message = error instanceof Error ? error.message : String(error); logger.debug({ err: error }, "unhandled tool error"); + noteToolError(error); process.stderr.write(`ERROR: ${message}\n`); process.exitCode = 1; } @@ -263,6 +265,11 @@ export async function runTool( if (argv === process.argv) { recordRunOnExit(tool); + watchToolRun({ + tool, + argv: argv.slice(2), + commandPath: () => currentRunLabel() ?? commandPathFromArgv(program, argv.slice(2)), + }); } if (!opts.ignoreParams?.includes("verbose")) { diff --git a/src/utils/cli/failures/caller.ts b/src/utils/cli/failures/caller.ts new file mode 100644 index 0000000000..5efd4f64e9 --- /dev/null +++ b/src/utils/cli/failures/caller.ts @@ -0,0 +1,147 @@ +import { existsSync, readdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { AgentRuntimeContext } from "@genesiscz/utils/agent/context"; +import { readLastLines } from "@genesiscz/utils/fs/read-lines"; +import { SafeJSON } from "@genesiscz/utils/json"; +import { logger } from "@genesiscz/utils/logger"; +import type { FailureCaller } from "./record"; + +const TAIL_BYTES = 512 * 1024; +const CODEX_DAYS_BACK = 7; + +/** Claude names a project folder after the session's cwd with every non-alphanumeric character as `-`. */ +export function claudeProjectFolder(cwd: string): string { + return cwd.replace(/[^A-Za-z0-9]/g, "-"); +} + +function claudeTranscript(sessionId: string, cwds: string[], configDir: string): string | null { + const projects = join(configDir, "projects"); + const file = `${sessionId}.jsonl`; + + for (const cwd of cwds) { + const direct = join(projects, claudeProjectFolder(cwd), file); + if (existsSync(direct)) { + return direct; + } + } + + if (!existsSync(projects)) { + return null; + } + + for (const folder of readdirSync(projects)) { + const candidate = join(projects, folder, file); + if (existsSync(candidate)) { + return candidate; + } + } + + return null; +} + +function codexRollout(threadId: string, codexHome: string, now: Date): string | null { + for (let back = 0; back < CODEX_DAYS_BACK; back++) { + const day = new Date(now.getTime() - back * 86_400_000); + const dir = join( + codexHome, + "sessions", + String(day.getFullYear()), + String(day.getMonth() + 1).padStart(2, "0"), + String(day.getDate()).padStart(2, "0") + ); + if (!existsSync(dir)) { + continue; + } + + const name = readdirSync(dir).find((entry) => entry.endsWith(`-${threadId}.jsonl`)); + if (name) { + return join(dir, name); + } + } + + return null; +} + +/** The newest model the transcript tail names: Claude's assistant `message.model`, Codex's `turn_context` model. */ +export function modelInTranscriptLines(lines: string[]): string | null { + for (let index = lines.length - 1; index >= 0; index--) { + const line = lines[index]; + if (!line.includes('"model"')) { + continue; + } + + let entry: unknown; + try { + entry = SafeJSON.parse(line, { strict: true }); + } catch (error) { + logger.debug({ error }, "failures: a transcript line did not parse"); + continue; + } + + if (typeof entry !== "object" || entry === null) { + continue; + } + + const record = entry as { type?: unknown; message?: { model?: unknown }; payload?: { model?: unknown } }; + const model = + record.type === "assistant" + ? record.message?.model + : record.type === "turn_context" + ? record.payload?.model + : undefined; + if (typeof model === "string" && model && !model.startsWith("<")) { + return model; + } + } + + return null; +} + +export interface CallerInputs { + context: AgentRuntimeContext; + processEnv: NodeJS.ProcessEnv; + now?: Date; + home?: string; +} + +/** The transcript of the calling session, when this machine has it. */ +export function transcriptOf({ context, processEnv, now = new Date(), home = homedir() }: CallerInputs): string | null { + if (!context.sessionId) { + return null; + } + + if (context.agent === "claude-code") { + const configDir = processEnv.CLAUDE_CONFIG_DIR?.trim() || join(home, ".claude"); + return claudeTranscript(context.sessionId, [context.cwd, context.repoRoot], configDir); + } + + if (context.agent === "codex") { + return codexRollout(context.sessionId, processEnv.CODEX_HOME?.trim() || join(home, ".codex"), now); + } + + return null; +} + +export function failureCaller(inputs: CallerInputs): FailureCaller { + const { context, processEnv } = inputs; + const transcript = transcriptOf(inputs); + let model: string | null = null; + if (transcript) { + model = modelInTranscriptLines(readLastLines(transcript, { maxBytes: TAIL_BYTES }).lines); + } + + return { + agent: context.agent, + sessionId: context.sessionId, + sessionTitle: context.sessionTitle, + model, + effort: processEnv.CLAUDE_EFFORT?.trim() || null, + aiAgent: context.aiAgent ?? (processEnv.AI_AGENT?.trim() || null), + project: context.project, + repoRoot: context.repoRoot, + branch: context.branch, + commitSha: context.commitSha, + worktreePath: context.worktreePath, + }; +} diff --git a/src/utils/cli/failures/failures.test.ts b/src/utils/cli/failures/failures.test.ts new file mode 100644 index 0000000000..c53617385a --- /dev/null +++ b/src/utils/cli/failures/failures.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { claudeProjectFolder, modelInTranscriptLines } from "./caller"; +import { appendFailure, readFailures, type ToolFailure } from "./record"; +import { classifyExit, commandPathFromArgv, jsonErrorMessage, messageOfTail, redactArgv } from "./watch"; + +describe("redactArgv", () => { + it("masks the value of a secret-named flag in both spellings and keeps the rest", () => { + expect(redactArgv(["login", "--api-key", "abc123", "--token=xyz", "--name", "work"])).toEqual([ + "login", + "--api-key", + "[redacted]", + "--token=[redacted]", + "--name", + "work", + ]); + }); + + it("masks a secret-shaped argument without a flag", () => { + expect(redactArgv(["set", "sk-ant-abcdefghijklmnopqrstuvwxyz0123"])).toEqual(["set", "[redacted]"]); + }); +}); + +describe("classifyExit", () => { + const none = { noted: null, stderrTail: "" }; + + it("is no failure on exit 0 and an interrupt on 130 or 143", () => { + expect(classifyExit({ exitCode: 0, ...none })).toBeNull(); + expect(classifyExit({ exitCode: 130, ...none })).toBe("interrupt"); + expect(classifyExit({ exitCode: 143, ...none })).toBe("interrupt"); + }); + + it("names commander's refusal and a bare help as usage, anything else as exit", () => { + expect(classifyExit({ exitCode: 1, noted: null, stderrTail: "error: unknown option '--x'\n" })).toBe("usage"); + expect(classifyExit({ exitCode: 1, noted: null, stderrTail: "Usage: tools ts [options]\n" })).toBe("usage"); + expect(classifyExit({ exitCode: 2, noted: null, stderrTail: "could not find it\n" })).toBe("exit"); + }); + + it("a fatal crash wins even when the exit event reports 0 (Bun after an unhandled rejection)", () => { + expect( + classifyExit({ exitCode: 0, noted: { kind: "crash", error: new Error("x"), fatal: true }, stderrTail: "" }) + ).toBe("crash"); + expect( + classifyExit({ exitCode: 1, noted: { kind: "error", error: new Error("x"), fatal: true }, stderrTail: "" }) + ).toBe("error"); + }); +}); + +describe("messageOfTail", () => { + it("picks commander's error line over the help printed after it", () => { + expect( + messageOfTail("error: missing required argument 'entry'\n\nUsage: tools ts imports analyze \n") + ).toBe("error: missing required argument 'entry'"); + }); + + it("falls back to the last non-empty line", () => { + expect(messageOfTail("working…\nno such worktree: feat/x\n\n")).toBe("no such worktree: feat/x"); + expect(messageOfTail("")).toBeNull(); + }); +}); + +describe("jsonErrorMessage", () => { + it("reads the last JSON error line in each common shape and ignores results", () => { + expect( + jsonErrorMessage( + '{"type":"progress"}\n{"type":"error","command":"update","message":"index is not built"}\n' + ) + ).toBe("index is not built"); + expect(jsonErrorMessage('{"error":{"message":"rate limited"}}')).toBe("rate limited"); + expect(jsonErrorMessage('{"error":"busy"}')).toBe("busy"); + expect(jsonErrorMessage('{"rows":[]}\nplain text')).toBeNull(); + }); +}); + +describe("commandPathFromArgv", () => { + interface Node { + name(): string; + aliases(): string[]; + commands: Node[]; + } + const node = (name: string, commands: Node[] = [], aliases: string[] = []): Node => ({ + name: () => name, + aliases: () => aliases, + commands, + }); + const program = node("ai", [node("usage", [node("sessions")], ["u"])]); + + it("follows the leading words through subcommands and aliases, and stops at an option", () => { + expect(commandPathFromArgv(program, ["u", "sessions", "--bad"])).toBe("ai usage sessions"); + expect(commandPathFromArgv(program, ["usage", "--hours", "sessions"])).toBe("ai usage"); + expect(commandPathFromArgv(program, ["nope"])).toBe("ai"); + }); +}); + +describe("modelInTranscriptLines", () => { + it("reads the newest model of a Claude or a Codex transcript and skips synthetic turns", () => { + const claude = [ + '{"type":"assistant","message":{"model":"claude-sonnet-5-5"}}', + '{"type":"user","message":{"content":"x"}}', + '{"type":"assistant","message":{"model":""}}', + ]; + expect(modelInTranscriptLines(claude)).toBe("claude-sonnet-5-5"); + expect(modelInTranscriptLines(['{"type":"turn_context","payload":{"model":"gpt-6.1-sol"}}'])).toBe( + "gpt-6.1-sol" + ); + expect(modelInTranscriptLines(['{"type":"user"}'])).toBeNull(); + }); + + it("names the project folder the way Claude does", () => { + expect(claudeProjectFolder("/Users/me/Projects/My App.v2")).toBe("-Users-me-Projects-My-App-v2"); + }); +}); + +describe("failure records", () => { + it("reads back what was appended, by day, and skips a cut line", async () => { + const dir = mkdtempSync(join(tmpdir(), "gt-failures-")); + const failure: ToolFailure = { + v: 1, + id: "a-1", + at: new Date(2026, 9, 22, 12).toISOString(), + kind: "usage", + tool: "ts", + command: "ts skeleton", + argv: ["skeleton", "--bogus"], + exitCode: 1, + message: "error: unknown option '--bogus'", + errorName: null, + stack: null, + stderrTail: null, + fatal: true, + durationMs: 120, + pid: 1, + ppid: 0, + cwd: "/repo", + traceId: null, + toolsRoot: "/repo", + caller: null, + }; + appendFailure(failure, dir); + await Bun.write( + join(dir, "2026-10-22-failures.jsonl"), + `${await Bun.file(join(dir, "2026-10-22-failures.jsonl")).text()}{"v":1,"to` + ); + + expect([...readFailures({ fromDay: "2026-10-22", toDay: "2026-10-22", dir })]).toEqual([failure]); + expect([...readFailures({ fromDay: "2026-10-23", toDay: "2026-10-24", dir })]).toEqual([]); + }); +}); diff --git a/src/utils/cli/failures/record.ts b/src/utils/cli/failures/record.ts new file mode 100644 index 0000000000..2f5d121de1 --- /dev/null +++ b/src/utils/cli/failures/record.ts @@ -0,0 +1,140 @@ +import { appendFileSync, existsSync, mkdirSync, readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { AgentRuntimeContext } from "@genesiscz/utils/agent/context"; +import { formatLocalDate } from "@genesiscz/utils/date"; +import { readLinesSync } from "@genesiscz/utils/fs/read-lines"; +import { SafeJSON } from "@genesiscz/utils/json"; +import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + +/** + * - `usage`: commander refused the command line (unknown option, missing argument). + * - `error`: the tool's action threw and runTool printed `ERROR: …`. + * - `crash`: an uncaught exception or an unhandled rejection; `fatal` says whether it ended the process. + * - `exit`: a non-zero exit with none of the above (the tool printed its own message and exited 1). + * - `interrupt`: Ctrl-C or SIGTERM (130 / 143); often a harness timeout rather than a bug. + */ +export const FAILURE_KINDS = ["usage", "error", "crash", "exit", "interrupt"] as const; +export type FailureKind = (typeof FAILURE_KINDS)[number]; + +/** Who ran the tool. Model and effort come from the agent's own transcript and env, never guessed. */ +export interface FailureCaller { + agent: AgentRuntimeContext["agent"]; + sessionId: string | null; + sessionTitle: string | null; + model: string | null; + effort: string | null; + /** `AI_AGENT` as the harness sets it, e.g. `claude-code_2-1-296_agent`. */ + aiAgent: string | null; + project: string; + repoRoot: string; + branch: string | null; + commitSha: string | null; + worktreePath: string | null; +} + +export interface ToolFailure { + v: 1; + id: string; + at: string; + kind: FailureKind; + tool: string; + /** The command path, `ai usage sessions`: names only, never argument values. */ + command: string | null; + /** The arguments after the tool name, secrets masked. */ + argv: string[]; + exitCode: number | null; + message: string | null; + errorName: string | null; + stack: string | null; + /** The last stderr the process wrote, ANSI stripped and secrets masked. */ + stderrTail: string | null; + fatal: boolean; + durationMs: number; + pid: number; + ppid: number; + cwd: string; + /** Set when GenesisTools.app started the command. */ + traceId: string | null; + /** The GenesisTools checkout that ran the tool: the main one or a worktree. */ + toolsRoot: string; + caller: FailureCaller | null; +} + +const FILE = /^(\d{4}-\d{2}-\d{2})-failures\.jsonl$/; + +export function failuresDir(): string { + return genesisToolsDir("logs"); +} + +export function failureLogPath(day: string, dir = failuresDir()): string { + return join(dir, `${day}-failures.jsonl`); +} + +/** One line per failure, appended with a single write so concurrent tools never interleave a record. */ +export function appendFailure(failure: ToolFailure, dir = failuresDir()): void { + const path = failureLogPath(formatLocalDate(new Date(failure.at)), dir); + mkdirSync(dirname(path), { recursive: true }); + appendFileSync(path, `${SafeJSON.stringify(failure)}\n`); +} + +/** The days that have a failure log, oldest first. */ +export function failureDays(dir = failuresDir()): string[] { + if (!existsSync(dir)) { + return []; + } + + return readdirSync(dir) + .flatMap((name) => FILE.exec(name)?.[1] ?? []) + .sort(); +} + +export function isToolFailure(value: unknown): value is ToolFailure { + if (typeof value !== "object" || value === null) { + return false; + } + + const record = value as Record; + return ( + record.v === 1 && + typeof record.tool === "string" && + typeof record.at === "string" && + FAILURE_KINDS.includes(record.kind as FailureKind) + ); +} + +/** Every failure recorded on `fromDay` through `toDay` (inclusive, `YYYY-MM-DD`), oldest first. */ +export function* readFailures({ + fromDay, + toDay, + dir = failuresDir(), +}: { + fromDay: string; + toDay: string; + dir?: string; +}): Generator { + for (const day of failureDays(dir)) { + if (day < fromDay || day > toDay) { + continue; + } + + for (const line of readLinesSync(failureLogPath(day, dir))) { + if (!line.trim()) { + continue; + } + + let parsed: unknown; + try { + parsed = SafeJSON.parse(line, { strict: true }); + } catch (error) { + // A line cut by a full disk or a killed writer; the rest of the day still reads. + logger.debug({ error, day }, "failures: skipped an unreadable line"); + continue; + } + + if (isToolFailure(parsed)) { + yield parsed; + } + } + } +} diff --git a/src/utils/cli/failures/watch.ts b/src/utils/cli/failures/watch.ts new file mode 100644 index 0000000000..893694ac92 --- /dev/null +++ b/src/utils/cli/failures/watch.ts @@ -0,0 +1,345 @@ +import { resolve } from "node:path"; +import { format } from "node:util"; +import { env } from "@genesiscz/utils/env"; +import { SafeJSON } from "@genesiscz/utils/json"; +import { logger } from "@genesiscz/utils/logger"; +import { stripAnsi } from "@genesiscz/utils/string"; +import { currentTraceId } from "@genesiscz/utils/trace"; +import { redactSecretsInText } from "@genesiscz/utils/transclude/redact"; +import { failureCaller } from "./caller"; +import { appendFailure, type FailureCaller, type FailureKind, type ToolFailure } from "./record"; + +const TAIL_CHARS = 4000; +const TOOLS_ROOT = resolve(import.meta.dir, "../../../.."); +const SECRET_FLAG = /^--?[\w-]*(?:token|secret|password|passwd|api[-_]?key|auth|cookie|credential)[\w-]*$/i; +/** Commander's own refusals; each starts a stderr write with `error: `. */ +const USAGE_LINE = + /^error: (?:unknown option|unknown command|missing required argument|option .+ argument missing|required option|too many arguments|option .+ argument .+ is invalid|command-argument value .+ is invalid|option .+ cannot be used with)/m; + +interface Noted { + kind: "usage" | "error" | "crash"; + error: unknown; + fatal: boolean; +} + +interface Run { + tool: string; + argv: string[]; + commandPath: () => string | null; + stderr: string; + /** Only read for a JSON error object; never stored, since stdout is the tool's result. */ + stdout: string; + noted: Noted | null; + ended: boolean; +} + +let run: Run | null = null; + +/** Masks the value of a secret-named flag (`--token x`, `--api-key=x`) and any secret-shaped argument. */ +export function redactArgv(args: readonly string[]): string[] { + return args.map((arg, index) => { + const assigned = /^(--?[\w-]+)=/.exec(arg); + if (assigned && SECRET_FLAG.test(assigned[1])) { + return `${assigned[1]}=[redacted]`; + } + + if (index > 0 && SECRET_FLAG.test(args[index - 1]) && !arg.startsWith("-")) { + return "[redacted]"; + } + + return redactSecretsInText(arg); + }); +} + +interface CommandNode { + name(): string; + aliases(): string[]; + readonly commands: readonly CommandNode[]; +} + +/** `ai usage sessions` for `tools ai usage sessions --bad`: the deepest subcommand the leading words name. */ +export function commandPathFromArgv(program: CommandNode, args: readonly string[]): string { + const names = [program.name()]; + let current = program; + + for (const word of args) { + if (word.startsWith("-")) { + break; + } + + const next = current.commands.find((command) => command.name() === word || command.aliases().includes(word)); + if (!next) { + break; + } + + names.push(next.name()); + current = next; + } + + return names.join(" "); +} + +/** What a run that ends with `exitCode` was. Null when it is not a failure. */ +export function classifyExit({ + exitCode, + noted, + stderrTail, +}: { + exitCode: number; + noted: Noted | null; + stderrTail: string; +}): FailureKind | null { + if (noted?.kind === "crash" && noted.fatal) { + return "crash"; + } + + if (exitCode === 0) { + return null; + } + + if (exitCode === 130 || exitCode === 143) { + return "interrupt"; + } + + if (noted?.kind === "error" || noted?.kind === "usage") { + return noted.kind; + } + + if (USAGE_LINE.test(stderrTail) || /^Usage: /.test(stderrTail.trimStart())) { + return "usage"; + } + + return "exit"; +} + +/** The line a reader needs: commander's `error:` line, else the last non-empty stderr line. */ +export function messageOfTail(tail: string): string | null { + const usage = USAGE_LINE.exec(tail); + if (usage) { + const end = tail.indexOf("\n", usage.index); + return tail.slice(usage.index, end === -1 ? undefined : end); + } + + if (/^Usage: /.test(tail.trimStart())) { + return "no subcommand given; the help was printed"; + } + + const lines = tail.split("\n").filter((line) => line.trim()); + return lines.at(-1)?.trim() ?? null; +} + +/** + * The message of the last JSON error line a `--json` run printed: `{"type":"error","message":…}`, + * `{"error":"…"}` or `{"error":{"message":"…"}}` (du index, ai usage and others report failures on stdout). + */ +export function jsonErrorMessage(stdoutTail: string): string | null { + const lines = stdoutTail.split("\n").filter((line) => line.trimStart().startsWith("{")); + + for (const line of lines.reverse()) { + let value: unknown; + try { + value = SafeJSON.parse(line, { strict: true }); + } catch (error) { + logger.debug({ error }, "failures: a stdout line looked like JSON and was not"); + continue; + } + + if (typeof value !== "object" || value === null) { + continue; + } + + const record = value as { type?: unknown; message?: unknown; error?: unknown }; + if (record.type === "error" && typeof record.message === "string") { + return record.message; + } + + if (typeof record.error === "string") { + return record.error; + } + + if (typeof record.error === "object" && record.error !== null) { + const nested = (record.error as { message?: unknown }).message; + if (typeof nested === "string") { + return nested; + } + } + } + + return null; +} + +function text(chunk: unknown): string { + return typeof chunk === "string" ? chunk : chunk instanceof Uint8Array ? new TextDecoder().decode(chunk) : ""; +} + +function remember(chunk: unknown): void { + if (run) { + run.stderr = (run.stderr + text(chunk)).slice(-TAIL_CHARS); + } +} + +function tapStderr(): void { + const write = process.stderr.write; + process.stderr.write = function (this: NodeJS.WriteStream, ...args: Parameters) { + remember(args[0]); + return Reflect.apply(write, this, args); + } as typeof write; + + const writeOut = process.stdout.write; + process.stdout.write = function (this: NodeJS.WriteStream, ...args: Parameters) { + if (run) { + run.stdout = (run.stdout + text(args[0])).slice(-TAIL_CHARS); + } + + return Reflect.apply(writeOut, this, args); + } as typeof writeOut; + + // Bun's console.error and console.warn write to fd 2 without process.stderr.write. + const consoleError = console.error; + const consoleWarn = console.warn; + console.error = (...data: unknown[]) => { + remember(`${format(...data)}\n`); + consoleError(...data); + }; + console.warn = (...data: unknown[]) => { + remember(`${format(...data)}\n`); + consoleWarn(...data); + }; +} + +function caller(): FailureCaller | null { + try { + // Loaded here, on a failure only: the runtime context pulls the history index reader, which every tool + // would otherwise import at start. The exit event is synchronous, so the load is too. + const { + getAgentRuntimeContext, + }: typeof import("@genesiscz/utils/agent/runtime") = require("@genesiscz/utils/agent/runtime"); + const processEnv = env.getProcessEnv(); + return failureCaller({ context: getAgentRuntimeContext({}, processEnv), processEnv }); + } catch (error) { + logger.debug({ error }, "failures: no caller context"); + return null; + } +} + +function write(kind: FailureKind, exitCode: number | null, fatal: boolean): void { + if (!run) { + return; + } + + try { + const error = run.noted?.error; + const err = error instanceof Error ? error : null; + const tail = redactSecretsInText(stripAnsi(run.stderr)).trim(); + const message = err + ? err.message + : error !== undefined && error !== null + ? String(error) + : (jsonErrorMessage(run.stdout) ?? messageOfTail(tail)); + const failure: ToolFailure = { + v: 1, + id: `${Date.now().toString(36)}-${process.pid}`, + at: new Date().toISOString(), + kind, + tool: run.tool, + command: run.commandPath()?.replace(/^tools /, "") ?? null, + argv: redactArgv(run.argv), + exitCode, + message: message ? redactSecretsInText(stripAnsi(message)).slice(0, 2000) : null, + errorName: err?.name ?? null, + stack: err?.stack ? redactSecretsInText(err.stack).slice(0, 8000) : null, + stderrTail: tail || null, + fatal, + durationMs: Math.round(performance.now()), + pid: process.pid, + ppid: process.ppid, + cwd: process.cwd(), + traceId: currentTraceId() ?? null, + toolsRoot: TOOLS_ROOT, + caller: caller(), + }; + appendFailure(failure); + } catch (writeError) { + logger.debug({ error: writeError }, "failures: the record was not written"); + } +} + +function finish(code: number): void { + if (!run || run.ended) { + return; + } + + run.ended = true; + const exitCode = code || (typeof process.exitCode === "number" ? process.exitCode : 0); + const kind = classifyExit({ exitCode, noted: run.noted, stderrTail: stripAnsi(run.stderr) }); + if (kind) { + write(kind, exitCode, true); + } +} + +/** A crash some other listener handles: the process goes on, so it is written now, not at exit. */ +function noteCrash(error: unknown, fatal: boolean): void { + if (!run) { + return; + } + + run.noted = { kind: "crash", error, fatal }; + if (!fatal) { + write("crash", null, false); + run.noted = null; + } +} + +function onRejection(reason: unknown): void { + const handledElsewhere = process.listenerCount("unhandledRejection") > 1; + noteCrash(reason, !handledElsewhere); + if (handledElsewhere) { + return; + } + + // Bun ends the process on an unhandled rejection unless a listener exists; this listener keeps that ending. + logger.debug({ err: reason }, "unhandled rejection"); + process.stderr.write(`ERROR: unhandled rejection: ${reason instanceof Error ? reason.message : String(reason)}\n`); + process.exit(1); +} + +/** A parser of its own refused the command line (an entry point without commander). */ +export function noteUsageError(message: string): void { + if (run) { + run.noted = { kind: "usage", error: message, fatal: true }; + } +} + +/** The action threw and runTool printed it. */ +export function noteToolError(error: unknown): void { + if (run) { + run.noted = { kind: "error", error, fatal: true }; + } +} + +/** + * Records every failed run of this process into `~/.genesis-tools/logs/-failures.jsonl`: a usage error, a + * thrown action, a crash, a plain non-zero exit, an interrupt. `runTool` calls it; an entry point without commander + * calls it itself. Once per process; only a real CLI run, never a test that passes its own argv. + */ +export function watchToolRun({ + tool, + argv = process.argv.slice(2), + commandPath = () => null, +}: { + tool: string; + argv?: string[]; + commandPath?: () => string | null; +}): void { + if (run) { + return; + } + + run = { tool, argv, commandPath, stderr: "", stdout: "", noted: null, ended: false }; + tapStderr(); + process.on("uncaughtExceptionMonitor", (error) => { + noteCrash(error, process.listenerCount("uncaughtException") === 0); + }); + process.on("unhandledRejection", onRejection); + process.once("exit", finish); +} diff --git a/src/utils/cli/run-record.ts b/src/utils/cli/run-record.ts index b0b1cf15d7..feebd4d513 100644 --- a/src/utils/cli/run-record.ts +++ b/src/utils/cli/run-record.ts @@ -8,6 +8,10 @@ export function setRunLabel(label: string): void { runLabel = label; } +export function currentRunLabel(): string | null { + return runLabel; +} + /** * One `[profile:cli]` line per command run, written at exit: wall time since the process started (Bun * startup and imports included, which is what a caller waits for), CPU time, peak memory, and From f7da1c03c0ff78db607b47f83a61c54b0b59d6fb Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 22 Oct 2026 16:55:15 +0200 Subject: [PATCH 07/30] feat(tools): tools genesis-tools logs lists, groups and shows failed runs, refused command lines, crashes, day-log records and slow timers --- src/genesis-tools/README.md | 37 +++ src/genesis-tools/commands/logs.ts | 393 +++++++++++++++++++++++++ src/genesis-tools/index.ts | 14 + src/genesis-tools/lib/crash-reports.ts | 37 +++ src/genesis-tools/lib/day-log.ts | 154 ++++++++++ src/genesis-tools/lib/failures.ts | 205 +++++++++++++ src/genesis-tools/lib/logs.test.ts | 152 ++++++++++ src/genesis-tools/lib/profile-log.ts | 73 +++++ src/genesis-tools/lib/window.ts | 58 ++++ 9 files changed, 1123 insertions(+) create mode 100644 src/genesis-tools/README.md create mode 100644 src/genesis-tools/commands/logs.ts create mode 100644 src/genesis-tools/index.ts create mode 100644 src/genesis-tools/lib/crash-reports.ts create mode 100644 src/genesis-tools/lib/day-log.ts create mode 100644 src/genesis-tools/lib/failures.ts create mode 100644 src/genesis-tools/lib/logs.test.ts create mode 100644 src/genesis-tools/lib/profile-log.ts create mode 100644 src/genesis-tools/lib/window.ts diff --git a/src/genesis-tools/README.md b/src/genesis-tools/README.md new file mode 100644 index 0000000000..9ca1a7bbec --- /dev/null +++ b/src/genesis-tools/README.md @@ -0,0 +1,37 @@ +# tools genesis-tools + +Developer tools for GenesisTools itself: what failed, crashed or ran slow, and who ran it. + +## Failure log + +Every tool run through `runTool` (and the few entry points without commander, through `watchToolRun`) writes one +JSON line per failed run to `~/.genesis-tools/logs/-failures.jsonl` +(`src/utils/cli/failures/`). Kinds: + +| kind | when | +|---|---| +| `usage` | commander refused the command line: unknown option or command, missing argument, no subcommand | +| `error` | the action threw and `runTool` printed `ERROR: …` | +| `crash` | an uncaught exception or an unhandled rejection; `fatal: false` when another handler kept the process alive | +| `exit` | a non-zero exit with none of the above (the tool printed its own message) | +| `interrupt` | Ctrl-C or SIGTERM (130, 143), often a harness timeout; hidden unless `--interrupts` | + +Each record carries the tool, the command path, the arguments with secrets masked, the exit code, the message, the +stack, the last 4 KB of stderr, the cwd, the pid, the run time, the app trace id when GenesisTools.app started it, the +checkout that ran it, and the caller: agent (Claude Code, Codex, Grok, Copilot, or a shell), session id and title, +model and effort (from the session's own transcript), project, branch and commit. + +## Commands + +```bash +tools genesis-tools logs failures [--since 24h] [--tool git] [--kind usage,error] [--model haiku] [--agent codex] +tools genesis-tools logs show # one failure in full, with the day-log lines of its pid +tools genesis-tools logs stats --by tool,command,model +tools genesis-tools logs usage # refused command lines: where the help or an alias is missing +tools genesis-tools logs crashes # crash records and macOS crash reports of GenesisTools processes +tools genesis-tools logs day --level warn --tool ai --since 2h # the pino day log, through rg +tools genesis-tools logs profile --min-ms 5000 --scope cli # slow timers (tools config profiling) +tools genesis-tools logs paths +``` + +Every listing takes `--json`. `--since` reads `30m`, `24h`, `7d`, `HH:MM` or `YYYY-MM-DD`. diff --git a/src/genesis-tools/commands/logs.ts b/src/genesis-tools/commands/logs.ts new file mode 100644 index 0000000000..efb32fe9da --- /dev/null +++ b/src/genesis-tools/commands/logs.ts @@ -0,0 +1,393 @@ +import { parseVariadic } from "@genesiscz/utils/cli"; +import { FAILURE_KINDS, type FailureKind, failuresDir, type ToolFailure } from "@genesiscz/utils/cli/failures/record"; +import { formatLocalDate } from "@genesiscz/utils/date"; +import { out } from "@genesiscz/utils/logger"; +import { dayLogsDir } from "@genesiscz/utils/logger/day-logs"; +import { formatTable, truncateDisplay } from "@genesiscz/utils/table"; +import type { Command } from "commander"; +import pc from "picocolors"; +import { crashReports, crashReportsDir } from "../lib/crash-reports"; +import { LEVELS, type LevelName, levelName, queryDayLog } from "../lib/day-log"; +import { type FailureFilter, GROUP_KEYS, groupFailures, isGroupKey, loadFailures, usageGaps } from "../lib/failures"; +import { queryProfileLog } from "../lib/profile-log"; +import { daysIn, resolveWindow, type TimeWindow } from "../lib/window"; + +interface WindowOptions { + since?: string; + until?: string; + json?: boolean; +} + +interface FilterOptions extends WindowOptions { + tool?: string[]; + kind?: string[]; + command?: string; + agent?: string; + model?: string; + session?: string; + exit?: string[]; + grep?: string; + interrupts?: boolean; +} + +const KIND_COLOR: Record string> = { + usage: pc.yellow, + error: pc.red, + crash: pc.magenta, + exit: pc.red, + interrupt: pc.dim, +}; + +function withWindow(command: Command, since = "24h"): Command { + return command + .option("--since ", "Start of the window: 30m, 24h, 7d, HH:MM, YYYY-MM-DD", since) + .option("--until ", "End of the window (default: now)") + .option("--json", "Print JSON"); +} + +function withFilters(command: Command): Command { + return withWindow(command) + .option("--tool ", "Only these tools (comma or space separated)") + .option("--kind ", `Only these kinds: ${FAILURE_KINDS.join(", ")}`) + .option("--command ", "Command path contains this (usage sessions)") + .option("--agent ", "claude-code, codex, grok, copilot or unknown (a shell)") + .option("--model ", "Model contains this (sonnet, opus, haiku, gpt-6)") + .option("--session ", "Session id or its leading part") + .option("--exit ", "Only these exit codes") + .option("--grep ", "Message, arguments or stderr contain this") + .option("--interrupts", "Include Ctrl-C and SIGTERM endings"); +} + +function windowOf(options: WindowOptions): TimeWindow { + return resolveWindow({ since: options.since, until: options.until }); +} + +function filterOf(options: FilterOptions): FailureFilter { + const kinds = parseVariadic(options.kind); + const unknown = kinds.filter((kind) => !FAILURE_KINDS.some((known) => known === kind)); + if (unknown.length > 0) { + throw new Error(`--kind ${unknown.join(", ")} is not a kind; use ${FAILURE_KINDS.join(", ")}`); + } + + return { + kinds: FAILURE_KINDS.filter((kind) => kinds.includes(kind)), + tools: parseVariadic(options.tool), + command: options.command, + agent: options.agent, + model: options.model, + session: options.session, + exitCodes: parseVariadic(options.exit).map(Number), + grep: options.grep, + interrupts: options.interrupts === true, + }; +} + +function clock(iso: string, window: TimeWindow): string { + const at = new Date(iso); + const time = at.toTimeString().slice(0, 8); + return daysIn(window).length > 1 ? `${formatLocalDate(at).slice(5)} ${time}` : time; +} + +/** Day-log times are UTC; a reader works in local time. */ +function localTime(iso: string, millis = false): string { + const at = new Date(iso); + const time = at.toTimeString().slice(0, 8); + return millis ? `${time}.${String(at.getMilliseconds()).padStart(3, "0")}` : time; +} + +function who(failure: ToolFailure): string { + const caller = failure.caller; + if (!caller || caller.agent === "unknown") { + return "shell"; + } + + return caller.model ?? caller.agent; +} + +function printFailures(failures: ToolFailure[], window: TimeWindow): void { + if (failures.length === 0) { + out.println(pc.dim("No failures in this window.")); + return; + } + + const rows = failures.map((failure) => [ + pc.dim(clock(failure.at, window)), + KIND_COLOR[failure.kind](failure.kind), + failure.command ?? failure.tool, + String(failure.exitCode ?? "-"), + pc.cyan(who(failure)), + truncateDisplay(failure.message, 90), + pc.dim(failure.id), + ]); + out.println(formatTable(rows, ["TIME", "KIND", "COMMAND", "EXIT", "WHO", "MESSAGE", "ID"])); + out.println(pc.dim(`\n${failures.length} failures. Details: tools genesis-tools logs show `)); +} + +function registerFailures(logs: Command): void { + withFilters(logs.command("failures").alias("list").description("Failed tool runs, newest last")) + .option("--limit ", "Show the newest N", "40") + .action((options: FilterOptions & { limit: string }) => { + const window = windowOf(options); + const failures = loadFailures({ window, filter: filterOf(options) }).slice(-Number(options.limit)); + if (options.json) { + out.result(failures); + return; + } + + printFailures(failures, window); + }); + + withFilters(logs.command("stats").description("Failure counts grouped by tool, kind, model and more")) + .option("--by ", `Group by: ${GROUP_KEYS.join(", ")}`, ["tool", "kind"]) + .option("--top ", "Show the top N groups", "30") + .action((options: FilterOptions & { by: string[]; top: string }) => { + const by = parseVariadic(options.by); + const unknown = by.filter((key) => !isGroupKey(key)); + if (unknown.length > 0) { + throw new Error(`--by ${unknown.join(", ")} is not a key; use ${GROUP_KEYS.join(", ")}`); + } + + const ordered = by.filter(isGroupKey); + const failures = loadFailures({ window: windowOf(options), filter: filterOf(options) }); + const groups = groupFailures(failures, ordered).slice(0, Number(options.top)); + if (options.json) { + out.result({ total: failures.length, by: ordered, groups }); + return; + } + + const rows = groups.map((group) => [ + String(group.count), + ...group.key.map((value) => truncateDisplay(value, 70)), + String(group.sessions), + truncateDisplay(group.models.join(", "), 40), + pc.dim(group.lastId), + ]); + out.println( + formatTable(rows, ["COUNT", ...ordered.map((key) => key.toUpperCase()), "SESSIONS", "MODELS", "LAST"]) + ); + out.println(pc.dim(`\n${failures.length} failures in ${groups.length} groups.`)); + }); + + withFilters(logs.command("usage").description("Command lines the tools refused: where help or an alias is missing")) + .option("--top ", "Show the top N", "25") + .action((options: FilterOptions & { top: string }) => { + const gaps = usageGaps(loadFailures({ window: windowOf(options), filter: filterOf(options) })).slice( + 0, + Number(options.top) + ); + if (options.json) { + out.result(gaps); + return; + } + + if (gaps.length === 0) { + out.println(pc.dim("No refused command lines in this window.")); + return; + } + + for (const gap of gaps) { + out.println(`${pc.bold(String(gap.count).padStart(4))} ${pc.cyan(gap.command)} ${gap.message}`); + out.println(pc.dim(` models: ${gap.models.join(", ") || "-"}`)); + for (const example of gap.examples) { + out.println(pc.dim(` $ ${example}`)); + } + } + }); +} + +function registerShow(logs: Command): void { + logs.command("show") + .description("One failure in full, with the day-log lines its process wrote") + .argument("", "The failure id (from the list), or its leading part") + .option("--since ", "How far back to look for the id", "7d") + .option("--json", "Print JSON") + .action(async (id: string, options: WindowOptions) => { + const failure = loadFailures({ window: windowOf(options), filter: { interrupts: true } }) + .reverse() + .find((candidate) => candidate.id.startsWith(id)); + if (!failure) { + throw new Error(`no failure with id ${id} in the last ${options.since ?? "7d"} (try --since 30d)`); + } + + const end = new Date(failure.at); + const start = new Date(end.getTime() - failure.durationMs - 1000); + const lines = await queryDayLog({ + window: { from: start, to: new Date(end.getTime() + 1000) }, + pid: failure.pid, + limit: 200, + }); + if (options.json) { + out.result({ failure, dayLog: lines.map((line) => line.record) }); + return; + } + + const caller = failure.caller; + const rows: Array<[string, string | null | undefined]> = [ + ["kind", `${failure.kind}${failure.fatal ? "" : " (the process went on)"}`], + ["at", new Date(failure.at).toLocaleString()], + ["command", `tools ${failure.tool} ${failure.argv.join(" ")}`], + ["exit", String(failure.exitCode ?? "-")], + ["message", failure.message], + ["cwd", failure.cwd], + ["took", `${failure.durationMs} ms`], + ["pid", `${failure.pid} (parent ${failure.ppid})`], + ["checkout", failure.toolsRoot], + ["agent", caller ? `${caller.agent}${caller.aiAgent ? ` (${caller.aiAgent})` : ""}` : null], + ["model", caller?.model ? `${caller.model}${caller.effort ? `, effort ${caller.effort}` : ""}` : null], + [ + "session", + caller?.sessionId + ? `${caller.sessionId}${caller.sessionTitle ? ` ${caller.sessionTitle}` : ""}` + : null, + ], + ["project", caller ? `${caller.project} @ ${caller.branch ?? "?"} ${caller.commitSha ?? ""}` : null], + ["app trace", failure.traceId], + ]; + for (const [name, value] of rows) { + if (value) { + out.println(`${pc.dim(name.padEnd(10))} ${value}`); + } + } + + if (failure.stack) { + out.println(`\n${pc.bold("stack")}\n${pc.dim(failure.stack)}`); + } + + if (failure.stderrTail) { + out.println(`\n${pc.bold("stderr (tail)")}\n${failure.stderrTail}`); + } + + out.println(`\n${pc.bold(`day log, pid ${failure.pid}`)} ${pc.dim(`(${lines.length} lines)`)}`); + for (const line of lines) { + out.println(`${pc.dim(localTime(line.time, true))} ${levelName(line.level).padEnd(5)} ${line.msg}`); + } + }); +} + +function registerCrashes(logs: Command): void { + withWindow( + logs.command("crashes").description("Crashed tool runs and macOS crash reports of GenesisTools processes"), + "7d" + ).action((options: WindowOptions) => { + const window = windowOf(options); + const failures = loadFailures({ window, filter: { kinds: ["crash"] } }); + const reports = crashReports(window); + if (options.json) { + out.result({ failures, reports }); + return; + } + + printFailures(failures, window); + out.println(`\n${pc.bold("macOS crash reports")} ${pc.dim(crashReportsDir())}`); + if (reports.length === 0) { + out.println(pc.dim("None in this window.")); + } + + for (const report of reports) { + out.println(`${pc.dim(clock(report.at, window))} ${report.process} ${pc.dim(report.path)}`); + } + }); +} + +function registerDay(logs: Command): void { + withWindow(logs.command("day").description("The day log (every tool's pino records), filtered"), "1h") + .option("--level ", `Lowest level: ${Object.keys(LEVELS).join(", ")}`, "warn") + .option("--tool ", "Only records of this tool") + .option("--pid ", "Only records of this process") + .option("--grep ", "The record contains this") + .option("--limit ", "Show the newest N", "100") + .action( + async ( + options: WindowOptions & { level: string; tool?: string; pid?: string; grep?: string; limit: string } + ) => { + const minLevel = LEVELS[options.level as LevelName]; + if (minLevel === undefined) { + throw new Error(`--level ${options.level} is not a level; use ${Object.keys(LEVELS).join(", ")}`); + } + + const entries = await queryDayLog({ + window: windowOf(options), + minLevel, + tool: options.tool, + pid: options.pid ? Number(options.pid) : undefined, + grep: options.grep, + limit: Number(options.limit), + }); + if (options.json) { + out.result(entries.map((entry) => entry.record)); + return; + } + + for (const entry of entries) { + const level = levelName(entry.level); + const color = + entry.level >= LEVELS.error ? pc.red : entry.level >= LEVELS.warn ? pc.yellow : pc.dim; + out.println( + `${pc.dim(localTime(entry.time))} ${color(level.padEnd(5))} ${pc.cyan((entry.tool ?? "-").padEnd(14))} ${pc.dim(String(entry.pid ?? "").padStart(6))} ${entry.msg}` + ); + } + + out.println(pc.dim(`\n${entries.length} records.`)); + } + ); +} + +function registerProfile(logs: Command): void { + withWindow( + logs.command("profile").description("Slow timers from the profiling log (tools config profiling)"), + "24h" + ) + .option("--min-ms ", "Only timers at least this long", "1000") + .option("--scope ", "Only these scopes (cli, spawn, agent-sessions, …)") + .option("--grep ", "The label contains this") + .option("--pid ", "Only this process") + .option("--top ", "Show the slowest N", "40") + .action( + ( + options: WindowOptions & { minMs: string; scope?: string[]; grep?: string; pid?: string; top: string } + ) => { + const entries = queryProfileLog({ + days: daysIn(windowOf(options)), + minMs: Number(options.minMs), + scopes: parseVariadic(options.scope), + grep: options.grep, + pid: options.pid ? Number(options.pid) : undefined, + }) + .sort((a, b) => b.ms - a.ms) + .slice(0, Number(options.top)); + if (options.json) { + out.result(entries); + return; + } + + const rows = entries.map((entry) => [ + `${(entry.ms / 1000).toFixed(2)} s`, + entry.scope, + truncateDisplay(entry.label, 100), + String(entry.pid), + pc.dim(entry.day), + ]); + out.println(formatTable(rows, ["TIME", "SCOPE", "LABEL", "PID", "DAY"])); + } + ); +} + +export function registerLogsCommand(program: Command): void { + const logs = program + .command("logs") + .description("Failures, crashes, the day log and profiling of every GenesisTools tool"); + registerFailures(logs); + registerShow(logs); + registerCrashes(logs); + registerDay(logs); + registerProfile(logs); + logs.command("paths") + .description("Where each log lives") + .action(() => { + const day = formatLocalDate(new Date()); + out.println(`failures ${failuresDir()}/${day}-failures.jsonl`); + out.println(`day log ${dayLogsDir()}/${day}.log`); + out.println(`profiling ${dayLogsDir()}/${day}-profiling.log`); + out.println(`crashes ${crashReportsDir()}`); + }); +} diff --git a/src/genesis-tools/index.ts b/src/genesis-tools/index.ts new file mode 100644 index 0000000000..2ab17a6b6c --- /dev/null +++ b/src/genesis-tools/index.ts @@ -0,0 +1,14 @@ +#!/usr/bin/env bun +import { runTool } from "@genesiscz/utils/cli"; +import { Command } from "commander"; +import { registerLogsCommand } from "./commands/logs"; + +const program = new Command() + .name("genesis-tools") + .description("Developer tools for GenesisTools itself: what failed, crashed or ran slow, and who ran it"); + +registerLogsCommand(program); + +if (import.meta.main) { + await runTool(program, { tool: "genesis-tools" }); +} diff --git a/src/genesis-tools/lib/crash-reports.ts b/src/genesis-tools/lib/crash-reports.ts new file mode 100644 index 0000000000..2ed9203c40 --- /dev/null +++ b/src/genesis-tools/lib/crash-reports.ts @@ -0,0 +1,37 @@ +import { existsSync, readdirSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { TimeWindow } from "./window"; + +/** The processes GenesisTools runs as: the app's faces, the Genesis apps, and bun under every `tools` command. */ +const OURS = /^(GenesisTools|Genesis|Genesis Markdown|GenesisTools Preview|bun)[-_ ].*\.ips$/; + +export interface CrashReport { + path: string; + process: string; + at: string; +} + +export function crashReportsDir(home = homedir()): string { + return join(home, "Library", "Logs", "DiagnosticReports"); +} + +/** macOS crash and hang reports of GenesisTools processes written inside the window, newest first. */ +export function crashReports(window: TimeWindow, dir = crashReportsDir()): CrashReport[] { + if (!existsSync(dir)) { + return []; + } + + return readdirSync(dir) + .filter((name) => OURS.test(name)) + .flatMap((name): CrashReport[] => { + const path = join(dir, name); + const mtime = statSync(path).mtime; + if (mtime < window.from || mtime > window.to) { + return []; + } + + return [{ path, process: name.replace(/[-_ ]\d{4}-\d{2}-\d{2}.*$/, ""), at: mtime.toISOString() }]; + }) + .sort((a, b) => b.at.localeCompare(a.at)); +} diff --git a/src/genesis-tools/lib/day-log.ts b/src/genesis-tools/lib/day-log.ts new file mode 100644 index 0000000000..488ee461c1 --- /dev/null +++ b/src/genesis-tools/lib/day-log.ts @@ -0,0 +1,154 @@ +import { ripgrepFiles } from "@genesiscz/utils/compress/search"; +import { readLinesSync } from "@genesiscz/utils/fs/read-lines"; +import { SafeJSON } from "@genesiscz/utils/json"; +import { logger } from "@genesiscz/utils/logger"; +import { listDayLogs } from "@genesiscz/utils/logger/day-logs"; +import { ripgrepBinary } from "@genesiscz/utils/ripgrep"; +import { daysIn, inWindow, type TimeWindow } from "./window"; + +export const LEVELS = { trace: 10, debug: 20, info: 30, warn: 40, error: 50, fatal: 60 } as const; +export type LevelName = keyof typeof LEVELS; + +export function levelName(level: number): string { + const found = Object.entries(LEVELS).find(([, value]) => value === level); + return found ? found[0] : String(level); +} + +export interface DayLogQuery { + window: TimeWindow; + minLevel?: number; + tool?: string; + pid?: number; + /** Case-insensitive substring of the whole record. */ + grep?: string; + /** The newest N matching records. */ + limit?: number; +} + +export interface DayLogEntry { + time: string; + level: number; + pid: number | null; + tool: string | null; + msg: string; + record: Record; +} + +export function parseDayLogLine(line: string): DayLogEntry | null { + if (!line.startsWith("{")) { + return null; + } + + let parsed: unknown; + try { + parsed = SafeJSON.parse(line, { strict: true }); + } catch (error) { + logger.debug({ error }, "day log: a line did not parse"); + return null; + } + + if (typeof parsed !== "object" || parsed === null) { + return null; + } + + const record = parsed as Record; + if (typeof record.time !== "string" || typeof record.level !== "number") { + return null; + } + + return { + time: record.time, + level: record.level, + pid: typeof record.pid === "number" ? record.pid : null, + tool: typeof record.tool === "string" ? record.tool : null, + msg: typeof record.msg === "string" ? record.msg : "", + record, + }; +} + +export function matchesDayLog(entry: DayLogEntry, query: DayLogQuery, line: string): boolean { + if (!inWindow(entry.time, query.window)) { + return false; + } + + if (query.minLevel !== undefined && entry.level < query.minLevel) { + return false; + } + + if (query.tool && entry.tool !== query.tool) { + return false; + } + + if (query.pid !== undefined && entry.pid !== query.pid) { + return false; + } + + return !query.grep || line.toLowerCase().includes(query.grep.toLowerCase()); +} + +/** + * The cheapest fixed string every wanted line must hold, so rg reads the day log (often 200 MB or more) instead + * of a JSON parse per line. Null when no filter narrows it. + */ +export function prefilterFor(query: DayLogQuery): { pattern: string; fixed: boolean } | null { + if (query.pid !== undefined) { + return { pattern: `"pid":${query.pid},`, fixed: true }; + } + + if (query.tool) { + return { pattern: `"tool":${SafeJSON.stringify(query.tool)}`, fixed: true }; + } + + if (query.minLevel !== undefined && query.minLevel >= LEVELS.warn) { + const levels = Object.values(LEVELS).filter((level) => level >= (query.minLevel ?? 0)); + return { pattern: `"level":(${levels.join("|")})[,}]`, fixed: false }; + } + + if (query.grep) { + return { pattern: query.grep, fixed: true }; + } + + return null; +} + +async function candidateLines(files: string[], query: DayLogQuery): Promise { + const prefilter = prefilterFor(query); + const rg = ripgrepBinary(); + + if (!prefilter || !rg) { + if (prefilter) { + logger.debug("day log: rg is not on PATH, reading every line"); + } + + return files.flatMap((file) => [...readLinesSync(file)]); + } + + const args = ["--no-config", "--no-filename", "--no-line-number", ...(prefilter.fixed ? ["-F", "-i"] : [])]; + const run = await ripgrepFiles({ rg, args: [...args, "-e", prefilter.pattern], files, timeoutMs: 120_000 }); + if (run.signal || run.code > 1) { + throw new Error(`rg over the day logs failed (${run.signal ?? `exit ${run.code}`}): ${run.stderr.trim()}`); + } + + return run.stdout.split("\n"); +} + +/** The newest `limit` matching records, oldest first. */ +export async function queryDayLog(query: DayLogQuery): Promise { + const days = new Set(daysIn(query.window)); + const files = listDayLogs() + .filter((file) => days.has(file.day)) + .sort((a, b) => a.day.localeCompare(b.day)) + .map((file) => file.path); + logger.debug({ files, prefilter: prefilterFor(query) }, "day log: query"); + + const found: DayLogEntry[] = []; + for (const line of await candidateLines(files, query)) { + const entry = parseDayLogLine(line); + if (entry && matchesDayLog(entry, query, line)) { + found.push(entry); + } + } + + found.sort((a, b) => a.time.localeCompare(b.time)); + return query.limit ? found.slice(-query.limit) : found; +} diff --git a/src/genesis-tools/lib/failures.ts b/src/genesis-tools/lib/failures.ts new file mode 100644 index 0000000000..8f14e7b708 --- /dev/null +++ b/src/genesis-tools/lib/failures.ts @@ -0,0 +1,205 @@ +import { type FailureKind, readFailures, type ToolFailure } from "@genesiscz/utils/cli/failures/record"; +import { formatLocalDate } from "@genesiscz/utils/date"; +import { daysIn, inWindow, type TimeWindow } from "./window"; + +export interface FailureFilter { + kinds?: FailureKind[]; + tools?: string[]; + /** Substring of the command path (`usage sessions`). */ + command?: string; + /** `claude-code`, `codex`, `grok`, `unknown` (a shell). */ + agent?: string; + /** Substring of the model, case-insensitive (`sonnet`, `opus-5`, `gpt-6`). */ + model?: string; + /** A session id or its leading part. */ + session?: string; + exitCodes?: number[]; + /** Substring of the message, the arguments or the stderr tail, case-insensitive. */ + grep?: string; + /** Interrupts (Ctrl-C, SIGTERM) are left out unless asked for. */ + interrupts?: boolean; +} + +export function matchesFailure(failure: ToolFailure, filter: FailureFilter): boolean { + if (filter.kinds?.length) { + if (!filter.kinds.includes(failure.kind)) { + return false; + } + } else if (!filter.interrupts && failure.kind === "interrupt") { + return false; + } + + if (filter.tools?.length && !filter.tools.includes(failure.tool)) { + return false; + } + + if (filter.command && !(failure.command ?? "").includes(filter.command)) { + return false; + } + + if (filter.agent && (failure.caller?.agent ?? "unknown") !== filter.agent) { + return false; + } + + if (filter.model && !(failure.caller?.model ?? "").toLowerCase().includes(filter.model.toLowerCase())) { + return false; + } + + if (filter.session && !(failure.caller?.sessionId ?? "").startsWith(filter.session)) { + return false; + } + + if (filter.exitCodes?.length && !filter.exitCodes.includes(failure.exitCode ?? -1)) { + return false; + } + + if (filter.grep) { + const needle = filter.grep.toLowerCase(); + const haystack = [failure.message ?? "", failure.argv.join(" "), failure.stderrTail ?? ""].join("\n"); + if (!haystack.toLowerCase().includes(needle)) { + return false; + } + } + + return true; +} + +/** The failures in the window that pass the filter, oldest first. */ +export function loadFailures({ + window, + filter = {}, + dir, +}: { + window: TimeWindow; + filter?: FailureFilter; + dir?: string; +}): ToolFailure[] { + const days = daysIn(window); + const found: ToolFailure[] = []; + + for (const failure of readFailures({ fromDay: days[0], toDay: days.at(-1) ?? days[0], dir })) { + if (inWindow(failure.at, window) && matchesFailure(failure, filter)) { + found.push(failure); + } + } + + return found; +} + +export const GROUP_KEYS = ["tool", "kind", "command", "model", "agent", "message", "exit", "day", "session"] as const; +export type GroupKey = (typeof GROUP_KEYS)[number]; + +export function isGroupKey(value: string): value is GroupKey { + return GROUP_KEYS.some((key) => key === value); +} + +function keyOf(failure: ToolFailure, by: GroupKey): string { + switch (by) { + case "tool": + return failure.tool; + case "kind": + return failure.kind; + case "command": + return failure.command ?? failure.tool; + case "model": + return failure.caller?.model ?? "(none)"; + case "agent": + return failure.caller?.agent ?? "unknown"; + case "message": + return failure.message ?? "(no message)"; + case "exit": + return String(failure.exitCode ?? "-"); + case "day": + return formatLocalDate(new Date(failure.at)); + case "session": + return failure.caller?.sessionId ?? "(none)"; + } +} + +export interface FailureGroup { + key: string[]; + count: number; + lastAt: string; + lastId: string; + models: string[]; + sessions: number; +} + +/** Counts by the given keys, most frequent first. */ +export function groupFailures(failures: ToolFailure[], by: GroupKey[]): FailureGroup[] { + const groups = new Map }>(); + + for (const failure of failures) { + const key = by.map((name) => keyOf(failure, name)); + const id = key.join("\u0000"); + const group = groups.get(id) ?? { + key, + count: 0, + lastAt: failure.at, + lastId: failure.id, + models: [], + sessions: 0, + sessionIds: new Set(), + }; + group.count++; + if (failure.at >= group.lastAt) { + group.lastAt = failure.at; + group.lastId = failure.id; + } + + const model = failure.caller?.model; + if (model && !group.models.includes(model)) { + group.models.push(model); + } + + if (failure.caller?.sessionId) { + group.sessionIds.add(failure.caller.sessionId); + } + + groups.set(id, group); + } + + return [...groups.values()] + .map(({ sessionIds, ...group }) => ({ ...group, sessions: sessionIds.size })) + .sort((a, b) => b.count - a.count || b.lastAt.localeCompare(a.lastAt)); +} + +export interface UsageGap { + tool: string; + command: string; + message: string; + count: number; + models: string[]; + /** Up to three distinct command lines that hit it, newest first. */ + examples: string[]; +} + +/** + * The command lines tools refused, grouped by what they refused: each row is a place where the help, an alias or a + * clearer error would have saved a retry. Agents' misuses count the same as a person's. + */ +export function usageGaps(failures: ToolFailure[]): UsageGap[] { + const usage = failures.filter((failure) => failure.kind === "usage"); + const gaps = new Map(); + + for (const failure of [...usage].reverse()) { + const command = failure.command ?? failure.tool; + const message = failure.message ?? "(no message)"; + const id = `${command}\u0000${message}`; + const gap = gaps.get(id) ?? { tool: failure.tool, command, message, count: 0, models: [], examples: [] }; + gap.count++; + const model = failure.caller?.model; + if (model && !gap.models.includes(model)) { + gap.models.push(model); + } + + const example = `tools ${failure.tool} ${failure.argv.join(" ")}`.trim(); + if (gap.examples.length < 3 && !gap.examples.includes(example)) { + gap.examples.push(example); + } + + gaps.set(id, gap); + } + + return [...gaps.values()].sort((a, b) => b.count - a.count); +} diff --git a/src/genesis-tools/lib/logs.test.ts b/src/genesis-tools/lib/logs.test.ts new file mode 100644 index 0000000000..37383883ce --- /dev/null +++ b/src/genesis-tools/lib/logs.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "bun:test"; +import type { FailureCaller, ToolFailure } from "@genesiscz/utils/cli/failures/record"; +import { matchesDayLog, parseDayLogLine, prefilterFor } from "./day-log"; +import { groupFailures, matchesFailure, usageGaps } from "./failures"; +import { parseProfileLine } from "./profile-log"; +import { daysIn, resolveWindow } from "./window"; + +const NOW = new Date(2026, 9, 22, 16, 0); + +function caller(model: string | null, sessionId: string | null): FailureCaller { + return { + agent: sessionId ? "claude-code" : "unknown", + sessionId, + sessionTitle: null, + model, + effort: null, + aiAgent: null, + project: "app", + repoRoot: "/repo", + branch: "main", + commitSha: null, + worktreePath: null, + }; +} + +function failure(overrides: Partial): ToolFailure { + return { + v: 1, + id: "x", + at: NOW.toISOString(), + kind: "usage", + tool: "git", + command: "git base", + argv: ["base", "--repo", "."], + exitCode: 1, + message: "error: unknown option '--repo'", + errorName: null, + stack: null, + stderrTail: null, + fatal: true, + durationMs: 100, + pid: 1, + ppid: 0, + cwd: "/repo", + traceId: null, + toolsRoot: "/repo", + caller: caller("claude-sonnet-5-5", "s-111"), + ...overrides, + }; +} + +describe("window", () => { + it("reads relative and clock times and lists every day it touches", () => { + const window = resolveWindow({ since: "2d", now: NOW }); + expect(window.from).toEqual(new Date(2026, 9, 20, 16, 0)); + expect(daysIn(window)).toEqual(["2026-10-20", "2026-10-21", "2026-10-22"]); + expect(resolveWindow({ since: "09:30", now: NOW }).from).toEqual(new Date(2026, 9, 22, 9, 30)); + expect(() => resolveWindow({ since: "soon", now: NOW })).toThrow("--since soon is not a time"); + }); +}); + +describe("failure filters and groups", () => { + const rows = [ + failure({ id: "a" }), + failure({ id: "b", caller: caller("claude-haiku-5-5", "s-222") }), + failure({ id: "c", kind: "interrupt", exitCode: 130, message: null }), + failure({ + id: "d", + kind: "error", + tool: "ts", + command: "ts skeleton", + argv: ["skeleton", "src"], + message: "boom", + caller: caller(null, null), + }), + ]; + + it("leaves interrupts out unless asked and matches model by substring", () => { + expect(rows.filter((row) => matchesFailure(row, {})).map((row) => row.id)).toEqual(["a", "b", "d"]); + expect(rows.filter((row) => matchesFailure(row, { interrupts: true })).length).toBe(4); + expect(rows.filter((row) => matchesFailure(row, { model: "HAIKU" })).map((row) => row.id)).toEqual(["b"]); + expect(rows.filter((row) => matchesFailure(row, { agent: "unknown" })).map((row) => row.id)).toEqual(["d"]); + expect(rows.filter((row) => matchesFailure(row, { grep: "--repo" })).map((row) => row.id)).toEqual(["a", "b"]); + }); + + it("counts groups and the sessions and models in each", () => { + const groups = groupFailures(rows.slice(0, 2).concat(rows[3]), ["tool", "kind"]); + expect(groups.map((group) => [group.key, group.count, group.sessions, group.models])).toEqual([ + [["git", "usage"], 2, 2, ["claude-sonnet-5-5", "claude-haiku-5-5"]], + [["ts", "error"], 1, 0, []], + ]); + }); + + it("groups refused command lines by what was refused, with examples", () => { + expect(usageGaps(rows)).toEqual([ + { + tool: "git", + command: "git base", + message: "error: unknown option '--repo'", + count: 2, + models: ["claude-haiku-5-5", "claude-sonnet-5-5"], + examples: ["tools git base --repo ."], + }, + ]); + }); +}); + +describe("day log", () => { + const line = '{"level":50,"time":"2026-10-22T13:00:00.000Z","pid":42,"tool":"ai","msg":"[cmux] command failed"}'; + const window = { from: new Date("2026-10-22T12:00:00Z"), to: new Date("2026-10-22T14:00:00Z") }; + + it("parses a pino line and filters by level, tool, pid and text", () => { + const entry = parseDayLogLine(line); + expect(entry && [entry.level, entry.pid, entry.tool, entry.msg]).toEqual([ + 50, + 42, + "ai", + "[cmux] command failed", + ]); + if (!entry) { + return; + } + + expect(matchesDayLog(entry, { window, minLevel: 40, tool: "ai", pid: 42, grep: "CMUX" }, line)).toBe(true); + expect(matchesDayLog(entry, { window, minLevel: 60 }, line)).toBe(false); + expect(matchesDayLog(entry, { window: { from: window.to, to: window.to } }, line)).toBe(false); + expect(parseDayLogLine("not json")).toBeNull(); + }); + + it("picks the narrowest rg prefilter, and none for a plain window", () => { + expect(prefilterFor({ window, pid: 42, tool: "ai" })).toEqual({ pattern: '"pid":42,', fixed: true }); + expect(prefilterFor({ window, minLevel: 50 })).toEqual({ pattern: '"level":(50|60)[,}]', fixed: false }); + expect(prefilterFor({ window, minLevel: 30 })).toBeNull(); + }); +}); + +describe("profile log", () => { + it("reads milliseconds, seconds and a trace id, and skips a summary line", () => { + expect( + parseProfileLine("[profile:cli] claude info exit=0 cpu=254ms rss=78MB caller=shell 160.03ms pid=16615", "d") + ).toEqual({ + day: "d", + scope: "cli", + label: "claude info exit=0 cpu=254ms rss=78MB caller=shell", + ms: 160.03, + traceId: null, + pid: 16615, + }); + expect(parseProfileLine("[profile:agent-sessions] walk 1.5s trace=2e100b3a pid=9", "d")?.ms).toBe(1500); + expect(parseProfileLine("── cli ──", "d")).toBeNull(); + }); +}); diff --git a/src/genesis-tools/lib/profile-log.ts b/src/genesis-tools/lib/profile-log.ts new file mode 100644 index 0000000000..f800a21691 --- /dev/null +++ b/src/genesis-tools/lib/profile-log.ts @@ -0,0 +1,73 @@ +import { readLinesSync } from "@genesiscz/utils/fs/read-lines"; +import { listDayLogs } from "@genesiscz/utils/logger/day-logs"; + +export interface ProfileEntry { + day: string; + scope: string; + /** The label with its details, `hub widget watch exit=0 cpu=… caller=app`. */ + label: string; + ms: number; + traceId: string | null; + pid: number; +} + +const LINE = /^\[profile:([\w.-]+)\] (.*) (\d+(?:\.\d+)?)(ms|s)(?: trace=(\S+))? pid=(\d+)$/; + +/** `[profile:]