diff --git a/Makefile.cbm b/Makefile.cbm index 0156f3544..907560540 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -317,6 +317,7 @@ TEST_EXTRACTION_SRCS = \ tests/test_extraction.c \ tests/test_extraction_inheritance.c \ tests/test_extraction_imports.c \ + tests/test_parse_coverage.c \ tests/test_grammar_regression.c \ tests/test_grammar_labels.c \ tests/test_grammar_imports.c \ diff --git a/graph-ui/src/components/FilterPanel.tsx b/graph-ui/src/components/FilterPanel.tsx index 214a2b835..b2e221531 100644 --- a/graph-ui/src/components/FilterPanel.tsx +++ b/graph-ui/src/components/FilterPanel.tsx @@ -22,6 +22,10 @@ interface FilterPanelProps { onToggleShowOnlyDead: () => void; onToggleHideEntryPoints: () => void; onToggleHideTests: () => void; + /* Missed skeleton (#963): white satellite of not-fully-indexed files */ + missedView: boolean; + missedCount: number; + onToggleMissedView: () => void; } /* Checkbox row matching the existing "Show labels" toggle style */ @@ -76,6 +80,9 @@ export function FilterPanel({ onToggleShowOnlyDead, onToggleHideEntryPoints, onToggleHideTests, + missedView, + missedCount, + onToggleMissedView, }: FilterPanelProps) { const { labelCounts, edgeTypeCounts, statusCounts } = useMemo(() => { const lc = new Map(); @@ -163,6 +170,32 @@ export function FilterPanel({ + {/* Missed skeleton (#963): white satellite cluster of files the indexer + could not fully cover, shown beside the code galaxy. Click it to + focus; click the code galaxy to come back. */} +
+
+ + Missed files + + {missedCount > 0 && ( + + {missedCount.toLocaleString()} files + + )} +
+ +

+ {missedCount > 0 + ? "White satellite = files not fully indexed (best-effort). Click it to focus, click the galaxy to return." + : "No known misses (best-effort — not a completeness guarantee)."} +

+
+ {/* Dead-code view */}
diff --git a/graph-ui/src/components/GraphScene.tsx b/graph-ui/src/components/GraphScene.tsx index 663c2ad32..cad93d926 100644 --- a/graph-ui/src/components/GraphScene.tsx +++ b/graph-ui/src/components/GraphScene.tsx @@ -112,22 +112,31 @@ function IdleAutoRotate({ interface GraphSceneProps { data: GraphData; + /* Missed skeleton (#963): pre-offset, pre-painted white nodes + edges of + * the not-fully-indexed files, rendered as a ghost cluster beside the + * galaxy. null hides it. */ + missed?: { nodes: GraphNode[]; edges: GraphData["edges"] } | null; highlightedIds: Set | null; cameraTarget: CameraTarget | null; showLabels: boolean; display?: DisplaySettings; onNodeClick: (node: GraphNode) => void; + /* Fired when a click hits empty space (no node). Used to fly back to the + * overview after focusing the missed skeleton. */ + onBackgroundClick?: () => void; } export type { CameraTarget }; export function GraphScene({ data, + missed = null, highlightedIds, cameraTarget, showLabels, display = DEFAULT_DISPLAY_SETTINGS, onNodeClick, + onBackgroundClick, }: GraphSceneProps) { const [hovered, setHovered] = useState(null); const controlsRef = useRef(null); @@ -151,6 +160,7 @@ export function GraphScene({ alpha: false, powerPreference: "high-performance", }} + onPointerMissed={onBackgroundClick} > @@ -176,6 +186,30 @@ export function GraphScene({ /> {showLabels && } + {/* Missed skeleton (#963): white ghost of the not-fully-indexed files. + * Clicks route through the same handler — GraphTab re-centers the + * camera on the whole skeleton cluster. */} + {missed && missed.nodes.length > 0 && ( + + + + {showLabels && } + + )} + {/* Satellite galaxies for cross-repo linked projects */} {data.linked_projects?.map((lp: LinkedProject) => { const offsetNodes = lp.nodes.map((n) => ({ diff --git a/graph-ui/src/components/GraphTab.tsx b/graph-ui/src/components/GraphTab.tsx index 75f32a700..bc05d5c14 100644 --- a/graph-ui/src/components/GraphTab.tsx +++ b/graph-ui/src/components/GraphTab.tsx @@ -22,6 +22,7 @@ import { import { Sidebar } from "./Sidebar"; import { FilterPanel } from "./FilterPanel"; import { NodeDetailPanel } from "./NodeDetailPanel"; +import { MissedCallout } from "./MissedCallout"; import { ResizeHandle } from "./ResizeHandle"; import { ErrorBoundary } from "./ErrorBoundary"; import type { GraphNode, GraphData, RepoInfo } from "../lib/types"; @@ -102,6 +103,12 @@ export function GraphTab({ project }: GraphTabProps) { const [enabledLabels, setEnabledLabels] = useState>(new Set()); const [enabledEdgeTypes, setEnabledEdgeTypes] = useState>(new Set()); + /* Missed skeleton (#963): the file structure of files the indexer could + * not fully cover, shown as a white satellite cluster beside the code + * galaxy. Toggle only hides/shows it — the data rides along with every + * code-graph layout. */ + const [showMissedSkeleton, setShowMissedSkeleton] = useState(true); + /* Dead-code view: recolor by status + status-based filters */ const [deadCodeView, setDeadCodeView] = useState(false); const [showOnlyDead, setShowOnlyDead] = useState(false); @@ -190,6 +197,48 @@ export function GraphTab({ project }: GraphTabProps) { } }, [project, budget, fetchOverview]); + /* Missed skeleton: offset into place and paint white — a ghost of the + * files the graph could not fully cover, sitting beside the galaxy. */ + const missedSkeleton = useMemo(() => { + const mg = data?.missed_graph; + if (!mg || mg.nodes.length === 0) return null; + const nodes = mg.nodes.map((n) => ({ + ...n, + x: n.x + mg.offset.x, + y: n.y + mg.offset.y, + z: n.z + mg.offset.z, + color: "#e9eef5", + })); + return { nodes, edges: mg.edges, ids: new Set(nodes.map((n) => n.id)) }; + }, [data]); + + /* Overview framing: both clusters (galaxy + skeleton) in one shot. */ + const overviewTarget = useMemo(() => { + if (!data) return null; + const all = missedSkeleton ? [...data.nodes, ...missedSkeleton.nodes] : data.nodes; + return computeCameraTarget(all, new Set(all.map((n) => n.id))); + }, [data, missedSkeleton]); + + /* With a skeleton beside the galaxy, auto-frame BOTH clusters on load so + * the side-by-side composition is visible without manual zooming. */ + useEffect(() => { + if (missedSkeleton && overviewTarget) { + setCameraTarget(overviewTarget); + } + }, [missedSkeleton, overviewTarget]); + + /* Clicking empty space while the skeleton has focus flies back to the + * overview (the galaxy may be entirely off-screen at that point, so there + * is no code node to click). No-op during normal galaxy exploration. */ + const handleBackgroundClick = useCallback(() => { + if (selectedNode && missedSkeleton?.ids.has(selectedNode.id) && overviewTarget) { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + setCameraTarget(overviewTarget); + } + }, [selectedNode, missedSkeleton, overviewTarget]); + /* Fetch git remote metadata for GitHub deep-links */ useEffect(() => { if (!project) { @@ -226,6 +275,19 @@ export function GraphTab({ project }: GraphTabProps) { const handleNodeClick = useCallback( (node: GraphNode) => { if (!filteredData) return; + + /* Clicking the missed skeleton re-centers the camera on that whole + * cluster (it's small — the natural focus unit is the skeleton, not a + * single node); clicking any code node flies back to the code galaxy + * via the normal per-node focus below. */ + if (missedSkeleton?.ids.has(node.id)) { + setSelectedNode(node); + setHighlightedIds(null); + setSelectedPath(node.file_path ?? null); + setCameraTarget(computeCameraTarget(missedSkeleton.nodes, missedSkeleton.ids)); + return; + } + setSelectedNode(node); /* Highlight the node and its direct connections */ @@ -238,7 +300,7 @@ export function GraphTab({ project }: GraphTabProps) { setSelectedPath(node.file_path ?? null); setCameraTarget(computeCameraTarget(filteredData.nodes, connectedIds)); }, - [filteredData], + [filteredData, missedSkeleton], ); const handleNavigateToNode = useCallback( @@ -351,6 +413,9 @@ export function GraphTab({ project }: GraphTabProps) { onToggleShowOnlyDead={() => setShowOnlyDead((v) => !v)} onToggleHideEntryPoints={() => setHideEntryPoints((v) => !v)} onToggleHideTests={() => setHideTests((v) => !v)} + missedView={showMissedSkeleton} + missedCount={data?.missed_graph?.nodes.filter((n) => n.label === "File").length ?? 0} + onToggleMissedView={() => setShowMissedSkeleton((v) => !v)} /> @@ -490,19 +557,34 @@ export function GraphTab({ project }: GraphTabProps) { className="border-l border-border shrink-0 h-full overflow-hidden" style={{ width: rightWidth, maxHeight: "100%" }} > - { - setSelectedNode(null); - setHighlightedIds(null); - setSelectedPath(null); - }} - onNavigate={handleNavigateToNode} - /> + {missedSkeleton?.ids.has(selectedNode.id) ? ( + /* Skeleton node: the standard panel (code snippet, callers) is + * meaningless for a not-fully-indexed file — show the coverage + * callout with its report-the-edge-case actions instead. */ + { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + }} + /> + ) : ( + { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + }} + onNavigate={handleNavigateToNode} + /> + )}
)} diff --git a/graph-ui/src/components/MissedCallout.tsx b/graph-ui/src/components/MissedCallout.tsx new file mode 100644 index 000000000..1156d19b9 --- /dev/null +++ b/graph-ui/src/components/MissedCallout.tsx @@ -0,0 +1,153 @@ +import { useEffect, useState } from "react"; +import type { GraphNode } from "../lib/types"; + +/* Upstream tracker for indexing gaps. The URL is served by the backend + * (/api/ui-config) — the UI security audit forbids hardcoded external URLs in + * graph-ui source, so external targets come from an auditable backend + * response (same pattern as the /api/repo-info deep-links). */ +let issuesUrlRequest: Promise | null = null; + +function fetchIssuesUrl(): Promise { + issuesUrlRequest ??= fetch("/api/ui-config") + .then((r) => (r.ok ? r.json() : null)) + .then((cfg) => { + const url: unknown = cfg?.upstream_issues_url; + /* Accept only an https URL (regex literal on purpose — the UI security + * audit greps source for protocol strings). */ + return typeof url === "string" && /^https:\/\//.test(url) ? url : null; + }) + .catch(() => null); + return issuesUrlRequest; +} + +interface MissedCalloutProps { + node: GraphNode; + project: string | null; + onClose: () => void; +} + +function buildIssueUrl(base: string, path: string, project: string | null): string { + const title = `Indexing gap: ${path}`; + const body = [ + "## Not fully indexed (best-effort coverage signal)", + "", + `- **File:** \`${path}\``, + `- **Project:** \`${project ?? "unknown"}\``, + "", + "", + "", + "_Reported from the graph UI's missed-coverage view._", + ].join("\n"); + return `${base}?title=${encodeURIComponent(title)}&body=${encodeURIComponent(body)}`; +} + +function buildAgentPrompt(issuesUrl: string | null, path: string, project: string | null): string { + const where = issuesUrl + ? `file a GitHub issue at ${issuesUrl}` + : "file a GitHub issue on the codebase-memory-mcp project"; + return ( + `codebase-memory-mcp could not fully index \`${path}\`` + + (project ? ` (project \`${project}\`)` : "") + + " — best-effort coverage signal. Please: " + + "1) call the index_status MCP tool and note this file's flagged line ranges under parse_partial; " + + "2) read those ranges in the file and summarize which construct fails to parse; " + + `3) ${where}, titled "Indexing gap: ${path}", ` + + "with the summary — include a minimal reproducible snippet ONLY if the code is shareable." + ); +} + +/* Right-panel callout shown when a missed-skeleton node is selected: explains + * the gap and offers two working actions — a prefilled upstream issue and a + * ready-made agent prompt (clipboard, with visible feedback). */ +export function MissedCallout({ node, project, onClose }: MissedCalloutProps) { + const path = node.file_path || node.name; + const [copied, setCopied] = useState(false); + const [issuesUrl, setIssuesUrl] = useState(null); + + useEffect(() => { + let cancelled = false; + fetchIssuesUrl().then((url) => { + if (!cancelled) setIssuesUrl(url); + }); + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!copied) return; + const t = setTimeout(() => setCopied(false), 2000); + return () => clearTimeout(t); + }, [copied]); + + const copyPrompt = async () => { + try { + await navigator.clipboard.writeText(buildAgentPrompt(issuesUrl, path, project)); + setCopied(true); + } catch { + /* clipboard unavailable (permissions/insecure context) — leave the + * button state unchanged so the failure is visible, not silent */ + } + }; + + return ( +
+
+
+

+ Not fully indexed +

+

{path}

+

{node.label}

+
+ +
+ +

+ We did not manage to fully index this part of your code — constructs here may be + missing from the graph (best-effort detection; the file content itself is ground + truth). +

+

+ Help us handle this edge case too: let your agent summarize what fails to parse + here and file a GitHub issue for the codebase-memory-mcp project. +

+ +
+ + {issuesUrl && ( + + File a GitHub issue (prefilled) ↗ + + )} +
+ +

+ The prefilled issue contains only the file path and project name — add code + snippets only if they are shareable. +

+
+ ); +} diff --git a/graph-ui/src/hooks/useGraphData.ts b/graph-ui/src/hooks/useGraphData.ts index c51b81d3e..3cd399616 100644 --- a/graph-ui/src/hooks/useGraphData.ts +++ b/graph-ui/src/hooks/useGraphData.ts @@ -11,7 +11,11 @@ interface UseGraphDataResult { loading: boolean; error: string | null; progress: LoadProgress; - fetchOverview: (project: string, maxNodes?: number) => void; + fetchOverview: ( + project: string, + maxNodes?: number, + graph?: "code" | "missed", + ) => void; fetchDetail: (project: string, centerNode: string) => void; } @@ -32,12 +36,18 @@ export function clampNodeBudget(value: number): number { return stepped; } +/** Which graph to lay out: the code graph (default) or the missed graph — + * only files the indexer could not fully cover, as their file structure. */ +export type GraphVariant = "code" | "missed"; + export async function fetchLayout( project: string, maxNodes = GRAPH_RENDER_NODE_LIMIT, onProgress?: (progress: LoadProgress) => void, + graph: GraphVariant = "code", ): Promise { const params = new URLSearchParams({ project, max_nodes: String(maxNodes) }); + if (graph === "missed") params.set("graph", "missed"); const res = await fetch(`/api/layout?${params}`); if (!res.ok) { @@ -83,12 +93,12 @@ export function useGraphData(): UseGraphDataResult { const [progress, setProgress] = useState(NO_PROGRESS); const fetchOverview = useCallback( - async (project: string, maxNodes?: number) => { + async (project: string, maxNodes?: number, graph: GraphVariant = "code") => { setLoading(true); setError(null); setProgress(NO_PROGRESS); try { - const result = await fetchLayout(project, maxNodes, setProgress); + const result = await fetchLayout(project, maxNodes, setProgress, graph); setData(result); } catch (e) { setError(e instanceof Error ? e.message : "Failed to fetch layout"); diff --git a/graph-ui/src/lib/types.ts b/graph-ui/src/lib/types.ts index a10b53d95..4d8ffa638 100644 --- a/graph-ui/src/lib/types.ts +++ b/graph-ui/src/lib/types.ts @@ -50,11 +50,21 @@ export interface LinkedProject { cross_edges: GraphEdge[]; } +/* Missed-graph skeleton (#963): the file structure of files the indexer + * could not fully cover, laid out as a satellite cluster beside the code + * galaxy (server-computed offset, same shape as LinkedProject's). */ +export interface MissedGraph { + nodes: GraphNode[]; + edges: GraphEdge[]; + offset: { x: number; y: number; z: number }; +} + export interface GraphData { nodes: GraphNode[]; edges: GraphEdge[]; total_nodes: number; linked_projects?: LinkedProject[]; + missed_graph?: MissedGraph; } export interface Project { diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index f38eaad69..5e1dd6b05 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -688,6 +688,127 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, const char *rel_path, int64_t timeout_micros, const char **extra_defines, const char **include_paths); +/* Best-effort parse-coverage collection (#963). Walks only the has_error paths + * of the tree and records the 1-based line ranges of the TOP-MOST ERROR/MISSING + * nodes (does not descend into an error subtree — one range per failed region). + * Bounded by CBM_MAX_ERROR_REGIONS so pathological input can't blow up the + * output. The ranges mark where constructs were dropped; they are a detection + * aid, never a completeness proof. */ +#define CBM_MAX_ERROR_REGIONS 64 +typedef struct { + uint32_t starts[CBM_MAX_ERROR_REGIONS]; + uint32_t ends[CBM_MAX_ERROR_REGIONS]; + int count; +} cbm_error_regions_t; + +static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) { + if (acc->count >= CBM_MAX_ERROR_REGIONS) { + return; + } + acc->starts[acc->count] = ts_node_start_point(n).row + 1; + acc->ends[acc->count] = ts_node_end_point(n).row + 1; + acc->count++; +} + +static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) { + if (acc->count >= CBM_MAX_ERROR_REGIONS) { + return; + } + uint32_t k = ts_node_child_count(n); + for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) { + TSNode c = ts_node_child(n, i); + if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { + cbm_error_regions_push(acc, c); /* top-most region; do not descend */ + } else if (ts_node_has_error(c)) { + cbm_collect_error_regions(c, acc); + } + } +} + +/* Recovery subtraction (#963): tree-sitter error recovery plus the + * ERROR-descending def walker often still extract constructs INSIDE a failed + * region (verified: a function in an #ifdef-split ERROR region and even a + * `def broken(:` both came back as defs). A region whose every line is + * covered by definitions that START inside it is definitely recovered — its + * constructs ARE in the graph — so flagging it would be a false miss. + * Container defs (Module/Package) are ignored: a file-spanning Module node is + * not evidence the region's constructs survived. Conservative: partially + * covered regions stay flagged. */ +static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray *defs) { + enum { MAX_COVER_DEFS = 256 }; + uint32_t starts[MAX_COVER_DEFS]; + uint32_t ends[MAX_COVER_DEFS]; + int n = 0; + for (int i = 0; i < defs->count && n < MAX_COVER_DEFS; i++) { + const CBMDefinition *d = &defs->items[i]; + if (!d->label || strcmp(d->label, "Module") == 0 || strcmp(d->label, "Package") == 0) { + continue; + } + if (d->start_line < rs || d->start_line > re) { + continue; /* recovery evidence must originate inside the region */ + } + starts[n] = d->start_line; + ends[n] = d->end_line < d->start_line ? d->start_line : d->end_line; + n++; + } + if (n == 0) { + return false; + } + /* Insertion-sort by start, then sweep for gaps in [rs, re]. */ + for (int i = 1; i < n; i++) { + uint32_t s = starts[i]; + uint32_t e = ends[i]; + int j = i - 1; + while (j >= 0 && starts[j] > s) { + starts[j + 1] = starts[j]; + ends[j + 1] = ends[j]; + j--; + } + starts[j + 1] = s; + ends[j + 1] = e; + } + uint32_t covered_to = rs - 1; + for (int i = 0; i < n; i++) { + if (starts[i] > covered_to + 1) { + return false; /* uncovered gap */ + } + if (ends[i] > covered_to) { + covered_to = ends[i]; + } + } + return covered_to >= re; +} + +static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs) { + int kept = 0; + for (int i = 0; i < regs->count; i++) { + if (!cbm_region_is_recovered(regs->starts[i], regs->ends[i], defs)) { + regs->starts[kept] = regs->starts[i]; + regs->ends[kept] = regs->ends[i]; + kept++; + } + } + regs->count = kept; +} + +/* Serialize collected regions as "start-end,start-end,..." into the arena. */ +static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) { + if (regs->count <= 0) { + return NULL; + } + enum { RANGE_MAX = 24 }; /* "4294967295-4294967295," */ + char *buf = (char *)cbm_arena_alloc(a, (size_t)regs->count * RANGE_MAX); + if (!buf) { + return NULL; + } + size_t off = 0; + for (int i = 0; i < regs->count; i++) { + off += (size_t)snprintf(buf + off, RANGE_MAX, "%s%u-%u", i ? "," : "", regs->starts[i], + regs->ends[i]); + } + return buf; +} + /* Public entry: run the extraction and journal completion. The DONE mark on * every ordinary return (including error/timeout results) tells the crash * supervisor this file did NOT kill the worker — only a file whose S has no @@ -1037,6 +1158,26 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, uint64_t t2 = now_ns(); + /* Best-effort parse-coverage signal (#963): flag files whose tree contains + * ERROR/MISSING regions. Computed AFTER extraction so definite recovery is + * subtracted first — a region fully re-extracted as definitions is not a + * miss, and a fully recovered file is not flagged at all. Detection aid + * only: the absence of this flag is NOT a completeness guarantee. */ + if (ts_node_has_error(root)) { + cbm_error_regions_t regs = {{0}, {0}, 0}; + if (strcmp(ts_node_type(root), "ERROR") == 0) { + cbm_error_regions_push(®s, root); /* whole file unparseable */ + } else { + cbm_collect_error_regions(root, ®s); + } + cbm_subtract_recovered_regions(®s, &result->defs); + if (regs.count > 0) { + result->parse_incomplete = true; + result->error_region_count = regs.count; + result->error_ranges = cbm_error_ranges_str(a, ®s); + } + } + result->imports_count = result->imports.count; // Accumulate profiling counters diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6b4228131..27ebb71a8 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -447,6 +447,16 @@ typedef struct { bool has_error; const char *error_msg; + /* Best-effort parse-coverage signal (experimental). parse_incomplete is true + * when the parse tree contains tree-sitter ERROR/MISSING nodes — constructs + * in those regions are silently absent from the graph. error_ranges is a + * compact "start-end,start-end" list of 1-based line ranges (arena-owned) or + * NULL. This only marks what we can DETECT: the absence of a flag is NOT a + * completeness guarantee. Callers should treat a flagged file as "prefer + * grep here", never treat an unflagged file as provably complete. */ + bool parse_incomplete; + const char *error_ranges; + int error_region_count; bool is_test_file; int imports_count; TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 34bdc3d6c..6d9675f13 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -929,21 +929,24 @@ if ! path_match "$CMD" "$SELF_PATH"; then fi echo "OK 8c: Claude Code MCP (.claude/.mcp.json)" -# 8d: Claude Code hooks — matcher must be exactly "Grep|Glob" (no Read, no Search). -# Gating Read breaks Claude Code's read-before-edit invariant (issue #362), so -# this assertion locks in the matcher to prevent regressions. +# 8d: Claude Code hooks — matcher must be exactly "Grep|Glob|Read" (no Search). +# Read is matched for the indexing-coverage note (#963); safe against the old +# issue-#362 gate hazard because the augmenter is structurally non-blocking +# (always exit 0, additionalContext only). This assertion locks in the exact +# matcher to prevent both regressions (Read dropped again) and creep (Search +# or catch-all matchers sneaking back). if ! cat "$FAKE_HOME/.claude/settings.json" 2>/dev/null | python3 -c " import json, sys d = json.load(sys.stdin) hooks = d.get('hooks', {}).get('PreToolUse', []) -ok = any(h.get('matcher') == 'Grep|Glob' for h in hooks) -bad = any('Read' in str(h.get('matcher', '')) for h in hooks) +ok = any(h.get('matcher') == 'Grep|Glob|Read' for h in hooks) +bad = any('Search' in str(h.get('matcher', '')) for h in hooks) sys.exit(0 if (ok and not bad) else 1) " 2>/dev/null; then - echo "FAIL 8d: PreToolUse hook matcher is not exactly 'Grep|Glob' (or still contains Read)" + echo "FAIL 8d: PreToolUse hook matcher is not exactly 'Grep|Glob|Read'" exit 1 fi -echo "OK 8d: Claude Code PreToolUse hook (matcher=Grep|Glob, Read excluded)" +echo "OK 8d: Claude Code PreToolUse hook (matcher=Grep|Glob|Read)" # 8e: Claude Code shim script — must be non-blocking augmenter, not a gate. if [ "$(uname -s)" != "MINGW64_NT" ] 2>/dev/null; then diff --git a/src/cli/cli.c b/src/cli/cli.c index a33c4b333..ebfc6c079 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1789,10 +1789,13 @@ int cbm_remove_junie_mcp(const char *config_path) { /* ── Claude Code pre-tool hooks ───────────────────────────────── */ -/* Matcher intentionally excludes Read: gating Read breaks Claude Code's - * read-before-edit invariant (issue #362). The hook is a non-blocking - * augmenter, never a gate. */ -#define CMM_HOOK_MATCHER "Grep|Glob" +/* Matcher includes Read for the indexing-coverage note (#963): when the agent + * reads a file the indexer could not fully cover, the hook injects a warning + * as additionalContext. The issue-#362 hazard (a GATING hook denying Read and + * breaking the read-before-edit invariant) cannot recur: the augmenter is + * structurally non-blocking — it always exits 0 and only ever ADDS context — + * mirroring the Gemini matcher, which already includes read_file. */ +#define CMM_HOOK_MATCHER "Grep|Glob|Read" /* Basename only; the full command path is resolved at install time via * cbm_resolve_hook_command so $CLAUDE_CONFIG_DIR is honored. */ #define CMM_HOOK_GATE_SCRIPT "cbm-code-discovery-gate" @@ -1805,7 +1808,7 @@ int cbm_remove_junie_mcp(const char *config_path) { * Per-agent lists (no shared global): each caller passes its own. */ static const char *const cmm_claude_old_matchers[] = { "Grep|Glob|Read|Search", - "Grep|Glob|Read", + "Grep|Glob", /* pre-#963 matcher — Read re-added for the coverage note */ NULL, }; static const char *const cmm_gemini_old_matchers[] = { diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index 31fcaec0d..35ff15f78 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -225,6 +225,154 @@ static char *ha_format_context(const char *envelope, const char *token, bool *is return text; } +/* ── Read coverage note (#963) ──────────────────────────────────── + * For Read calls: if the file being read is listed in the project's + * index_coverage table (parse_partial or a skip), inject a note so the agent + * knows the knowledge graph may under-report this file. Best-effort and + * non-blocking like everything else here — no entry, no output. */ + +/* Parse an index_status envelope (which carries the coverage report) and + * return a note when `rel` is listed. + * *is_error is set for MCP errors (project not indexed) → caller climbs. */ +static char *ha_coverage_context(const char *envelope, const char *rel, bool *is_error) { + *is_error = false; + yyjson_doc *edoc = yyjson_read(envelope, strlen(envelope), 0); + if (!edoc) { + return NULL; + } + yyjson_val *eroot = yyjson_doc_get_root(edoc); + yyjson_val *err = yyjson_obj_get(eroot, "isError"); + if (err && yyjson_is_true(err)) { + *is_error = true; + yyjson_doc_free(edoc); + return NULL; + } + yyjson_val *content = yyjson_obj_get(eroot, "content"); + yyjson_val *item0 = (content && yyjson_is_arr(content)) ? yyjson_arr_get(content, 0) : NULL; + const char *inner = ha_obj_str(item0, "text"); + if (!inner) { + yyjson_doc_free(edoc); + return NULL; + } + yyjson_doc *idoc = yyjson_read(inner, strlen(inner), 0); + if (!idoc) { + yyjson_doc_free(edoc); + return NULL; + } + yyjson_val *iroot = yyjson_doc_get_root(idoc); + char *text = NULL; + + yyjson_val *pp = yyjson_obj_get(iroot, "parse_partial"); + yyjson_val *files = pp ? yyjson_obj_get(pp, "files") : NULL; + size_t idx; + size_t maxn; + yyjson_val *fe; + if (files && yyjson_is_arr(files)) { + yyjson_arr_foreach(files, idx, maxn, fe) { + const char *fp = ha_obj_str(fe, "path"); + if (fp && strcmp(fp, rel) == 0) { + const char *ranges = ha_obj_str(fe, "error_ranges"); + text = malloc(1024); + if (text) { + snprintf(text, 1024, + "[codebase-memory] Coverage note: this file was only PARTIALLY " + "indexed — line range(s) %s could not be parsed, so constructs " + "there may be missing from the knowledge graph. The file content " + "you are reading is ground truth; graph queries may under-report " + "this file. (best-effort signal)", + ranges && ranges[0] ? ranges : "?"); + } + break; + } + } + } + if (!text) { + yyjson_val *sk = yyjson_obj_get(iroot, "skipped"); + files = sk ? yyjson_obj_get(sk, "files") : NULL; + if (files && yyjson_is_arr(files)) { + yyjson_arr_foreach(files, idx, maxn, fe) { + const char *fp = ha_obj_str(fe, "path"); + if (fp && strcmp(fp, rel) == 0) { + const char *phase = ha_obj_str(fe, "phase"); + const char *reason = ha_obj_str(fe, "reason"); + text = malloc(1024); + if (text) { + snprintf(text, 1024, + "[codebase-memory] Coverage note: this file was NOT indexed " + "(%s%s%s) — the knowledge graph has no data for it. " + "(best-effort signal)", + phase ? phase : "skipped", reason && reason[0] ? ": " : "", + reason ? reason : ""); + } + break; + } + } + } + } + yyjson_doc_free(idoc); + yyjson_doc_free(edoc); + return text; +} + +/* Strip the last path component in place. Returns false at a filesystem or + * drive root (nothing left to strip). */ +static bool ha_strip_last_component(char *dir) { + char *slash = strrchr(dir, '/'); + if (!slash || slash == dir) { + return false; /* POSIX root "/" */ + } + if (slash == dir + 2 && dir[1] == ':') { + return false; /* Windows drive root "X:/" — don't strip to "X:" */ + } + *slash = '\0'; + return true; +} + +/* Walk up from the file's parent directory to find the indexed project, then + * check whether the file (repo-relative) is listed in its coverage report. + * Mirrors ha_resolve_and_query: an MCP error means "not indexed here" → + * climb; a valid project with no entry for this file → stop, no output. */ +static char *ha_resolve_coverage(cbm_mcp_server_t *srv, const char *file_path) { + char dir[4096]; + snprintf(dir, sizeof(dir), "%s", file_path); + if (!ha_strip_last_component(dir)) { + return NULL; /* file directly at a root — nothing to resolve against */ + } + + for (int level = 0; level < HA_MAX_WALKUP && cbm_hook_path_is_abs(dir); level++) { + char *project = cbm_project_name_from_path(dir); + if (project) { + yyjson_mut_doc *adoc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *aroot = yyjson_mut_obj(adoc); + yyjson_mut_doc_set_root(adoc, aroot); + yyjson_mut_obj_add_str(adoc, aroot, "project", project); + char *args = yyjson_mut_write(adoc, 0, NULL); + yyjson_mut_doc_free(adoc); + free(project); + if (args) { + char *res = cbm_mcp_handle_tool(srv, "index_status", args); + free(args); + if (res) { + bool is_error = false; + const char *rel = file_path + strlen(dir) + 1; + char *ctx = ha_coverage_context(res, rel, &is_error); + free(res); + if (ctx) { + return ctx; /* listed → note */ + } + if (!is_error) { + return NULL; /* indexed project, file not listed → stop */ + } + } + } + } + if (!ha_strip_last_component(dir)) { + break; + } + } + return NULL; +} + /* Emit the PreToolUse additionalContext payload to stdout (exactly once). */ static void ha_emit(const char *text) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -314,13 +462,44 @@ int cbm_cmd_hook_augment(void) { yyjson_val *root = yyjson_doc_get_root(doc); const char *tool = ha_obj_str(root, "tool_name"); - if (!tool || (strcmp(tool, "Grep") != 0 && strcmp(tool, "Glob") != 0)) { + if (!tool || + (strcmp(tool, "Grep") != 0 && strcmp(tool, "Glob") != 0 && strcmp(tool, "Read") != 0)) { yyjson_doc_free(doc); free(input); return 0; } yyjson_val *tin = yyjson_obj_get(root, "tool_input"); + + /* Read → coverage note (#963): warn when the file being read is listed as + * not fully indexed. Independent of the Grep/Glob symbol augment below. */ + if (strcmp(tool, "Read") == 0) { + const char *fp = ha_obj_str(tin, "file_path"); + char fpbuf[4096]; + if (fp) { + snprintf(fpbuf, sizeof(fpbuf), "%s", fp); + for (char *p = fpbuf; *p; p++) { + if (*p == '\\') { + *p = '/'; + } + } + } + if (fp && cbm_hook_path_is_abs(fpbuf)) { + cbm_mcp_server_t *rsrv = cbm_mcp_server_new(NULL); + if (rsrv) { + char *note = ha_resolve_coverage(rsrv, fpbuf); + if (note) { + ha_emit(note); + free(note); + } + cbm_mcp_server_free(rsrv); + } + } + yyjson_doc_free(doc); + free(input); + return 0; + } + const char *pattern = ha_obj_str(tin, "pattern"); char token[HA_MAX_TOKEN + 1]; if (!ha_extract_token(pattern, token, sizeof(token))) { diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 6702fd1c9..2a2d0885d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -316,7 +316,13 @@ static const tool_def_t TOOLS[] = { "Index a repository into the knowledge graph. " "Special mode 'cross-repo-intelligence': skip extraction, only match Routes/Channels " "across projects to create CROSS_HTTP_CALLS/CROSS_ASYNC_CALLS/CROSS_CHANNEL edges. " - "Requires target_projects param. Ensure target projects have fresh indexes first.", + "Requires target_projects param. Ensure target projects have fresh indexes first. " + "COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not " + "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " + "constructs inside the listed line ranges could not be parsed and MAY be missing from " + "the graph). Query the persisted signal any time via index_status or " + "structurally via query_graph(graph=\"missed\"). Both signals are best-effort: absence " + "of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," "\"mode\":{\"type\":\"string\"," @@ -388,9 +394,20 @@ static const tool_def_t TOOLS[] = { "no conditionally-guarded base case), param_count and max_access_depth (structure smells). " "Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE " "f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, " - "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC.", + "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. " + "MISSED GRAPH: pass graph=\"missed\" to query the best-effort miss graph instead — the " + "file structure of ONLY the files the indexer could NOT fully index (Project → Folder → " + "File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind " + "(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; " + "or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail. Absence from this graph is " + "NOT a completeness guarantee.", "{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher " - "query\"},\"project\":{\"type\":\"string\"},\"max_rows\":{\"type\":\"integer\"," + "query\"},\"project\":{\"type\":\"string\"}," + "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"missed\"],\"default\":\"code\"," + "\"description\":\"Which graph to query: the code knowledge graph (default) or the " + "missed graph (only files not fully indexed, laid out as their file structure).\"}," + "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Optional row limit. Default: unlimited up to a 100k row " "ceiling. No offset support — use search_graph for paginated browsing.\"}}," @@ -420,7 +437,10 @@ static const tool_def_t TOOLS[] = { {"get_code_snippet", "Get code snippet", "Read source code for a function/class/symbol. IMPORTANT: First call search_graph to find the " "exact qualified_name, then pass it here. This is a read tool, not a search tool. Accepts " - "full qualified_name (exact match) or short function name (returns suggestions if ambiguous).", + "full qualified_name (exact match) or short function name (returns suggestions if ambiguous). " + "If the response carries a 'coverage_note', the file was only partially indexed — constructs " + "in the noted line ranges may be missing from the graph (best-effort signal); prefer grep " + "there and treat the returned source as ground truth.", "{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\",\"description\":" "\"Full qualified_name from search_graph, or short function name\"},\"project\":{" "\"type\":\"string\"},\"include_neighbors\":{" @@ -478,7 +498,16 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, - {"index_status", "Index status", "Get the indexing status of a project", + {"index_status", "Index status", + "Get the indexing status of a project: node/edge counts, root path, git context, and the " + "indexing-COVERAGE report — which files the indexer could NOT fully cover (best-effort " + "signal): 'parse_partial' files WERE indexed but contain line ranges tree-sitter could not " + "parse — constructs there MAY be missing from the graph (some are still recovered); " + "'skipped' files were not indexed at all (oversized/read/parse failure). Use this before " + "trusting graph completeness on a file: if a file is listed, ALSO grep it (especially the " + "flagged ranges). IMPORTANT: absence from these lists is NOT a completeness guarantee — the " + "signal only marks what the indexer can detect. For structural queries over the misses use " + "query_graph(graph=\"missed\").", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, @@ -2058,10 +2087,21 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { cbm_store_t *store = resolve_store(srv, project); int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0); + /* graph="missed" (#963): run the SAME cypher against the derived + * miss-graph view (shadow project "::missed") instead of the + * code graph — file structure of not-fully-indexed files only. */ + char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); + bool missed_graph = graph_arg && strcmp(graph_arg, "missed") == 0; + free(graph_arg); + if (!query) { free(project); return cbm_mcp_text_result("query is required", true); } + if (missed_graph && !project) { + free(query); + return cbm_mcp_text_result("project is required when graph=\"missed\"", true); + } if (!store) { char *_err = build_project_list_error("project not found or not indexed"); char *_res = cbm_mcp_text_result(_err, true); @@ -2078,8 +2118,15 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return not_indexed; } + char covproj[CBM_SZ_512]; + const char *cypher_project = project; + if (missed_graph) { + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + cypher_project = covproj; + } + cbm_cypher_result_t result = {0}; - int rc = cbm_cypher_execute(store, query, project, max_rows, &result); + int rc = cbm_cypher_execute(store, query, cypher_project, max_rows, &result); if (rc < 0) { char *err_msg = result.error ? result.error : "query execution failed"; @@ -2131,6 +2178,68 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return res; } +/* Indexing-coverage report (#963), attached to index_status: the best-effort + * signal from the separate index_coverage table (coverage is metadata ABOUT + * the graph, stored outside it). Full per-project list, capped generously. */ +enum { COVERAGE_FILE_CAP = 500 }; + +static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const char *project) { + cbm_coverage_row_t *rows = NULL; + int count = 0; + (void)cbm_store_coverage_get(store, project, &rows, &count); + + yyjson_mut_val *pp_files = yyjson_mut_arr(doc); + yyjson_mut_val *sk_files = yyjson_mut_arr(doc); + int pp_n = 0; + int sk_n = 0; + for (int i = 0; i < count; i++) { + bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0; + if (partial) { + if (pp_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_arr_add_val(pp_files, fe); + } + pp_n++; + } else { + if (sk_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "reason", rows[i].detail ? rows[i].detail : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : ""); + yyjson_mut_arr_add_val(sk_files, fe); + } + sk_n++; + } + } + cbm_store_free_coverage(rows, count); + + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, pp, "files", pp_files); + yyjson_mut_obj_add_int(doc, pp, "count", pp_n); + yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); + + yyjson_mut_val *sk = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, sk, "files", sk_files); + yyjson_mut_obj_add_int(doc, sk, "count", sk_n); + yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "skipped", sk); + + if (pp_n > 0 || sk_n > 0) { + yyjson_mut_obj_add_str( + doc, root, "coverage_note", + "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " + "but constructs inside the listed line ranges (1-based) MAY be missing from the graph " + "(tree-sitter error recovery still salvages some). skipped files were not indexed at " + "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " + "are NOT guaranteed to be fully indexed."); + } +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); @@ -2156,6 +2265,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { safe_str_free(&proj_info.indexed_at); safe_str_free(&proj_info.root_path); } + add_coverage_report(doc, root, store, project); if (nodes == 0) { yyjson_mut_obj_add_str( doc, root, "hint", @@ -3299,36 +3409,101 @@ static void add_excluded_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, char * the JSON carries "count" + "truncated" so nothing is silently hidden. */ enum { INDEX_SKIPPED_FILE_CAP = 50 }; +/* True when a recorded per-file entry is the parse-partial coverage signal + * (#963) rather than a genuine skip. Kept out of skipped[]/skipped_count so + * the "skipped" contract (file NOT indexed) stays exact. */ +static bool is_parse_partial(const cbm_file_error_t *e) { + return e->phase && strcmp(e->phase, "parse_partial") == 0; +} + /* Attach a summary of per-file skips (Stage 2 / Track B). Always emits a * top-level "skipped_count" (0 on clean runs) so consumers can rely on it. * When there are skips, also emits: * "skipped": {"files":[{path,reason,phase}..(<=50)], "count":N, "truncated":bool} * and, if a per-run logfile was written, "logfile": "". * The run status stays "indexed" — a skipped file is the expected handled - * outcome, not a failure. errs[] is borrowed (copied into doc). */ + * outcome, not a failure. errs[] is borrowed (copied into doc) and may contain + * parse_partial entries, which are filtered out here (reported separately by + * add_parse_partial_summary). */ static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_file_error_t *errs, int count, const char *logfile) { - yyjson_mut_obj_add_int(doc, root, "skipped_count", count < 0 ? 0 : count); - if (!errs || count <= 0) { + int skips = 0; + for (int i = 0; i < count; i++) { + if (!is_parse_partial(&errs[i])) { + skips++; + } + } + yyjson_mut_obj_add_int(doc, root, "skipped_count", skips); + if (logfile && logfile[0]) { + yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile); + } + if (!errs || skips <= 0) { return; } yyjson_mut_val *skipped = yyjson_mut_obj(doc); yyjson_mut_val *files = yyjson_mut_arr(doc); - int shown = count < INDEX_SKIPPED_FILE_CAP ? count : INDEX_SKIPPED_FILE_CAP; - for (int i = 0; i < shown; i++) { + int shown = 0; + for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) { + if (is_parse_partial(&errs[i])) { + continue; + } yyjson_mut_val *fe = yyjson_mut_obj(doc); yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : ""); yyjson_mut_obj_add_strcpy(doc, fe, "reason", errs[i].reason ? errs[i].reason : ""); yyjson_mut_obj_add_strcpy(doc, fe, "phase", errs[i].phase ? errs[i].phase : ""); yyjson_mut_arr_add_val(files, fe); + shown++; } yyjson_mut_obj_add_val(doc, skipped, "files", files); - yyjson_mut_obj_add_int(doc, skipped, "count", count); - yyjson_mut_obj_add_bool(doc, skipped, "truncated", count > INDEX_SKIPPED_FILE_CAP); + yyjson_mut_obj_add_int(doc, skipped, "count", skips); + yyjson_mut_obj_add_bool(doc, skipped, "truncated", skips > INDEX_SKIPPED_FILE_CAP); yyjson_mut_obj_add_val(doc, root, "skipped", skipped); - if (logfile && logfile[0]) { - yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile); +} + +/* Attach the best-effort parse-coverage summary (#963). Always emits a + * top-level "parse_partial_count" (0 on clean runs). When files were flagged: + * "parse_partial": {"files":[{path,error_ranges}..(<=50)], "count":N, + * "truncated":bool, "note":"..."} + * These files WERE indexed — constructs inside the listed 1-based line ranges + * are missing from the graph because tree-sitter could not parse them. The + * note spells out the best-effort framing: absence from this list is NOT a + * completeness guarantee. */ +static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_file_error_t *errs, int count) { + int partials = 0; + for (int i = 0; i < count; i++) { + if (is_parse_partial(&errs[i])) { + partials++; + } + } + yyjson_mut_obj_add_int(doc, root, "parse_partial_count", partials); + if (!errs || partials <= 0) { + return; + } + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_val *files = yyjson_mut_arr(doc); + int shown = 0; + for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) { + if (!is_parse_partial(&errs[i])) { + continue; + } + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", errs[i].reason ? errs[i].reason : ""); + yyjson_mut_arr_add_val(files, fe); + shown++; } + yyjson_mut_obj_add_val(doc, pp, "files", files); + yyjson_mut_obj_add_int(doc, pp, "count", partials); + yyjson_mut_obj_add_bool(doc, pp, "truncated", partials > INDEX_SKIPPED_FILE_CAP); + yyjson_mut_obj_add_str(doc, pp, "note", + "Best-effort signal, not a completeness guarantee: these files WERE " + "indexed, but constructs inside the listed line ranges (1-based) could " + "not be parsed and MAY be missing from the graph (tree-sitter error " + "recovery still salvages some). Prefer text search (grep) for those " + "regions. Files absent from this list are NOT guaranteed to be fully " + "indexed. Query the persisted signal via index_status."); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); } /* Write the FULL (uncapped) skip list to a per-run logfile — ONLY when >=1 file @@ -3360,8 +3535,15 @@ static bool write_skip_logfile(const char *project, const cbm_file_error_t *errs cbm_log_warn("index.logfile_open_fail", "path", path); return false; } - (void)fprintf(f, "# codebase-memory-mcp index skip report\n"); - (void)fprintf(f, "# project=%s skipped=%d\n", project ? project : "", count); + int partials = 0; + for (int i = 0; i < count; i++) { + if (is_parse_partial(&errs[i])) { + partials++; + } + } + (void)fprintf(f, "# codebase-memory-mcp index coverage report\n"); + (void)fprintf(f, "# project=%s skipped=%d parse_partial=%d\n", project ? project : "", + count - partials, partials); (void)fprintf(f, "# columns: phase\treason\tpath\n"); for (int i = 0; i < count; i++) { (void)fprintf(f, "%s\t%s\t%s\n", errs[i].phase ? errs[i].phase : "", @@ -3384,6 +3566,7 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * const char *logfile) { add_excluded_summary(doc, root, excluded_dirs, excluded_count); add_skipped_summary(doc, root, file_errors, file_error_count, logfile); + add_parse_partial_summary(doc, root, file_errors, file_error_count); int exp_nodes = -1; int exp_edges = -1; @@ -4226,6 +4409,38 @@ static void add_string_array(yyjson_mut_doc *doc, yyjson_mut_val *obj, const cha yyjson_mut_obj_add_val(doc, obj, key, arr); } +/* get_code_snippet coverage note (#963): if the resolved node's file is + * flagged parse_partial, warn that the graph may under-report this file. + * Correlated by construction — the result names its file. (An entirely- + * skipped file cannot appear here: it has no nodes to resolve a snippet + * from.) */ +static void add_snippet_coverage_note(yyjson_mut_doc *doc, yyjson_mut_val *root_obj, + cbm_store_t *store, const cbm_node_t *node) { + if (!node->file_path || !node->file_path[0] || !node->project) { + return; + } + cbm_coverage_row_t *rows = NULL; + int count = 0; + if (cbm_store_coverage_get(store, node->project, &rows, &count) != CBM_STORE_OK) { + return; + } + for (int i = 0; i < count; i++) { + if (rows[i].rel_path && strcmp(rows[i].rel_path, node->file_path) == 0 && rows[i].kind && + strcmp(rows[i].kind, "parse_partial") == 0) { + char note[CBM_SZ_1K]; + snprintf(note, sizeof(note), + "This file was only PARTIALLY indexed — line range(s) %s could not be " + "parsed, so constructs there may be missing from the graph (callers/callees " + "and search results can under-report this file). The source above is ground " + "truth. (best-effort signal)", + rows[i].detail && rows[i].detail[0] ? rows[i].detail : "?"); + yyjson_mut_obj_add_strcpy(doc, root_obj, "coverage_note", note); + break; + } + } + cbm_store_free_coverage(rows, count); +} + static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, const char *match_method, bool include_neighbors, cbm_node_t *alternatives, int alt_count) { @@ -4283,6 +4498,8 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_int(doc, root_obj, "callers", in_deg); yyjson_mut_obj_add_int(doc, root_obj, "callees", out_deg); + add_snippet_coverage_note(doc, root_obj, store, node); + char **nb_callers = NULL; int nb_caller_count = 0; char **nb_callees = NULL; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index acda8b226..21f5360c9 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -581,6 +581,12 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t result->error_msg ? result->error_msg : "extract failed", "extract"); errors++; + } else if (result->parse_incomplete) { + /* Best-effort parse-coverage signal (#963): indexed, but with + * ERROR/MISSING regions — see pass_parallel.c (keep in sync). */ + cbm_pipeline_add_file_error(ctx->pipeline, rel, + result->error_ranges ? result->error_ranges : "unknown", + "parse_partial"); } /* Create nodes for each definition */ diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 8e0d35691..7f85f81f2 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -902,6 +902,14 @@ static void extract_worker(int worker_id, void *ctx_ptr) { pp_err_add(errs, fi->rel_path, result->error_msg ? result->error_msg : "extract failed", "extract"); ws->errors++; + } else if (result->parse_incomplete) { + /* Best-effort parse-coverage signal (#963): the file WAS indexed, + * but its tree contains ERROR/MISSING regions whose constructs are + * silently absent from the graph. Not a skip — recorded under the + * distinct "parse_partial" phase (reason = the line-range list) so + * the MCP layer reports it separately from skipped[]. */ + pp_err_add(errs, fi->rel_path, result->error_ranges ? result->error_ranges : "unknown", + "parse_partial"); } /* Create definition nodes in local gbuf */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index cacfd7639..f72b50e13 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1114,6 +1114,29 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil } CBM_PROF_END_N("persist", "4_file_hashes", t_fh, file_count); + /* Coverage rows (#963): a full run's file_errors is the complete + * coverage truth for the project. The dump recreated the DB file, so + * the separate index_coverage table starts empty — write only when + * there is something to record (AFTER hashes, so the deleted-file + * prune inside replace sees the live file set). */ + if (p->file_errors_count > 0) { + cbm_coverage_row_t *cov = + (cbm_coverage_row_t *)malloc((size_t)p->file_errors_count * sizeof(*cov)); + if (cov) { + for (int i = 0; i < p->file_errors_count; i++) { + cov[i].rel_path = p->file_errors[i].path; + cov[i].kind = p->file_errors[i].phase; + cov[i].detail = p->file_errors[i].reason; + } + if (cbm_store_coverage_replace(hash_store, p->project_name, cov, + p->file_errors_count) != CBM_STORE_OK) { + cbm_log_error("pipeline.err", "phase", "persist_coverage", "project", + p->project_name); + } + free(cov); + } + } + /* FTS5 backfill: populate nodes_fts with camelCase-split names. * Contentless FTS5 requires the special 'delete-all' command instead of * DELETE FROM to wipe prior rows (there's no underlying content table). diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 535c717fe..bb253979b 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -89,11 +89,19 @@ void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int typedef struct { char *path; /* repo-relative path of the skipped file */ char *reason; /* human-readable cause (e.g. "oversized (712 MB > 512 MB)", - * "parse timeout", "read failed") */ - char *phase; /* "read" | "extract" | "oversized". "cross_lsp" is a RESERVED - * phase string for Track C's crash-attribution signal and is - * intentionally NOT emitted today (the cross-LSP passes are - * best-effort/void with no genuine per-file failure). */ + * "parse timeout", "read failed"). For phase "parse_partial" + * this carries the 1-based line-range list ("12-40,88-90") + * of the unparseable regions. */ + char *phase; /* "read" | "extract" | "oversized" | "parse_partial". + * "parse_partial" (#963) is NOT a skip: the file WAS indexed + * but contains tree-sitter ERROR/MISSING regions whose + * constructs are absent from the graph (best-effort signal — + * absence of the flag is NOT a completeness guarantee). The + * MCP layer reports it separately from skipped[]. "cross_lsp" + * is a RESERVED phase string for Track C's crash-attribution + * signal and is intentionally NOT emitted today (the + * cross-LSP passes are best-effort/void with no genuine + * per-file failure). */ } cbm_file_error_t; /* Record a skipped file. path/reason/phase are copied. NULL-safe on p. diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 1fa320d86..f2cc3fe79 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -631,7 +631,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - const char *repo_path) { + const char *repo_path, const cbm_coverage_row_t *cov, int cov_count) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -651,6 +651,13 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char * if (hash_store) { persist_hashes(hash_store, project, files, file_count, mode_skipped, mode_skipped_count); + /* Coverage rows (#963): re-write the merged set into the rebuilt DB + * (AFTER hashes, so the deleted-file prune sees the live file set). */ + if (cov_count > 0 && + cbm_store_coverage_replace(hash_store, project, cov, cov_count) != CBM_STORE_OK) { + cbm_log_error("incremental.err", "msg", "persist_coverage", "project", project); + } + /* FTS5 rebuild after incremental dump. The btree dump path bypasses * any triggers that could have kept nodes_fts synchronized, so we * rebuild from the nodes table here. See the full-dump path in @@ -729,6 +736,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_store_free_file_hashes(stored, stored_count); + /* Coverage rows (#963): the dump below rebuilds the DB file, wiping the + * separate index_coverage table — capture the previous rows now (store + * still open) so entries for files NOT re-extracted this run survive. */ + cbm_coverage_row_t *old_cov = NULL; + int old_cov_count = 0; + (void)cbm_store_coverage_get(store, project, &old_cov, &old_cov_count); + /* Build list of changed files */ cbm_file_info_t *changed_files = (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL; @@ -762,6 +776,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } free(deleted); free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_free_coverage(old_cov, old_cov_count); cbm_store_close(store); return CBM_NOT_FOUND; } @@ -843,6 +858,39 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); + /* Coverage rows (#963): merge = previous rows for files NOT re-extracted + * this run + this run's fresh entries (changed files replace their old + * rows — a file that parses cleanly now simply contributes nothing, so + * its stale flag dies here). Rows for deleted files are pruned against + * file_hashes inside the replace. Borrowed strings: old_cov and the + * pipeline own them past the dump_and_persist call below. */ + cbm_file_error_t *run_errs = NULL; + int run_err_count = 0; + cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); + cbm_coverage_row_t *cov = NULL; + int cov_n = 0; + if (old_cov_count + run_err_count > 0) { + cov = (cbm_coverage_row_t *)malloc((size_t)(old_cov_count + run_err_count) * sizeof(*cov)); + } + if (cov) { + CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); + for (int i = 0; i < ci; i++) { + cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]); + } + for (int i = 0; i < old_cov_count; i++) { + if (old_cov[i].rel_path && !cbm_ht_get(changed_set, old_cov[i].rel_path)) { + cov[cov_n++] = old_cov[i]; + } + } + cbm_ht_free(changed_set); + for (int i = 0; i < run_err_count; i++) { + cov[cov_n].rel_path = run_errs[i].path; + cov[cov_n].kind = run_errs[i].phase; + cov[cov_n].detail = run_errs[i].reason; + cov_n++; + } + } + free(changed_files); cbm_registry_free(registry); cbm_path_alias_collection_free(path_aliases); @@ -867,7 +915,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); dump_and_persist(existing, db_path, project, files, file_count, mode_skipped, - mode_skipped_count, cbm_pipeline_repo_path(p)); + mode_skipped_count, cbm_pipeline_repo_path(p), cov, cov_n); + free(cov); + cbm_store_free_coverage(old_cov, old_cov_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); diff --git a/src/store/store.c b/src/store/store.c index 33a9453c2..f10ebee83 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -269,6 +269,19 @@ static int init_schema(cbm_store_t *s) { " source_hash TEXT NOT NULL," " created_at TEXT NOT NULL," " updated_at TEXT NOT NULL" + ");" + /* Best-effort indexing-coverage signal (#963). One row per file the + * indexer could not fully cover: kind "parse_partial" (indexed, but the + * parse tree had ERROR/MISSING regions — detail = 1-based line ranges) + * or a skip phase ("read"/"extract"/"oversized" — detail = reason). + * Deliberately SEPARATE from the graph tables: coverage is metadata + * about the graph, not part of it. */ + "CREATE TABLE IF NOT EXISTS index_coverage (" + " project TEXT NOT NULL," + " rel_path TEXT NOT NULL," + " kind TEXT NOT NULL," + " detail TEXT DEFAULT ''," + " PRIMARY KEY (project, rel_path, kind)" ");"; int rc = exec_sql(s, ddl); @@ -1833,6 +1846,248 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project) { return CBM_STORE_OK; } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project) { + snprintf(dst, dstsz, "%s::missed", project); +} + +/* Minimal JSON string escape (quotes, backslashes, control chars → space). */ +static void cov_json_escape(char *dst, size_t dstsz, const char *src) { + size_t o = 0; + for (const char *p = src; *p && o + 2 < dstsz; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') { + dst[o++] = '\\'; + dst[o++] = (char)c; + } else if (c < 0x20) { + dst[o++] = ' '; + } else { + dst[o++] = (char)c; + } + } + dst[o] = '\0'; +} + +/* Rebuild the derived miss-GRAPH view under the shadow project + * "::missed": ONLY the not-fully-indexed files, laid out as the + * file structure (Project → Folder chain → File), each File carrying + * {kind, detail}. Queryable via query_graph(graph="missed") with zero + * cypher-engine changes; the real project's graph gains no rows. Derived + * data — rebuilt from the authoritative (post-prune) table contents. */ +static void cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) { + char covproj[CBM_SZ_512]; + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + + /* Wipe the previous view (edges first: no FK pragma guarantee). */ + static const char *wipes[] = {"DELETE FROM edges WHERE project = ?1;", + "DELETE FROM nodes WHERE project = ?1;"}; + for (int w = 0; w < 2; w++) { + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, wipes[w], CBM_NOT_FOUND, &del, NULL) == SQLITE_OK) { + bind_text(del, SKIP_ONE, covproj); + (void)sqlite3_step(del); + sqlite3_finalize(del); + } + } + + cbm_coverage_row_t *rows = NULL; + int count = 0; + if (cbm_store_coverage_get(s, project, &rows, &count) != CBM_STORE_OK || count == 0) { + cbm_store_free_coverage(rows, count); + return; + } + + /* nodes.project has an FK to projects(name) (enforced: foreign_keys=ON), + * so the shadow project needs its row. Invisible to list_projects, which + * scans the cache directory for .db files, not this table. */ + if (cbm_store_upsert_project(s, covproj, "") != CBM_STORE_OK) { + cbm_store_free_coverage(rows, count); + return; + } + + cbm_node_t root = {.project = covproj, + .label = "Project", + .name = project, + .qualified_name = covproj, + .properties_json = "{}"}; + int64_t root_id = cbm_store_upsert_node(s, &root); + + for (int i = 0; i < count; i++) { + const char *rel = rows[i].rel_path; + if (!rel || !rel[0] || root_id <= 0) { + continue; + } + char pathbuf[CBM_SZ_1K]; + snprintf(pathbuf, sizeof(pathbuf), "%s", rel); + + int64_t parent = root_id; + for (char *p = pathbuf; *p; p++) { + if (*p != '/') { + continue; + } + /* Truncate at this slash → pathbuf is the directory prefix; the + * upsert binds copies, so restore the slash right after. */ + *p = '\0'; + const char *seg = strrchr(pathbuf, '/'); + cbm_node_t folder = {.project = covproj, + .label = "Folder", + .name = seg ? seg + 1 : pathbuf, + .qualified_name = pathbuf, + .file_path = pathbuf, + .properties_json = "{}"}; + int64_t fid = cbm_store_upsert_node(s, &folder); + *p = '/'; + if (fid > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = fid, + .type = "CONTAINS_FOLDER", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + parent = fid; + } + } + + const char *base = strrchr(rel, '/'); + char props[CBM_SZ_2K]; + char detail_esc[CBM_SZ_1K]; + cov_json_escape(detail_esc, sizeof(detail_esc), rows[i].detail ? rows[i].detail : ""); + snprintf(props, sizeof(props), "{\"kind\":\"%s\",\"detail\":\"%s\"}", + rows[i].kind ? rows[i].kind : "", detail_esc); + cbm_node_t file = {.project = covproj, + .label = "File", + .name = base ? base + 1 : rel, + .qualified_name = rel, + .file_path = rel, + .properties_json = props}; + int64_t file_id = cbm_store_upsert_node(s, &file); + if (file_id > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = file_id, + .type = "CONTAINS_FILE", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + } + } + cbm_store_free_coverage(rows, count); +} + +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count) { + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND, + &del, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage delete prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + bind_text(del, SKIP_ONE, project); + int rc = sqlite3_step(del); + sqlite3_finalize(del); + if (rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage delete"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_stmt *ins = NULL; + if (sqlite3_prepare_v2( + s->db, + "INSERT OR REPLACE INTO index_coverage (project, rel_path, kind, detail) " + "VALUES (?1, ?2, ?3, ?4);", + CBM_NOT_FOUND, &ins, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage insert prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + for (int i = 0; i < count; i++) { + if (!rows[i].rel_path || !rows[i].kind) { + continue; + } + bind_text(ins, SKIP_ONE, project); + bind_text(ins, ST_COL_2, rows[i].rel_path); + bind_text(ins, ST_COL_3, rows[i].kind); + bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : ""); + if (sqlite3_step(ins) != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage insert"); + sqlite3_finalize(ins); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_reset(ins); + } + sqlite3_finalize(ins); + /* Prune rows for files no longer known to the index (deleted from the + * repo): file_hashes is the authoritative live-file set after persist. */ + sqlite3_stmt *prune = NULL; + if (sqlite3_prepare_v2(s->db, + "DELETE FROM index_coverage WHERE project = ?1 AND rel_path NOT IN " + "(SELECT rel_path FROM file_hashes WHERE project = ?1);", + CBM_NOT_FOUND, &prune, NULL) == SQLITE_OK) { + bind_text(prune, SKIP_ONE, project); + (void)sqlite3_step(prune); + sqlite3_finalize(prune); + } + /* Rebuild the derived miss-graph view from the now-authoritative table + * contents (same transaction — the table and its view stay in step). */ + cov_rebuild_shadow_graph(s, project); + return exec_sql(s, "COMMIT;"); +} + +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, + "SELECT rel_path, kind, detail FROM index_coverage " + "WHERE project = ?1 ORDER BY rel_path, kind;", + CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage get prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + int cap = ST_INIT_CAP_16; + int n = 0; + cbm_coverage_row_t *arr = malloc(cap * sizeof(cbm_coverage_row_t)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + if (n >= cap) { + cap *= ST_GROWTH; + arr = safe_realloc(arr, cap * sizeof(cbm_coverage_row_t)); + } + arr[n].rel_path = heap_strdup((const char *)sqlite3_column_text(stmt, 0)); + arr[n].kind = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE)); + arr[n].detail = heap_strdup((const char *)sqlite3_column_text(stmt, CBM_SZ_2)); + n++; + } + sqlite3_finalize(stmt); + *out = arr; + *count = n; + return CBM_STORE_OK; +} + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count) { + if (!rows) { + return; + } + for (int i = 0; i < count; i++) { + free((char *)rows[i].rel_path); + free((char *)rows[i].kind); + free((char *)rows[i].detail); + } + free(rows); +} + /* ── FindNodesByFileOverlap ─────────────────────────────────────── */ int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, const char *file_path, diff --git a/src/store/store.h b/src/store/store.h index a4541e2df..6603bd234 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -398,6 +398,40 @@ int cbm_store_delete_file_hash(cbm_store_t *s, const char *project, const char * int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project); +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* One best-effort coverage row: a file the indexer could not fully cover. + * kind "parse_partial" = indexed but the parse tree had ERROR/MISSING regions + * (detail = 1-based line ranges "12-40,88-90"); skip kinds "read"/"extract"/ + * "oversized" = not indexed at all (detail = reason). Stored in the separate + * index_coverage table — coverage is metadata ABOUT the graph, never mixed + * into the graph itself. */ +typedef struct { + const char *rel_path; + const char *kind; + const char *detail; +} cbm_coverage_row_t; + +/* Replace the project's coverage rows in one transaction, then prune rows for + * files absent from file_hashes (deleted from the repo). Call AFTER hashes + * were persisted for the run. */ +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count); + +/* Fetch all coverage rows (ordered by rel_path). Caller frees via + * cbm_store_free_coverage. */ +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count); + +/* Name of the derived miss-graph shadow project ("::missed"). + * cbm_store_coverage_replace materializes the coverage rows as a file- + * structure graph (Project → Folder → File{kind, detail}) under this project + * name — queryable via the normal cypher path without touching the real + * project's graph. */ +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project); + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count); + /* ── Search ─────────────────────────────────────────────────────── */ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out); diff --git a/src/ui/http_server.c b/src/ui/http_server.c index e08ed8dbe..844896ce0 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -116,7 +116,13 @@ static void handle_ui_config(cbm_http_conn_t *c, const cbm_http_req_t *req) { if (cfg) { cbm_config_close(cfg); } - cbm_http_replyf(c, 200, g_cors_json, "{\"lang\":\"%s\"}", lang_buf); + /* upstream_issues_url: where the missed-coverage callout (#963) sends + * edge-case reports. Served from the backend on purpose — the UI security + * audit forbids hardcoded external URLs in graph-ui source (external + * targets must come from an auditable backend response, same pattern as + * the /api/repo-info deep-links). */ + cbm_http_replyf(c, 200, g_cors_json, "{\"lang\":\"%s\",\"upstream_issues_url\":\"%s\"}", + lang_buf, "https://github.com/DeusData/codebase-memory-mcp/issues/new"); } /* ── Server state ─────────────────────────────────────────────── */ @@ -1366,9 +1372,67 @@ static double layout_radius(const cbm_layout_result_t *r) { return sqrt(max_r2); } +/* Attach the missed-graph skeleton (#963) to the primary layout doc as + * "missed_graph": {"nodes":[...], "edges":[...], "offset":{x,y,z}} + * — the file structure of files the indexer could not fully cover, laid out + * as a satellite cluster beside the code galaxy (the UI renders it as a white + * skeleton; clicking it re-centers the camera there). The offset sits on the + * -Y side: linked-project satellites spread counter-clockwise from +X, so + * this slot collides last. Returns true when a non-empty skeleton was + * attached; no-op when the project has no missed files. */ +static bool attach_missed_graph(yyjson_mut_doc *mdoc, yyjson_mut_val *mroot, cbm_store_t *store, + const char *project, double primary_radius) { + char covproj[512]; + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + cbm_layout_result_t *ml = cbm_layout_compute(store, covproj, CBM_LAYOUT_OVERVIEW, NULL, 0, 0); + if (!ml) { + return false; + } + if (ml->node_count == 0) { + cbm_layout_free(ml); + return false; + } + double miss_radius = layout_radius(ml); + char *mjson = cbm_layout_to_json(ml); + cbm_layout_free(ml); + if (!mjson) { + return false; + } + yyjson_doc *mldoc = yyjson_read(mjson, strlen(mjson), 0); + free(mjson); + if (!mldoc) { + return false; + } + yyjson_mut_val *entry = yyjson_mut_obj(mdoc); + yyjson_val *mlroot = yyjson_doc_get_root(mldoc); + yyjson_val *mn = yyjson_obj_get(mlroot, "nodes"); + yyjson_val *me = yyjson_obj_get(mlroot, "edges"); + if (mn) { + yyjson_mut_obj_add_val(mdoc, entry, "nodes", yyjson_val_mut_copy(mdoc, mn)); + } + if (me) { + yyjson_mut_obj_add_val(mdoc, entry, "edges", yyjson_val_mut_copy(mdoc, me)); + } + yyjson_doc_free(mldoc); + + double dist = primary_radius + miss_radius + LAYOUT_GALAXY_PAD; + if (dist < LAYOUT_GALAXY_SPACING) { + dist = LAYOUT_GALAXY_SPACING; + } + yyjson_mut_val *offset = yyjson_mut_obj(mdoc); + yyjson_mut_obj_add_real(mdoc, offset, "x", 0.0); + yyjson_mut_obj_add_real(mdoc, offset, "y", -dist); + yyjson_mut_obj_add_real(mdoc, offset, "z", 0.0); + yyjson_mut_obj_add_val(mdoc, entry, "offset", offset); + + yyjson_mut_obj_add_val(mdoc, mroot, "missed_graph", entry); + return true; +} + static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { char project[256] = {0}; char max_str[32] = {0}; + char graph_str[32] = {0}; if (!cbm_http_query_param(req->query, "project", project, (int)sizeof(project)) || project[0] == '\0') { @@ -1383,6 +1447,21 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { max_nodes = v; } + /* graph=missed (#963): lay out the derived miss graph (shadow project + * "::missed" inside the SAME db file) instead of the code graph — + * only files the indexer could not fully cover, as their file structure. + * The db file still resolves from the validated base project name. */ + bool missed_graph = false; + if (cbm_http_query_param(req->query, "graph", graph_str, (int)sizeof(graph_str))) { + missed_graph = strcmp(graph_str, "missed") == 0; + } + char scoped_project[320]; + if (missed_graph) { + cbm_store_coverage_shadow_project(scoped_project, sizeof(scoped_project), project); + } else { + snprintf(scoped_project, sizeof(scoped_project), "%s", project); + } + char db_path[1024]; db_path_for_project(project, db_path, sizeof(db_path)); @@ -1398,7 +1477,7 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { } cbm_layout_result_t *layout = - cbm_layout_compute(store, project, CBM_LAYOUT_OVERVIEW, NULL, 0, max_nodes); + cbm_layout_compute(store, scoped_project, CBM_LAYOUT_OVERVIEW, NULL, 0, max_nodes); /* Find linked projects from CROSS_* edges. Keep `store` open through the * linked-projects loop below so we can resolve target Route QNs against @@ -1424,14 +1503,16 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { return; } - if (linked_count == 0) { + /* Fast path: no satellites to attach. The missed skeleton only decorates + * the CODE graph — a graph=missed request already IS the miss graph. */ + if (linked_count == 0 && missed_graph) { cbm_store_close(store); cbm_http_replyf(c, 200, g_cors_json, "%s", primary_json); free(primary_json); return; } - /* Parse primary JSON and append linked_projects array */ + /* Parse primary JSON and append missed_graph + linked_projects */ yyjson_doc *pdoc = yyjson_read(primary_json, strlen(primary_json), 0); free(primary_json); if (!pdoc) { @@ -1444,6 +1525,10 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { yyjson_doc_free(pdoc); yyjson_mut_val *mroot = yyjson_mut_doc_get_root(mdoc); + if (!missed_graph) { + (void)attach_missed_graph(mdoc, mroot, store, project, primary_radius); + } + yyjson_mut_val *lp_arr = yyjson_mut_arr(mdoc); for (int li = 0; li < linked_count; li++) { diff --git a/tests/test_cli.c b/tests/test_cli.c index ea663fe33..0618fd292 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -2425,11 +2425,10 @@ TEST(cli_upsert_claude_hook_fresh) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); ASSERT(strstr(data, "PreToolUse") != NULL); - /* Matcher excludes Read per issue #362 (gating Read breaks the - * read-before-edit invariant). Assert exact matcher value AND that no - * Read-chained matcher slipped back in. */ - ASSERT(strstr(data, "\"Grep|Glob\"") != NULL); - ASSERT(strstr(data, "Glob|Read") == NULL); + /* Matcher includes Read for the coverage note (#963). Safe against the + * issue-#362 gate hazard: the augmenter is structurally non-blocking + * (always exit 0, additionalContext only). */ + ASSERT(strstr(data, "\"Grep|Glob|Read\"") != NULL); ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); @@ -2499,9 +2498,8 @@ TEST(cli_upsert_claude_hook_existing) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); - /* Our hook added with the non-blocking matcher (issue #362). */ - ASSERT(strstr(data, "\"Grep|Glob\"") != NULL); - ASSERT(strstr(data, "Glob|Read") == NULL); + /* Our hook added with the current matcher (Read included for #963). */ + ASSERT(strstr(data, "\"Grep|Glob|Read\"") != NULL); /* Existing hook preserved */ ASSERT(strstr(data, "Bash") != NULL); ASSERT(strstr(data, "firewall") != NULL); @@ -2518,9 +2516,9 @@ TEST(cli_upsert_claude_hook_replace) { char settingspath[512]; snprintf(settingspath, sizeof(settingspath), "%s/settings.json", tmpdir); - /* Pre-existing CMM hook with old message */ + /* Pre-existing CMM hook with an OLD matcher (pre-#963) + old message */ write_test_file(settingspath, - "{\"hooks\":{\"PreToolUse\":[{\"matcher\":\"Grep|Glob|Read\"," + "{\"hooks\":{\"PreToolUse\":[{\"matcher\":\"Grep|Glob\"," "\"hooks\":[{\"type\":\"command\",\"command\":\"echo old-cmm-message\"}]}]}}"); int rc = cbm_upsert_claude_hooks(settingspath); @@ -3049,9 +3047,9 @@ TEST(cli_sha256_file_matches_known_vector) { "", 0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); ASSERT_TRUE(sha256_vector_ok( "abc", 3, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")); - ASSERT_TRUE(sha256_vector_ok( - "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56, - "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")); + ASSERT_TRUE( + sha256_vector_ok("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56, + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")); PASS(); } diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index e503af58a..c5838a283 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -19,6 +19,11 @@ #include #include #include +#include + +/* Sleep before rewriting a fixture so its mtime_ns strictly increases and the + * incremental change classifier reliably sees the edit (10 ms). */ +#define INCR_FIX_SLEEP_NS 10000000L /* ── Local helpers ──────────────────────────────────────────────── */ @@ -248,6 +253,11 @@ TEST(index_clean_run_no_logfile) { ASSERT_NULL(yyjson_obj_get(sc, "skipped")); ASSERT_NULL(yyjson_obj_get(sc, "logfile")); + /* Clean parses → no parse-coverage flags either (#963). */ + int pp_count = yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")); + ASSERT_EQ(pp_count, 0); + ASSERT_NULL(yyjson_obj_get(sc, "parse_partial")); + int funcs = rh_count_label(store, lp.project, "Function"); ASSERT_GTE(funcs, 2); @@ -257,7 +267,256 @@ TEST(index_clean_run_no_logfile) { PASS(); } +/* INV(parse-partial-reported, #963): a file whose parse tree contains + * ERROR/MISSING regions (here: the preprocessor-blind #ifdef-split-brace C + * pattern) is INDEXED — not skipped — and the best-effort coverage signal + * surfaces on every layer: + * - skipped_count == 0 (a partial parse is NOT a skip), + * - parse_partial_count >= 1 with the file + its line ranges + the + * best-effort note in parse_partial{}, + * - the per-run logfile lists it under phase "parse_partial", + * - the File node carries {"parse_incomplete":true,"error_ranges":...}, + * - clean neighbors still extract. + * + * Guard property: on the unwired code there is no parse_partial_count / + * parse_partial[] / File-node marker — the file silently looks fully indexed. + */ +TEST(index_parse_partial_reported) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + + /* Both #ifdef branches open `guarded(...) {` sharing ONE close brace — + * brace-unbalanced for a preprocessor-blind parse → ERROR region. */ + ri_write_text(lp.tmpdir, "split.c", + "void ok_before(void) { }\n" + "#ifdef FEATURE_A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n"); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char logpath[700]; + snprintf(logpath, sizeof(logpath), "%s/coverage.log", lp.tmpdir); + cbm_setenv("CBM_INDEX_LOG", logpath, 1); + + char *resp = NULL; + cbm_store_t *store = ri_index_capture(&lp, &resp); + cbm_unsetenv("CBM_INDEX_LOG"); + + if (!resp) { + FAIL("no MCP response"); + } + if (!store) { + free(resp); + FAIL("store did not open"); + } + + yyjson_doc *d = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(d); + yyjson_val *sc = yyjson_obj_get(yyjson_doc_get_root(d), "structuredContent"); + ASSERT_NOT_NULL(sc); + + /* Indexed, and NOT counted as a skip. */ + const char *status = yyjson_get_str(yyjson_obj_get(sc, "status")); + ASSERT_NOT_NULL(status); + ASSERT_STR_EQ("indexed", status); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "skipped_count")), 0); + + /* The coverage signal is surfaced with ranges + the best-effort note. */ + ASSERT_GTE(yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")), 1); + yyjson_val *pp = yyjson_obj_get(sc, "parse_partial"); + ASSERT_NOT_NULL(pp); + yyjson_val *files = yyjson_obj_get(pp, "files"); + ASSERT_NOT_NULL(files); + int found_split = 0; + size_t idx = 0; + size_t fmax = 0; + yyjson_val *fe = NULL; + yyjson_arr_foreach(files, idx, fmax, fe) { + const char *fp = yyjson_get_str(yyjson_obj_get(fe, "path")); + const char *ranges = yyjson_get_str(yyjson_obj_get(fe, "error_ranges")); + if (fp && strstr(fp, "split.c")) { + found_split = 1; + ASSERT_NOT_NULL(ranges); + ASSERT_GT((int)strlen(ranges), 0); + } + } + ASSERT_TRUE(found_split); + const char *note = yyjson_get_str(yyjson_obj_get(pp, "note")); + ASSERT_NOT_NULL(note); + ASSERT_NOT_NULL(strstr(note, "guarantee")); + + /* Logfile lists it under the distinct phase. */ + const char *logfile = yyjson_get_str(yyjson_obj_get(sc, "logfile")); + ASSERT_NOT_NULL(logfile); + char *logtext = ri_slurp(logfile); + ASSERT_NOT_NULL(logtext); + ASSERT_NOT_NULL(strstr(logtext, "parse_partial")); + ASSERT_NOT_NULL(strstr(logtext, "split.c")); + free(logtext); + + /* The signal is persisted in the SEPARATE index_coverage table (never + * mixed into the graph tables) and reported by index_status, + * exactly as the index_repository tool description advertises. */ + cbm_coverage_row_t *rows = NULL; + int cov_count = 0; + ASSERT_EQ(cbm_store_coverage_get(store, lp.project, &rows, &cov_count), CBM_STORE_OK); + int marked = 0; + for (int i = 0; i < cov_count; i++) { + if (rows[i].rel_path && strstr(rows[i].rel_path, "split.c")) { + ASSERT_NOT_NULL(rows[i].kind); + ASSERT_STR_EQ("parse_partial", rows[i].kind); + ASSERT_NOT_NULL(rows[i].detail); + ASSERT_GT((int)strlen(rows[i].detail), 0); + marked = 1; + } + } + cbm_store_free_coverage(rows, cov_count); + ASSERT_TRUE(marked); + + char qargs[900]; + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *qresp = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); + ASSERT_NOT_NULL(qresp); + ASSERT_NOT_NULL(strstr(qresp, "split.c")); + ASSERT_NOT_NULL(strstr(qresp, "parse_partial")); + free(qresp); + + /* get_code_snippet on a symbol from the flagged file carries the + * correlated coverage note (the result names its file, so the warning is + * precisely anchored); the note never fires for clean files. */ + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\",\"qualified_name\":\"ok_before\"}", + lp.project); + char *sresp = cbm_mcp_handle_tool(lp.srv, "get_code_snippet", qargs); + ASSERT_NOT_NULL(sresp); + ASSERT_NOT_NULL(strstr(sresp, "coverage_note")); + ASSERT_NOT_NULL(strstr(sresp, "PARTIALLY indexed")); + free(sresp); + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\",\"qualified_name\":\"alpha\"}", lp.project); + char *cresp2 = cbm_mcp_handle_tool(lp.srv, "get_code_snippet", qargs); + ASSERT_NOT_NULL(cresp2); + ASSERT_NULL(strstr(cresp2, "coverage_note")); /* good.py: no note */ + free(cresp2); + + /* The missed GRAPH: the same signal is queryable as a file-structure + * graph via query_graph(graph="missed") — exactly the query the tool + * description advertises — and it lives under the shadow project, so the + * REAL code graph gained no coverage rows. */ + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"graph\":\"missed\",\"query\":\"MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail\"}", + lp.project); + char *gresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(gresp); + ASSERT_NOT_NULL(strstr(gresp, "split.c")); + free(gresp); + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.kind = " + "\\\"parse_partial\\\" RETURN f.file_path\"}", + lp.project); + char *cresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(cresp); + ASSERT_NULL(strstr(cresp, "split.c")); /* code graph: no coverage rows */ + free(cresp); + + /* Clean neighbors still extract. */ + int funcs = rh_count_label(store, lp.project, "Function"); + ASSERT_GTE(funcs, 1); + + yyjson_doc_free(d); + free(resp); + rh_cleanup(&lp, store); + PASS(); +} + +/* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must + * stay FRESH — after the broken file is fixed and the project re-indexed + * (incremental route: the DB already exists), its parse_partial row is gone + * and index_status reports it no longer. A stale flag on a fixed file + * would make the whole signal untrustworthy. */ +TEST(index_parse_partial_clears_on_fix) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + + /* The #ifdef-split-brace pattern: an UNRECOVERED miss (the first branch's + * `guarded` never becomes a def), so the flag survives recovery + * subtraction. A `def broken(:`-style fixture would NOT work here — its + * def is recovered and the region is dropped. */ + ri_write_text(lp.tmpdir, "flaky.c", + "void ok_before(void) { }\n" + "#ifdef A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n"); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char *resp = NULL; + cbm_store_t *store = ri_index_capture(&lp, &resp); + if (!resp) { + FAIL("no MCP response"); + } + free(resp); + if (!store) { + FAIL("store did not open"); + } + cbm_store_close(store); + + /* Flagged after the first (full) index. */ + char qargs[900]; + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *cov1 = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); + ASSERT_NOT_NULL(cov1); + ASSERT_NOT_NULL(strstr(cov1, "flaky.c")); + free(cov1); + + /* Fix the file; ensure a newer mtime so change detection can't miss it. */ + struct timespec ts = {0, INCR_FIX_SLEEP_NS}; + nanosleep(&ts, NULL); + ri_write_text(lp.tmpdir, "flaky.c", + "void ok_before(void) { }\n" + "static int fixed(int x) {\n" + " return x + 1;\n" + "}\n"); + + /* Re-index WITHOUT deleting the DB → routes through the incremental path. */ + char iargs[700]; + snprintf(iargs, sizeof(iargs), "{\"repo_path\":\"%s\"}", lp.tmpdir); + char *resp2 = cbm_mcp_handle_tool(lp.srv, "index_repository", iargs); + ASSERT_NOT_NULL(resp2); + free(resp2); + + char *cov2 = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); + ASSERT_NOT_NULL(cov2); + ASSERT_NULL(strstr(cov2, "flaky.c")); + free(cov2); + + store = cbm_store_open_path(lp.dbpath); + if (!store) { + FAIL("store did not reopen"); + } + rh_cleanup(&lp, store); + PASS(); +} + SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); + RUN_TEST(index_parse_partial_reported); + RUN_TEST(index_parse_partial_clears_on_fix); } diff --git a/tests/test_main.c b/tests/test_main.c index 67ba5df3d..4ca0f93c7 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -155,6 +155,7 @@ extern void suite_subprocess(void); extern void suite_extraction(void); extern void suite_extraction_inheritance(void); extern void suite_extraction_imports(void); +extern void suite_parse_coverage(void); extern void suite_grammar_regression(void); extern void suite_grammar_labels(void); extern void suite_grammar_imports(void); @@ -286,6 +287,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(extraction); RUN_SELECTED_SUITE(extraction_inheritance); RUN_SELECTED_SUITE(extraction_imports); + RUN_SELECTED_SUITE(parse_coverage); RUN_SELECTED_SUITE(grammar_regression); RUN_SELECTED_SUITE(grammar_labels); RUN_SELECTED_SUITE(grammar_imports); diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c new file mode 100644 index 000000000..7d3645ffb --- /dev/null +++ b/tests/test_parse_coverage.c @@ -0,0 +1,260 @@ +/* + * test_parse_coverage.c — Reproduce-first suite for the best-effort + * parse-coverage signal (#963, Signal A). + * + * ── The gap being reproduced ──────────────────────────────────────────────── + * When tree-sitter hits a construct it cannot parse (ERROR/MISSING nodes in + * the tree), extraction silently drops every definition inside the failed + * region — the file looks fully indexed but is not. `ts_node_has_error(root)` + * detects this, yet nothing consumed it: CBMFileResult gained the fields + * parse_incomplete / error_ranges / error_region_count, but the parse site in + * cbm_extract_file_impl never sets them. + * + * Canonical trigger: the preprocessor-blind #ifdef-split-brace pattern in C — + * both branches open `fn(...) {` and share ONE closing brace, so the raw text + * is brace-unbalanced → ERROR node → the guarded function never becomes a + * Function node while neighbors extract fine. + * + * ── The contract these tests enforce ──────────────────────────────────────── + * RED (unfixed): parse_incomplete is never set → flagged-file tests fail. + * GREEN (fixed): cbm_extract_file sets parse_incomplete=true iff the tree + * contains ERROR/MISSING nodes, records the 1-based line + * ranges of the TOP-MOST error regions ("start-end,..."), + * bounded by the 64-region cap, and clean files stay + * completely unflagged (no false positives). + * + * BEST-EFFORT framing (must never be weakened the other way): a flag means + * "constructs here were dropped — prefer grep"; the ABSENCE of a flag is NOT + * a completeness guarantee. These tests only pin down the detectable class. + */ + +#include "test_framework.h" +#include "cbm.h" +#include +#include +#include + +/* Convenience extract wrapper (same shape as test_extraction_imports.c). */ +static CBMFileResult *do_extract(const char *src, CBMLanguage lang, const char *path) { + return cbm_extract_file(src, (int)strlen(src), lang, "covproj", path, 0, NULL, NULL); +} + +/* Return 1 if any extracted definition has the given short name. */ +static int has_def(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, name) == 0) { + return 1; + } + } + return 0; +} + +/* ── Fixtures ─────────────────────────────────────────────────────────────── */ + +/* #ifdef-split-brace: both branches open `guarded(...) {`, one shared `}`. + * Preprocessor-blind parse sees unbalanced braces → ERROR region around + * lines 5–11; ok_before/ok_after remain extractable. */ +static const char *C_IFDEF_SPLIT = "#include \n" /* 1 */ + "\n" /* 2 */ + "void ok_before(void) { printf(\"a\"); }\n" /* 3 */ + "\n" /* 4 */ + "#ifdef FEATURE_A\n" /* 5 */ + "static int guarded(int x) {\n" /* 6 */ + "#else\n" /* 7 */ + "static int guarded_alt(int x) {\n" /* 8 */ + "#endif\n" /* 9 */ + " return x + 1;\n" /* 10 */ + "}\n" /* 11 */ + "\n" /* 12 */ + "void ok_after(void) { printf(\"b\"); }\n"; /* 13 */ + +static const char *C_CLEAN = "#include \n" + "\n" + "void alpha(void) { printf(\"a\"); }\n" + "\n" + "static int beta(int x) {\n" + " return x + 1;\n" + "}\n"; + +/* `def broken(:` parses with an ERROR region, but tree-sitter error recovery + * still yields the `broken` function def — a DEFINITELY RECOVERED miss. */ +static const char *PY_BROKEN_RECOVERED = "def ok():\n" + " return 1\n" + "\n" + "def broken(:\n" + " pass\n" + "\n" + "def ok2():\n" + " return 2\n"; + +/* Pure operator garbage between defs: an ERROR region no def walker can + * recover anything from — a genuine, unrecovered miss. */ +static const char *PY_GARBAGE = "def ok():\n" + " return 1\n" + "\n" + "%%% ((( garbage ))) %%%\n" + "??? !!!\n" + "\n" + "def ok2():\n" + " return 2\n"; + +static const char *PY_CLEAN = "def ok():\n" + " return 1\n" + "\n" + "def ok2():\n" + " return 2\n"; + +/* ── Tests ────────────────────────────────────────────────────────────────── */ + +TEST(c_ifdef_split_brace_sets_parse_incomplete) { + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); /* parse succeeded — this is the silent-partial class */ + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_NOT_NULL(r->error_ranges); + ASSERT_GT((int)strlen(r->error_ranges), 0); + cbm_free_result(r); + PASS(); +} + +TEST(c_ifdef_split_brace_neighbors_still_extracted) { + /* Documents WHY the flag matters: the file is partially indexed — + * neighbors extract, so nothing else hints at the dropped region. */ + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "ok_before")); + ASSERT_TRUE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +TEST(c_error_range_points_at_failed_region) { + /* The recorded range must overlap the #ifdef construct (lines 5–11) so an + * agent can be pointed at the exact unparsed region. Format is + * "start-end[,start-end...]", 1-based, inclusive. */ + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_NOT_NULL(r->error_ranges); + unsigned int start = 0; + unsigned int end = 0; + ASSERT_EQ(sscanf(r->error_ranges, "%u-%u", &start, &end), 2); + ASSERT_GTE(start, 1u); + ASSERT_LTE(start, 11u); /* starts at or before the region's last line */ + ASSERT_GTE(end, 5u); /* ends at or after the region's first line */ + ASSERT_LTE(end, 13u); /* never past EOF */ + ASSERT_LTE(start, end); + cbm_free_result(r); + PASS(); +} + +TEST(c_clean_file_not_flagged) { + /* No false positives: a clean parse must stay completely unflagged. */ + CBMFileResult *r = do_extract(C_CLEAN, CBM_LANG_C, "clean.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + ASSERT_TRUE(has_def(r, "alpha")); + ASSERT_TRUE(has_def(r, "beta")); + cbm_free_result(r); + PASS(); +} + +TEST(py_unrecovered_garbage_sets_parse_incomplete) { + CBMFileResult *r = do_extract(PY_GARBAGE, CBM_LANG_PYTHON, "garbage.py"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_NOT_NULL(r->error_ranges); + ASSERT_TRUE(has_def(r, "ok")); /* partial: clean defs still extracted */ + cbm_free_result(r); + PASS(); +} + +TEST(py_recovered_def_not_flagged) { + /* Recovery subtraction: `def broken(:` produces an ERROR region, but the + * def walker still recovers `broken` covering the whole region — the + * construct IS in the graph, so flagging it would be a false miss. */ + CBMFileResult *r = do_extract(PY_BROKEN_RECOVERED, CBM_LANG_PYTHON, "broken.py"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "broken")); /* the recovery that justifies unflagging */ + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + +TEST(py_clean_file_not_flagged) { + CBMFileResult *r = do_extract(PY_CLEAN, CBM_LANG_PYTHON, "clean.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + +TEST(error_region_cap_is_honored) { + /* Pathological input: many separate unrecoverable garbage blocks + * interleaved with valid defs. The collector must stay bounded by its + * 64-region cap (matches CBM_MAX_ERROR_REGIONS in cbm.c) — pathological + * input can't blow up the report, and the flag itself still fires. */ + enum { GARBAGE_BLOCKS = 200, LINE_CAP = 64 }; + char *src = (char *)malloc(GARBAGE_BLOCKS * 96 + 1); + ASSERT_NOT_NULL(src); + size_t off = 0; + for (int i = 0; i < GARBAGE_BLOCKS; i++) { + off += (size_t)snprintf( + src + off, 96, "def ok%d():\n return %d\n%%%%%% garbage%d ((( %%%%%%\n", i, i, i); + } + CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "many_errors.py"); + free(src); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_LTE(r->error_region_count, LINE_CAP); + ASSERT_NOT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + +/* Trailing recovered functions AFTER the failed #ifdef region must not + * unflag it: recovery evidence must originate INSIDE the region, and the + * unrecovered lines (the first branch's `guarded`) keep it flagged. */ +TEST(c_trailing_recovered_defs_keep_flag) { + const char *src = "void ok_before(void) { }\n" + "#ifdef A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n" + "void ok_after(void) { }\n" + "static int nested_ok(int y) { return y; }\n"; + CBMFileResult *r = do_extract(src, CBM_LANG_C, "probe.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "guarded_alt")); /* partial recovery inside the region */ + ASSERT_TRUE(r->parse_incomplete); /* ...but `guarded` is still lost */ + ASSERT_GTE(r->error_region_count, 1); + cbm_free_result(r); + PASS(); +} + +/* ── Suite ────────────────────────────────────────────────────────────────── */ + +SUITE(parse_coverage) { + RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete); + RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted); + RUN_TEST(c_error_range_points_at_failed_region); + RUN_TEST(c_clean_file_not_flagged); + RUN_TEST(py_unrecovered_garbage_sets_parse_incomplete); + RUN_TEST(py_recovered_def_not_flagged); + RUN_TEST(py_clean_file_not_flagged); + RUN_TEST(error_region_cap_is_honored); + RUN_TEST(c_trailing_recovered_defs_keep_flag); +} diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 5b30ef9f5..7cc1a019f 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1593,7 +1593,68 @@ TEST(store_count_nodes_unknown_project) { PASS(); } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* Round-trip + deleted-file prune + shadow miss-graph materialization + + * empty-replace wipe. The prune keys off file_hashes (the live-file set), so + * a row for a file with no hash row must not survive the replace. */ +TEST(store_coverage_roundtrip_prune_shadow) { + cbm_store_t *s = cbm_store_open_memory(); + cbm_store_upsert_project(s, "test", "/tmp/test"); + cbm_store_upsert_file_hash(s, "test", "src/a.py", "", 1, 10); + + cbm_coverage_row_t rows[] = { + {.rel_path = "src/a.py", .kind = "parse_partial", .detail = "4-7"}, + {.rel_path = "gone.py", .kind = "oversized", .detail = "too big"}, + }; + ASSERT_EQ(cbm_store_coverage_replace(s, "test", rows, 2), CBM_STORE_OK); + + cbm_coverage_row_t *got = NULL; + int n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); /* gone.py pruned — no file_hashes row */ + ASSERT_STR_EQ(got[0].rel_path, "src/a.py"); + ASSERT_STR_EQ(got[0].kind, "parse_partial"); + ASSERT_STR_EQ(got[0].detail, "4-7"); + cbm_store_free_coverage(got, n); + + /* Shadow miss-graph materialized under "test::missed": + * Project → Folder(src) → File(a.py){kind,detail}. */ + cbm_node_t *nodes = NULL; + int nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "File", &nodes, &nc), CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].file_path, "src/a.py"); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"kind\":\"parse_partial\"")); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"detail\":\"4-7\"")); + cbm_store_free_nodes(nodes, nc); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "Folder", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].name, "src"); + cbm_store_free_nodes(nodes, nc); + + /* Empty replace clears the table AND wipes the shadow graph. */ + ASSERT_EQ(cbm_store_coverage_replace(s, "test", NULL, 0), CBM_STORE_OK); + got = NULL; + n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + free(got); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "File", &nodes, &nc), CBM_STORE_OK); + ASSERT_EQ(nc, 0); + cbm_store_free_nodes(nodes, nc); + + cbm_store_close(s); + PASS(); +} + SUITE(store_nodes) { + RUN_TEST(store_coverage_roundtrip_prune_shadow); RUN_TEST(store_open_memory); RUN_TEST(store_close_null); RUN_TEST(store_open_memory_twice);