From a4c17bd4671c258170d77a1a4907893b2bcb6b63 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 8 Jul 2026 17:18:47 +0200 Subject: [PATCH 1/8] feat(index): best-effort parse-coverage signal for partially parsed files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files whose parse tree contains tree-sitter ERROR/MISSING regions were silently indexed as if complete — constructs inside those regions are absent from the graph with no signal anywhere (classic trigger: the preprocessor-blind #ifdef-split-brace pattern in C). - cbm_extract_file: set parse_incomplete and record the 1-based line ranges of the top-most error regions (bounded at 64), serialized as "start-end,start-end" into the result arena - pipeline: record such files under the new phase "parse_partial" (distinct from skips — the file IS indexed) in both the parallel and sequential extraction paths; stamp the File node with {parse_incomplete:true, error_ranges} on full and incremental runs via a new cbm_gbuf_set_node_props (upsert survivor rules bypass) - mcp: report parse_partial {files, count, truncated, note} separately from skipped[] (skipped_count semantics unchanged); the per-run logfile lists parse_partial rows; the index_repository tool description documents the signal and the query_graph pattern to find flagged files - tests: new parse_coverage suite (RED-first, 7 cases incl. false- positive guards and the region cap) plus an end-to-end resilience guard asserting response fields, logfile, File-node marker, and the advertised query_graph query The signal is explicitly best-effort: a flag means constructs in the listed ranges were dropped (prefer grep there); the absence of a flag is NOT a completeness guarantee. Refs #963 Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + internal/cbm/cbm.c | 71 ++++++++++ internal/cbm/cbm.h | 10 ++ src/graph_buffer/graph_buffer.c | 18 +++ src/graph_buffer/graph_buffer.h | 7 + src/mcp/mcp.c | 103 ++++++++++++-- src/pipeline/pass_definitions.c | 6 + src/pipeline/pass_parallel.c | 8 ++ src/pipeline/pipeline.c | 39 ++++++ src/pipeline/pipeline.h | 18 ++- src/pipeline/pipeline_incremental.c | 4 + src/pipeline/pipeline_internal.h | 8 ++ tests/test_index_resilience.c | 141 +++++++++++++++++++ tests/test_main.c | 2 + tests/test_parse_coverage.c | 208 ++++++++++++++++++++++++++++ 15 files changed, 627 insertions(+), 17 deletions(-) create mode 100644 tests/test_parse_coverage.c diff --git a/Makefile.cbm b/Makefile.cbm index 0156f3544..907560540 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -317,6 +317,7 @@ TEST_EXTRACTION_SRCS = \ tests/test_extraction.c \ tests/test_extraction_inheritance.c \ tests/test_extraction_imports.c \ + tests/test_parse_coverage.c \ tests/test_grammar_regression.c \ tests/test_grammar_labels.c \ tests/test_grammar_imports.c \ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index f38eaad69..c1becbfe2 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -688,6 +688,61 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, const char *rel_path, int64_t timeout_micros, const char **extra_defines, const char **include_paths); +/* Best-effort parse-coverage collection (#963). Walks only the has_error paths + * of the tree and records the 1-based line ranges of the TOP-MOST ERROR/MISSING + * nodes (does not descend into an error subtree — one range per failed region). + * Bounded by CBM_MAX_ERROR_REGIONS so pathological input can't blow up the + * output. The ranges mark where constructs were dropped; they are a detection + * aid, never a completeness proof. */ +#define CBM_MAX_ERROR_REGIONS 64 +typedef struct { + uint32_t starts[CBM_MAX_ERROR_REGIONS]; + uint32_t ends[CBM_MAX_ERROR_REGIONS]; + int count; +} cbm_error_regions_t; + +static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) { + if (acc->count >= CBM_MAX_ERROR_REGIONS) { + return; + } + acc->starts[acc->count] = ts_node_start_point(n).row + 1; + acc->ends[acc->count] = ts_node_end_point(n).row + 1; + acc->count++; +} + +static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) { + if (acc->count >= CBM_MAX_ERROR_REGIONS) { + return; + } + uint32_t k = ts_node_child_count(n); + for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) { + TSNode c = ts_node_child(n, i); + if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { + cbm_error_regions_push(acc, c); /* top-most region; do not descend */ + } else if (ts_node_has_error(c)) { + cbm_collect_error_regions(c, acc); + } + } +} + +/* Serialize collected regions as "start-end,start-end,..." into the arena. */ +static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) { + if (regs->count <= 0) { + return NULL; + } + enum { RANGE_MAX = 24 }; /* "4294967295-4294967295," */ + char *buf = (char *)cbm_arena_alloc(a, (size_t)regs->count * RANGE_MAX); + if (!buf) { + return NULL; + } + size_t off = 0; + for (int i = 0; i < regs->count; i++) { + off += (size_t)snprintf(buf + off, RANGE_MAX, "%s%u-%u", i ? "," : "", regs->starts[i], + regs->ends[i]); + } + return buf; +} + /* Public entry: run the extraction and journal completion. The DONE mark on * every ordinary return (including error/timeout results) tells the crash * supervisor this file did NOT kill the worker — only a file whose S has no @@ -788,6 +843,22 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, TSNode root = ts_tree_root_node(tree); + /* Best-effort parse-coverage signal (#963): flag files whose tree contains + * ERROR/MISSING nodes — constructs inside those regions are silently absent + * from the graph. Detection aid only: the absence of this flag is NOT a + * completeness guarantee. */ + if (ts_node_has_error(root)) { + result->parse_incomplete = true; + cbm_error_regions_t regs = {{0}, {0}, 0}; + if (strcmp(ts_node_type(root), "ERROR") == 0) { + cbm_error_regions_push(®s, root); /* whole file unparseable */ + } else { + cbm_collect_error_regions(root, ®s); + } + result->error_region_count = regs.count; + result->error_ranges = cbm_error_ranges_str(a, ®s); + } + // Compute module QN. Java/Go derive the module from the CONTAINING // DIRECTORY (package semantics) rather than baking the filename stem in, // so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6b4228131..27ebb71a8 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -447,6 +447,16 @@ typedef struct { bool has_error; const char *error_msg; + /* Best-effort parse-coverage signal (experimental). parse_incomplete is true + * when the parse tree contains tree-sitter ERROR/MISSING nodes — constructs + * in those regions are silently absent from the graph. error_ranges is a + * compact "start-end,start-end" list of 1-based line ranges (arena-owned) or + * NULL. This only marks what we can DETECT: the absence of a flag is NOT a + * completeness guarantee. Callers should treat a flagged file as "prefer + * grep here", never treat an unflagged file as provably complete. */ + bool parse_incomplete; + const char *error_ranges; + int error_region_count; bool is_test_file; int imports_count; TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index ca541e7d4..965cdd242 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -769,6 +769,24 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) return cbm_ht_get(gb->node_by_qn, qn); } +bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, + const char *properties_json) { + if (!gb || !qualified_name || !properties_json) { + return false; + } + cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name); + if (!node) { + return false; + } + char *new_props = heap_strdup(properties_json); + if (!new_props) { + return false; + } + free(node->properties_json); + node->properties_json = new_props; + return true; +} + const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) { if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) { return NULL; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 6c8c6babe..8cea45413 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -73,6 +73,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name const char *qualified_name, const char *file_path, int start_line, int end_line, const char *properties_json); +/* Replace an existing node's properties_json by qualified name (copied). + * Bypasses the upsert survivor rule — use for post-hoc annotation of a node + * that is known to exist (e.g. parse-coverage markers on File nodes, #963). + * Returns true when the node was found and updated. */ +bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, + const char *properties_json); + /* Find a node by qualified name. Returns NULL if not found. */ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 6702fd1c9..c87fbdfe8 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -316,7 +316,14 @@ static const tool_def_t TOOLS[] = { "Index a repository into the knowledge graph. " "Special mode 'cross-repo-intelligence': skip extraction, only match Routes/Channels " "across projects to create CROSS_HTTP_CALLS/CROSS_ASYNC_CALLS/CROSS_CHANNEL edges. " - "Requires target_projects param. Ensure target projects have fresh indexes first.", + "Requires target_projects param. Ensure target projects have fresh indexes first. " + "COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not " + "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " + "constructs inside the listed line ranges could not be parsed and are absent from the " + "graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their " + "File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true " + "RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness " + "guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," "\"mode\":{\"type\":\"string\"," @@ -3299,36 +3306,100 @@ static void add_excluded_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, char * the JSON carries "count" + "truncated" so nothing is silently hidden. */ enum { INDEX_SKIPPED_FILE_CAP = 50 }; +/* True when a recorded per-file entry is the parse-partial coverage signal + * (#963) rather than a genuine skip. Kept out of skipped[]/skipped_count so + * the "skipped" contract (file NOT indexed) stays exact. */ +static bool is_parse_partial(const cbm_file_error_t *e) { + return e->phase && strcmp(e->phase, "parse_partial") == 0; +} + /* Attach a summary of per-file skips (Stage 2 / Track B). Always emits a * top-level "skipped_count" (0 on clean runs) so consumers can rely on it. * When there are skips, also emits: * "skipped": {"files":[{path,reason,phase}..(<=50)], "count":N, "truncated":bool} * and, if a per-run logfile was written, "logfile": "". * The run status stays "indexed" — a skipped file is the expected handled - * outcome, not a failure. errs[] is borrowed (copied into doc). */ + * outcome, not a failure. errs[] is borrowed (copied into doc) and may contain + * parse_partial entries, which are filtered out here (reported separately by + * add_parse_partial_summary). */ static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_file_error_t *errs, int count, const char *logfile) { - yyjson_mut_obj_add_int(doc, root, "skipped_count", count < 0 ? 0 : count); - if (!errs || count <= 0) { + int skips = 0; + for (int i = 0; i < count; i++) { + if (!is_parse_partial(&errs[i])) { + skips++; + } + } + yyjson_mut_obj_add_int(doc, root, "skipped_count", skips); + if (logfile && logfile[0]) { + yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile); + } + if (!errs || skips <= 0) { return; } yyjson_mut_val *skipped = yyjson_mut_obj(doc); yyjson_mut_val *files = yyjson_mut_arr(doc); - int shown = count < INDEX_SKIPPED_FILE_CAP ? count : INDEX_SKIPPED_FILE_CAP; - for (int i = 0; i < shown; i++) { + int shown = 0; + for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) { + if (is_parse_partial(&errs[i])) { + continue; + } yyjson_mut_val *fe = yyjson_mut_obj(doc); yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : ""); yyjson_mut_obj_add_strcpy(doc, fe, "reason", errs[i].reason ? errs[i].reason : ""); yyjson_mut_obj_add_strcpy(doc, fe, "phase", errs[i].phase ? errs[i].phase : ""); yyjson_mut_arr_add_val(files, fe); + shown++; } yyjson_mut_obj_add_val(doc, skipped, "files", files); - yyjson_mut_obj_add_int(doc, skipped, "count", count); - yyjson_mut_obj_add_bool(doc, skipped, "truncated", count > INDEX_SKIPPED_FILE_CAP); + yyjson_mut_obj_add_int(doc, skipped, "count", skips); + yyjson_mut_obj_add_bool(doc, skipped, "truncated", skips > INDEX_SKIPPED_FILE_CAP); yyjson_mut_obj_add_val(doc, root, "skipped", skipped); - if (logfile && logfile[0]) { - yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile); +} + +/* Attach the best-effort parse-coverage summary (#963). Always emits a + * top-level "parse_partial_count" (0 on clean runs). When files were flagged: + * "parse_partial": {"files":[{path,error_ranges}..(<=50)], "count":N, + * "truncated":bool, "note":"..."} + * These files WERE indexed — constructs inside the listed 1-based line ranges + * are missing from the graph because tree-sitter could not parse them. The + * note spells out the best-effort framing: absence from this list is NOT a + * completeness guarantee. */ +static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_file_error_t *errs, int count) { + int partials = 0; + for (int i = 0; i < count; i++) { + if (is_parse_partial(&errs[i])) { + partials++; + } + } + yyjson_mut_obj_add_int(doc, root, "parse_partial_count", partials); + if (!errs || partials <= 0) { + return; } + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_val *files = yyjson_mut_arr(doc); + int shown = 0; + for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) { + if (!is_parse_partial(&errs[i])) { + continue; + } + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", errs[i].path ? errs[i].path : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", errs[i].reason ? errs[i].reason : ""); + yyjson_mut_arr_add_val(files, fe); + shown++; + } + yyjson_mut_obj_add_val(doc, pp, "files", files); + yyjson_mut_obj_add_int(doc, pp, "count", partials); + yyjson_mut_obj_add_bool(doc, pp, "truncated", partials > INDEX_SKIPPED_FILE_CAP); + yyjson_mut_obj_add_str(doc, pp, "note", + "Best-effort signal, not a completeness guarantee: these files WERE " + "indexed, but constructs inside the listed line ranges (1-based) could " + "not be parsed and are missing from the graph. Prefer text search " + "(grep) for those regions. Files absent from this list are NOT " + "guaranteed to be fully indexed."); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); } /* Write the FULL (uncapped) skip list to a per-run logfile — ONLY when >=1 file @@ -3360,8 +3431,15 @@ static bool write_skip_logfile(const char *project, const cbm_file_error_t *errs cbm_log_warn("index.logfile_open_fail", "path", path); return false; } - (void)fprintf(f, "# codebase-memory-mcp index skip report\n"); - (void)fprintf(f, "# project=%s skipped=%d\n", project ? project : "", count); + int partials = 0; + for (int i = 0; i < count; i++) { + if (is_parse_partial(&errs[i])) { + partials++; + } + } + (void)fprintf(f, "# codebase-memory-mcp index coverage report\n"); + (void)fprintf(f, "# project=%s skipped=%d parse_partial=%d\n", project ? project : "", + count - partials, partials); (void)fprintf(f, "# columns: phase\treason\tpath\n"); for (int i = 0; i < count; i++) { (void)fprintf(f, "%s\t%s\t%s\n", errs[i].phase ? errs[i].phase : "", @@ -3384,6 +3462,7 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * const char *logfile) { add_excluded_summary(doc, root, excluded_dirs, excluded_count); add_skipped_summary(doc, root, file_errors, file_error_count, logfile); + add_parse_partial_summary(doc, root, file_errors, file_error_count); int exp_nodes = -1; int exp_edges = -1; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index acda8b226..21f5360c9 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -581,6 +581,12 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t result->error_msg ? result->error_msg : "extract failed", "extract"); errors++; + } else if (result->parse_incomplete) { + /* Best-effort parse-coverage signal (#963): indexed, but with + * ERROR/MISSING regions — see pass_parallel.c (keep in sync). */ + cbm_pipeline_add_file_error(ctx->pipeline, rel, + result->error_ranges ? result->error_ranges : "unknown", + "parse_partial"); } /* Create nodes for each definition */ diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 8e0d35691..7f85f81f2 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -902,6 +902,14 @@ static void extract_worker(int worker_id, void *ctx_ptr) { pp_err_add(errs, fi->rel_path, result->error_msg ? result->error_msg : "extract failed", "extract"); ws->errors++; + } else if (result->parse_incomplete) { + /* Best-effort parse-coverage signal (#963): the file WAS indexed, + * but its tree contains ERROR/MISSING regions whose constructs are + * silently absent from the graph. Not a skip — recorded under the + * distinct "parse_partial" phase (reason = the line-range list) so + * the MCP layer reports it separately from skipped[]. */ + pp_err_add(errs, fi->rel_path, result->error_ranges ? result->error_ranges : "unknown", + "parse_partial"); } /* Create definition nodes in local gbuf */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index cacfd7639..dbce7198b 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -315,6 +315,40 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou } } +void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) { + if (!p || !gbuf) { + return; + } + int stamped = 0; + for (int i = 0; i < p->file_errors_count; i++) { + const cbm_file_error_t *e = &p->file_errors[i]; + if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) { + continue; + } + char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__"); + if (!file_qn) { + continue; + } + const char *slash = strrchr(e->path, '/'); + const char *basename = slash ? slash + SKIP_ONE : e->path; + const char *ext = strrchr(basename, '.'); + /* Full replacement props (set_node_props does not merge): keep the + * "extension" key pass_structure wrote, add the coverage marker. The + * ranges string is digits/commas/dashes — JSON-safe unescaped. */ + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), + "{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}", + ext ? ext : "", e->reason ? e->reason : ""); + if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) { + stamped++; + } + free(file_qn); + } + if (stamped > 0) { + cbm_log_info("index.parse_partial", "files", itoa_buf(stamped)); + } +} + void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) { if (nodes) { *nodes = p ? p->committed_nodes : -1; @@ -1262,6 +1296,11 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (check_cancel(p)) { return CBM_NOT_FOUND; } + if (rc == 0) { + /* Extraction merged its per-file errors — stamp parse-partial File + * nodes (#963) before downstream passes and the dump. */ + cbm_pipeline_stamp_parse_partial(p, p->gbuf); + } return rc; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 535c717fe..bb253979b 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -89,11 +89,19 @@ void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int typedef struct { char *path; /* repo-relative path of the skipped file */ char *reason; /* human-readable cause (e.g. "oversized (712 MB > 512 MB)", - * "parse timeout", "read failed") */ - char *phase; /* "read" | "extract" | "oversized". "cross_lsp" is a RESERVED - * phase string for Track C's crash-attribution signal and is - * intentionally NOT emitted today (the cross-LSP passes are - * best-effort/void with no genuine per-file failure). */ + * "parse timeout", "read failed"). For phase "parse_partial" + * this carries the 1-based line-range list ("12-40,88-90") + * of the unparseable regions. */ + char *phase; /* "read" | "extract" | "oversized" | "parse_partial". + * "parse_partial" (#963) is NOT a skip: the file WAS indexed + * but contains tree-sitter ERROR/MISSING regions whose + * constructs are absent from the graph (best-effort signal — + * absence of the flag is NOT a completeness guarantee). The + * MCP layer reports it separately from skipped[]. "cross_lsp" + * is a RESERVED phase string for Track C's crash-attribution + * signal and is intentionally NOT emitted today (the + * cross-LSP passes are best-effort/void with no genuine + * per-file failure). */ } cbm_file_error_t; /* Record a skipped file. path/reason/phase are copied. NULL-safe on p. diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 1fa320d86..4b5cedf8f 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -840,6 +840,10 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } run_extract_resolve(&ctx, changed_files, ci); + /* Stamp parse-partial File nodes (#963) for the changed slice. A changed + * file that parses cleanly now got fresh "{}" props above, so a stale + * marker from a previous run cannot survive the reindex. */ + cbm_pipeline_stamp_parse_partial(p, existing); cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index a3d806558..1648bddfe 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -520,6 +520,14 @@ void cbm_pipeline_create_route_nodes(cbm_gbuf_t *gb); /* ── Pass function prototypes ────────────────────────────────────── */ +/* Stamp File nodes for files recorded under phase "parse_partial" (#963): + * sets properties {"extension":...,"parse_incomplete":true,"error_ranges":...} + * on the (already-existing) File node so the coverage signal is queryable in + * the graph. Call after the extraction pass has merged its per-file errors — + * both the full path (pipeline.c) and the incremental path + * (pipeline_incremental.c) must call it. Implementation: pipeline.c. */ +void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf); + int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index e503af58a..db8cc709f 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -248,6 +248,11 @@ TEST(index_clean_run_no_logfile) { ASSERT_NULL(yyjson_obj_get(sc, "skipped")); ASSERT_NULL(yyjson_obj_get(sc, "logfile")); + /* Clean parses → no parse-coverage flags either (#963). */ + int pp_count = yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")); + ASSERT_EQ(pp_count, 0); + ASSERT_NULL(yyjson_obj_get(sc, "parse_partial")); + int funcs = rh_count_label(store, lp.project, "Function"); ASSERT_GTE(funcs, 2); @@ -257,7 +262,143 @@ TEST(index_clean_run_no_logfile) { PASS(); } +/* INV(parse-partial-reported, #963): a file whose parse tree contains + * ERROR/MISSING regions (here: the preprocessor-blind #ifdef-split-brace C + * pattern) is INDEXED — not skipped — and the best-effort coverage signal + * surfaces on every layer: + * - skipped_count == 0 (a partial parse is NOT a skip), + * - parse_partial_count >= 1 with the file + its line ranges + the + * best-effort note in parse_partial{}, + * - the per-run logfile lists it under phase "parse_partial", + * - the File node carries {"parse_incomplete":true,"error_ranges":...}, + * - clean neighbors still extract. + * + * Guard property: on the unwired code there is no parse_partial_count / + * parse_partial[] / File-node marker — the file silently looks fully indexed. + */ +TEST(index_parse_partial_reported) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + + /* Both #ifdef branches open `guarded(...) {` sharing ONE close brace — + * brace-unbalanced for a preprocessor-blind parse → ERROR region. */ + ri_write_text(lp.tmpdir, "split.c", + "void ok_before(void) { }\n" + "#ifdef FEATURE_A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n"); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char logpath[700]; + snprintf(logpath, sizeof(logpath), "%s/coverage.log", lp.tmpdir); + cbm_setenv("CBM_INDEX_LOG", logpath, 1); + + char *resp = NULL; + cbm_store_t *store = ri_index_capture(&lp, &resp); + cbm_unsetenv("CBM_INDEX_LOG"); + + if (!resp) { + FAIL("no MCP response"); + } + if (!store) { + free(resp); + FAIL("store did not open"); + } + + yyjson_doc *d = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(d); + yyjson_val *sc = yyjson_obj_get(yyjson_doc_get_root(d), "structuredContent"); + ASSERT_NOT_NULL(sc); + + /* Indexed, and NOT counted as a skip. */ + const char *status = yyjson_get_str(yyjson_obj_get(sc, "status")); + ASSERT_NOT_NULL(status); + ASSERT_STR_EQ("indexed", status); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "skipped_count")), 0); + + /* The coverage signal is surfaced with ranges + the best-effort note. */ + ASSERT_GTE(yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")), 1); + yyjson_val *pp = yyjson_obj_get(sc, "parse_partial"); + ASSERT_NOT_NULL(pp); + yyjson_val *files = yyjson_obj_get(pp, "files"); + ASSERT_NOT_NULL(files); + int found_split = 0; + size_t idx = 0; + size_t fmax = 0; + yyjson_val *fe = NULL; + yyjson_arr_foreach(files, idx, fmax, fe) { + const char *fp = yyjson_get_str(yyjson_obj_get(fe, "path")); + const char *ranges = yyjson_get_str(yyjson_obj_get(fe, "error_ranges")); + if (fp && strstr(fp, "split.c")) { + found_split = 1; + ASSERT_NOT_NULL(ranges); + ASSERT_GT((int)strlen(ranges), 0); + } + } + ASSERT_TRUE(found_split); + const char *note = yyjson_get_str(yyjson_obj_get(pp, "note")); + ASSERT_NOT_NULL(note); + ASSERT_NOT_NULL(strstr(note, "guarantee")); + + /* Logfile lists it under the distinct phase. */ + const char *logfile = yyjson_get_str(yyjson_obj_get(sc, "logfile")); + ASSERT_NOT_NULL(logfile); + char *logtext = ri_slurp(logfile); + ASSERT_NOT_NULL(logtext); + ASSERT_NOT_NULL(strstr(logtext, "parse_partial")); + ASSERT_NOT_NULL(strstr(logtext, "split.c")); + free(logtext); + + /* The File node carries the queryable marker. */ + cbm_node_t *nodes = NULL; + int count = 0; + int rc = cbm_store_find_nodes_by_qn_suffix(store, lp.project, "__file__", &nodes, &count); + ASSERT_EQ(rc, CBM_STORE_OK); + int marked = 0; + for (int i = 0; i < count; i++) { + if (nodes[i].file_path && strstr(nodes[i].file_path, "split.c")) { + ASSERT_NOT_NULL(nodes[i].properties_json); + ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"parse_incomplete\":true")); + ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"error_ranges\":\"")); + marked = 1; + } + } + cbm_store_free_nodes(nodes, count); + ASSERT_TRUE(marked); + + /* The marker is queryable exactly as the index_repository tool description + * advertises (agents are pointed at this query to find unindexed code). */ + char qargs[900]; + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.parse_incomplete = true " + "RETURN f.file_path, f.error_ranges\"}", + lp.project); + char *qresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(qresp); + ASSERT_NOT_NULL(strstr(qresp, "split.c")); + free(qresp); + + /* Clean neighbors still extract. */ + int funcs = rh_count_label(store, lp.project, "Function"); + ASSERT_GTE(funcs, 1); + + yyjson_doc_free(d); + free(resp); + rh_cleanup(&lp, store); + PASS(); +} + SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); + RUN_TEST(index_parse_partial_reported); } diff --git a/tests/test_main.c b/tests/test_main.c index 67ba5df3d..4ca0f93c7 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -155,6 +155,7 @@ extern void suite_subprocess(void); extern void suite_extraction(void); extern void suite_extraction_inheritance(void); extern void suite_extraction_imports(void); +extern void suite_parse_coverage(void); extern void suite_grammar_regression(void); extern void suite_grammar_labels(void); extern void suite_grammar_imports(void); @@ -286,6 +287,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(extraction); RUN_SELECTED_SUITE(extraction_inheritance); RUN_SELECTED_SUITE(extraction_imports); + RUN_SELECTED_SUITE(parse_coverage); RUN_SELECTED_SUITE(grammar_regression); RUN_SELECTED_SUITE(grammar_labels); RUN_SELECTED_SUITE(grammar_imports); diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c new file mode 100644 index 000000000..3b11cfc01 --- /dev/null +++ b/tests/test_parse_coverage.c @@ -0,0 +1,208 @@ +/* + * test_parse_coverage.c — Reproduce-first suite for the best-effort + * parse-coverage signal (#963, Signal A). + * + * ── The gap being reproduced ──────────────────────────────────────────────── + * When tree-sitter hits a construct it cannot parse (ERROR/MISSING nodes in + * the tree), extraction silently drops every definition inside the failed + * region — the file looks fully indexed but is not. `ts_node_has_error(root)` + * detects this, yet nothing consumed it: CBMFileResult gained the fields + * parse_incomplete / error_ranges / error_region_count, but the parse site in + * cbm_extract_file_impl never sets them. + * + * Canonical trigger: the preprocessor-blind #ifdef-split-brace pattern in C — + * both branches open `fn(...) {` and share ONE closing brace, so the raw text + * is brace-unbalanced → ERROR node → the guarded function never becomes a + * Function node while neighbors extract fine. + * + * ── The contract these tests enforce ──────────────────────────────────────── + * RED (unfixed): parse_incomplete is never set → flagged-file tests fail. + * GREEN (fixed): cbm_extract_file sets parse_incomplete=true iff the tree + * contains ERROR/MISSING nodes, records the 1-based line + * ranges of the TOP-MOST error regions ("start-end,..."), + * bounded by the 64-region cap, and clean files stay + * completely unflagged (no false positives). + * + * BEST-EFFORT framing (must never be weakened the other way): a flag means + * "constructs here were dropped — prefer grep"; the ABSENCE of a flag is NOT + * a completeness guarantee. These tests only pin down the detectable class. + */ + +#include "test_framework.h" +#include "cbm.h" +#include +#include +#include + +/* Convenience extract wrapper (same shape as test_extraction_imports.c). */ +static CBMFileResult *do_extract(const char *src, CBMLanguage lang, const char *path) { + return cbm_extract_file(src, (int)strlen(src), lang, "covproj", path, 0, NULL, NULL); +} + +/* Return 1 if any extracted definition has the given short name. */ +static int has_def(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, name) == 0) { + return 1; + } + } + return 0; +} + +/* ── Fixtures ─────────────────────────────────────────────────────────────── */ + +/* #ifdef-split-brace: both branches open `guarded(...) {`, one shared `}`. + * Preprocessor-blind parse sees unbalanced braces → ERROR region around + * lines 5–11; ok_before/ok_after remain extractable. */ +static const char *C_IFDEF_SPLIT = "#include \n" /* 1 */ + "\n" /* 2 */ + "void ok_before(void) { printf(\"a\"); }\n" /* 3 */ + "\n" /* 4 */ + "#ifdef FEATURE_A\n" /* 5 */ + "static int guarded(int x) {\n" /* 6 */ + "#else\n" /* 7 */ + "static int guarded_alt(int x) {\n" /* 8 */ + "#endif\n" /* 9 */ + " return x + 1;\n" /* 10 */ + "}\n" /* 11 */ + "\n" /* 12 */ + "void ok_after(void) { printf(\"b\"); }\n"; /* 13 */ + +static const char *C_CLEAN = "#include \n" + "\n" + "void alpha(void) { printf(\"a\"); }\n" + "\n" + "static int beta(int x) {\n" + " return x + 1;\n" + "}\n"; + +static const char *PY_BROKEN = "def ok():\n" + " return 1\n" + "\n" + "def broken(:\n" + " pass\n" + "\n" + "def ok2():\n" + " return 2\n"; + +static const char *PY_CLEAN = "def ok():\n" + " return 1\n" + "\n" + "def ok2():\n" + " return 2\n"; + +/* ── Tests ────────────────────────────────────────────────────────────────── */ + +TEST(c_ifdef_split_brace_sets_parse_incomplete) { + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); /* parse succeeded — this is the silent-partial class */ + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_NOT_NULL(r->error_ranges); + ASSERT_GT((int)strlen(r->error_ranges), 0); + cbm_free_result(r); + PASS(); +} + +TEST(c_ifdef_split_brace_neighbors_still_extracted) { + /* Documents WHY the flag matters: the file is partially indexed — + * neighbors extract, so nothing else hints at the dropped region. */ + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "ok_before")); + ASSERT_TRUE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +TEST(c_error_range_points_at_failed_region) { + /* The recorded range must overlap the #ifdef construct (lines 5–11) so an + * agent can be pointed at the exact unparsed region. Format is + * "start-end[,start-end...]", 1-based, inclusive. */ + CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_NOT_NULL(r->error_ranges); + unsigned int start = 0; + unsigned int end = 0; + ASSERT_EQ(sscanf(r->error_ranges, "%u-%u", &start, &end), 2); + ASSERT_GTE(start, 1u); + ASSERT_LTE(start, 11u); /* starts at or before the region's last line */ + ASSERT_GTE(end, 5u); /* ends at or after the region's first line */ + ASSERT_LTE(end, 13u); /* never past EOF */ + ASSERT_LTE(start, end); + cbm_free_result(r); + PASS(); +} + +TEST(c_clean_file_not_flagged) { + /* No false positives: a clean parse must stay completely unflagged. */ + CBMFileResult *r = do_extract(C_CLEAN, CBM_LANG_C, "clean.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + ASSERT_TRUE(has_def(r, "alpha")); + ASSERT_TRUE(has_def(r, "beta")); + cbm_free_result(r); + PASS(); +} + +TEST(py_syntax_error_sets_parse_incomplete) { + CBMFileResult *r = do_extract(PY_BROKEN, CBM_LANG_PYTHON, "broken.py"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_NOT_NULL(r->error_ranges); + ASSERT_TRUE(has_def(r, "ok")); /* partial: clean defs still extracted */ + cbm_free_result(r); + PASS(); +} + +TEST(py_clean_file_not_flagged) { + CBMFileResult *r = do_extract(PY_CLEAN, CBM_LANG_PYTHON, "clean.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + +TEST(error_region_cap_is_honored) { + /* Pathological input: many separate broken defs interleaved with valid + * ones. The collector must stay bounded by its 64-region cap (matches + * CBM_MAX_ERROR_REGIONS in cbm.c) — pathological input can't blow up the + * report, and the flag itself still fires. */ + enum { BROKEN_DEFS = 200, LINE_CAP = 64 }; + char *src = (char *)malloc(BROKEN_DEFS * 96 + 1); + ASSERT_NOT_NULL(src); + size_t off = 0; + for (int i = 0; i < BROKEN_DEFS; i++) { + off += (size_t)snprintf(src + off, 96, + "def ok%d():\n return %d\ndef broken%d(:\n pass\n", i, i, i); + } + CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "many_errors.py"); + free(src); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(r->parse_incomplete); + ASSERT_GTE(r->error_region_count, 1); + ASSERT_LTE(r->error_region_count, LINE_CAP); + ASSERT_NOT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + +/* ── Suite ────────────────────────────────────────────────────────────────── */ + +SUITE(parse_coverage) { + RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete); + RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted); + RUN_TEST(c_error_range_points_at_failed_region); + RUN_TEST(c_clean_file_not_flagged); + RUN_TEST(py_syntax_error_sets_parse_incomplete); + RUN_TEST(py_clean_file_not_flagged); + RUN_TEST(error_region_cap_is_honored); +} From f9d7f4aa61e69ca2d10fd3e109055161048b3971 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 8 Jul 2026 17:47:59 +0200 Subject: [PATCH 2/8] feat(index): persist coverage in separate table + queryable miss graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist the parse-coverage signal outside the graph and make the misses queryable both as a list and as a graph. - store: new index_coverage table (project, rel_path, kind, detail) — coverage is metadata ABOUT the graph and never mixes into the graph tables; replace/get/free APIs with a deleted-file prune keyed off file_hashes (the live-file set) - store: materialize a derived miss graph under the shadow project "::coverage" (Project -> Folder chain -> File{kind,detail}, CONTAINS_FOLDER/CONTAINS_FILE edges), rebuilt from the table inside the same transaction on every replace; invisible to list_projects (which scans the cache directory, not the projects table) - pipeline: coverage is written in the same persist step as every other post-dump SQL write (hashes, ADR, FTS). Full runs replace the whole set; incremental runs merge rows surviving from files not re-extracted with this run's fresh entries, so a fixed file's flag clears and deleted files prune automatically - mcp: new get_index_coverage tool (per-project parse_partial + skipped lists with the best-effort note); query_graph gains graph="coverage" to run the same cypher against the miss graph; the File-node property marker from the previous commit is dropped in favor of the separate table — the code graph gains no coverage rows - wording: constructs in flagged ranges MAY be missing — tree-sitter error recovery still salvages some (verified: a function inside a flagged #ifdef region and even a broken def were still extracted) - tests: store round-trip/prune/shadow-graph unit test; e2e asserts the response fields, the advertised coverage-graph query, code-graph purity, and that the flag clears after the file is fixed (incremental route) Refs #963 Signed-off-by: Martin Vogel --- src/graph_buffer/graph_buffer.c | 18 -- src/graph_buffer/graph_buffer.h | 7 - src/mcp/mcp.c | 147 ++++++++++++++-- src/pipeline/pipeline.c | 62 +++---- src/pipeline/pipeline_incremental.c | 58 ++++++- src/pipeline/pipeline_internal.h | 8 - src/store/store.c | 255 ++++++++++++++++++++++++++++ src/store/store.h | 34 ++++ tests/test_index_resilience.c | 122 +++++++++++-- tests/test_store_nodes.c | 63 +++++++ 10 files changed, 667 insertions(+), 107 deletions(-) diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 965cdd242..ca541e7d4 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -769,24 +769,6 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) return cbm_ht_get(gb->node_by_qn, qn); } -bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, - const char *properties_json) { - if (!gb || !qualified_name || !properties_json) { - return false; - } - cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name); - if (!node) { - return false; - } - char *new_props = heap_strdup(properties_json); - if (!new_props) { - return false; - } - free(node->properties_json); - node->properties_json = new_props; - return true; -} - const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) { if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) { return NULL; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 8cea45413..6c8c6babe 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -73,13 +73,6 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name const char *qualified_name, const char *file_path, int start_line, int end_line, const char *properties_json); -/* Replace an existing node's properties_json by qualified name (copied). - * Bypasses the upsert survivor rule — use for post-hoc annotation of a node - * that is known to exist (e.g. parse-coverage markers on File nodes, #963). - * Returns true when the node was found and updated. */ -bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, - const char *properties_json); - /* Find a node by qualified name. Returns NULL if not found. */ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c87fbdfe8..613a133b6 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -319,11 +319,10 @@ static const tool_def_t TOOLS[] = { "Requires target_projects param. Ensure target projects have fresh indexes first. " "COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not " "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " - "constructs inside the listed line ranges could not be parsed and are absent from the " - "graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their " - "File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true " - "RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness " - "guarantee; prefer grep inside flagged ranges.", + "constructs inside the listed line ranges could not be parsed and MAY be missing from " + "the graph). Query the persisted signal any time via the get_index_coverage tool or " + "structurally via query_graph(graph=\"coverage\"). Both signals are best-effort: absence " + "of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," "\"mode\":{\"type\":\"string\"," @@ -395,9 +394,20 @@ static const tool_def_t TOOLS[] = { "no conditionally-guarded base case), param_count and max_access_depth (structure smells). " "Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE " "f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, " - "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC.", + "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. " + "COVERAGE GRAPH: pass graph=\"coverage\" to query the best-effort miss graph instead — " + "the file structure of files the indexer could NOT fully cover (Project → Folder → " + "File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind " + "(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; " + "or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail. Absence from this graph is " + "NOT a completeness guarantee.", "{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher " - "query\"},\"project\":{\"type\":\"string\"},\"max_rows\":{\"type\":\"integer\"," + "query\"},\"project\":{\"type\":\"string\"}," + "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"coverage\"],\"default\":\"code\"," + "\"description\":\"Which graph to query: the code knowledge graph (default) or the " + "indexing-coverage miss graph (files not fully indexed, as a file-structure graph).\"}," + "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Optional row limit. Default: unlimited up to a 100k row " "ceiling. No offset support — use search_graph for paginated browsing.\"}}," @@ -489,6 +499,18 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, + {"get_index_coverage", "Index coverage", + "Report which files the indexer could NOT fully cover (best-effort signal, #963): " + "'parse_partial' files WERE indexed but contain line ranges tree-sitter could not parse — " + "constructs there MAY be missing from the graph (some are still recovered); 'skipped' files " + "were not indexed at all (oversized/read/parse failure). Use this before trusting graph " + "completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). " + "IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks " + "what the indexer can detect. For structural queries over the misses use " + "query_graph(graph=\"coverage\").", + "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" + "\"project\"]}"}, + {"detect_changes", "Detect changes", "Detect code changes and their impact", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"scope\":{\"type\":" "\"string\"},\"depth\":{\"type\":\"integer\",\"default\":2},\"base_branch\":{\"type\":" @@ -2065,10 +2087,21 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { cbm_store_t *store = resolve_store(srv, project); int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0); + /* graph="coverage" (#963): run the SAME cypher against the derived + * miss-graph view (shadow project "::coverage") instead of the + * code graph — file structure of not-fully-indexed files only. */ + char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); + bool coverage_graph = graph_arg && strcmp(graph_arg, "coverage") == 0; + free(graph_arg); + if (!query) { free(project); return cbm_mcp_text_result("query is required", true); } + if (coverage_graph && !project) { + free(query); + return cbm_mcp_text_result("project is required when graph=\"coverage\"", true); + } if (!store) { char *_err = build_project_list_error("project not found or not indexed"); char *_res = cbm_mcp_text_result(_err, true); @@ -2085,8 +2118,15 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return not_indexed; } + char covproj[CBM_SZ_512]; + const char *cypher_project = project; + if (coverage_graph) { + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + cypher_project = covproj; + } + cbm_cypher_result_t result = {0}; - int rc = cbm_cypher_execute(store, query, project, max_rows, &result); + int rc = cbm_cypher_execute(store, query, cypher_project, max_rows, &result); if (rc < 0) { char *err_msg = result.error ? result.error : "query execution failed"; @@ -2181,6 +2221,87 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { return result; } +/* get_index_coverage (#963): report the best-effort indexing-coverage signal + * from the separate index_coverage table (coverage is metadata ABOUT the + * graph, stored outside it). Full per-project list, capped generously. */ +enum { COVERAGE_FILE_CAP = 500 }; + +static char *handle_get_index_coverage(cbm_mcp_server_t *srv, const char *args) { + char *project = get_project_arg(args); + cbm_store_t *store = resolve_store(srv, project); + REQUIRE_STORE(store, project); + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + + if (project) { + yyjson_mut_obj_add_str(doc, root, "project", project); + cbm_coverage_row_t *rows = NULL; + int count = 0; + (void)cbm_store_coverage_get(store, project, &rows, &count); + + yyjson_mut_val *pp_files = yyjson_mut_arr(doc); + yyjson_mut_val *sk_files = yyjson_mut_arr(doc); + int pp_n = 0; + int sk_n = 0; + for (int i = 0; i < count; i++) { + bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0; + if (partial) { + if (pp_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_arr_add_val(pp_files, fe); + } + pp_n++; + } else { + if (sk_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "reason", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : ""); + yyjson_mut_arr_add_val(sk_files, fe); + } + sk_n++; + } + } + cbm_store_free_coverage(rows, count); + + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, pp, "files", pp_files); + yyjson_mut_obj_add_int(doc, pp, "count", pp_n); + yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); + + yyjson_mut_val *sk = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, sk, "files", sk_files); + yyjson_mut_obj_add_int(doc, sk, "count", sk_n); + yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "skipped", sk); + + yyjson_mut_obj_add_str( + doc, root, "note", + "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " + "but constructs inside the listed line ranges (1-based) MAY be missing from the graph " + "(tree-sitter error recovery still salvages some). skipped files were not indexed at " + "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " + "are NOT guaranteed to be fully indexed."); + } else { + yyjson_mut_obj_add_str(doc, root, "status", "no_project"); + } + + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(project); + + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; +} + /* delete_project: just erase the .db file (and WAL/SHM). */ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { char *name = get_project_arg(args); @@ -3396,9 +3517,10 @@ static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_obj_add_str(doc, pp, "note", "Best-effort signal, not a completeness guarantee: these files WERE " "indexed, but constructs inside the listed line ranges (1-based) could " - "not be parsed and are missing from the graph. Prefer text search " - "(grep) for those regions. Files absent from this list are NOT " - "guaranteed to be fully indexed."); + "not be parsed and MAY be missing from the graph (tree-sitter error " + "recovery still salvages some). Prefer text search (grep) for those " + "regions. Files absent from this list are NOT guaranteed to be fully " + "indexed. Query the persisted signal via get_index_coverage."); yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); } @@ -5856,6 +5978,9 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch if (strcmp(tool_name, "query_graph") == 0) { return handle_query_graph(srv, args_json); } + if (strcmp(tool_name, "get_index_coverage") == 0) { + return handle_get_index_coverage(srv, args_json); + } if (strcmp(tool_name, "index_status") == 0) { return handle_index_status(srv, args_json); } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index dbce7198b..f72b50e13 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -315,40 +315,6 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou } } -void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) { - if (!p || !gbuf) { - return; - } - int stamped = 0; - for (int i = 0; i < p->file_errors_count; i++) { - const cbm_file_error_t *e = &p->file_errors[i]; - if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) { - continue; - } - char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__"); - if (!file_qn) { - continue; - } - const char *slash = strrchr(e->path, '/'); - const char *basename = slash ? slash + SKIP_ONE : e->path; - const char *ext = strrchr(basename, '.'); - /* Full replacement props (set_node_props does not merge): keep the - * "extension" key pass_structure wrote, add the coverage marker. The - * ranges string is digits/commas/dashes — JSON-safe unescaped. */ - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), - "{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}", - ext ? ext : "", e->reason ? e->reason : ""); - if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) { - stamped++; - } - free(file_qn); - } - if (stamped > 0) { - cbm_log_info("index.parse_partial", "files", itoa_buf(stamped)); - } -} - void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) { if (nodes) { *nodes = p ? p->committed_nodes : -1; @@ -1148,6 +1114,29 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil } CBM_PROF_END_N("persist", "4_file_hashes", t_fh, file_count); + /* Coverage rows (#963): a full run's file_errors is the complete + * coverage truth for the project. The dump recreated the DB file, so + * the separate index_coverage table starts empty — write only when + * there is something to record (AFTER hashes, so the deleted-file + * prune inside replace sees the live file set). */ + if (p->file_errors_count > 0) { + cbm_coverage_row_t *cov = + (cbm_coverage_row_t *)malloc((size_t)p->file_errors_count * sizeof(*cov)); + if (cov) { + for (int i = 0; i < p->file_errors_count; i++) { + cov[i].rel_path = p->file_errors[i].path; + cov[i].kind = p->file_errors[i].phase; + cov[i].detail = p->file_errors[i].reason; + } + if (cbm_store_coverage_replace(hash_store, p->project_name, cov, + p->file_errors_count) != CBM_STORE_OK) { + cbm_log_error("pipeline.err", "phase", "persist_coverage", "project", + p->project_name); + } + free(cov); + } + } + /* FTS5 backfill: populate nodes_fts with camelCase-split names. * Contentless FTS5 requires the special 'delete-all' command instead of * DELETE FROM to wipe prior rows (there's no underlying content table). @@ -1296,11 +1285,6 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (check_cancel(p)) { return CBM_NOT_FOUND; } - if (rc == 0) { - /* Extraction merged its per-file errors — stamp parse-partial File - * nodes (#963) before downstream passes and the dump. */ - cbm_pipeline_stamp_parse_partial(p, p->gbuf); - } return rc; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4b5cedf8f..f2cc3fe79 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -631,7 +631,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - const char *repo_path) { + const char *repo_path, const cbm_coverage_row_t *cov, int cov_count) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -651,6 +651,13 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char * if (hash_store) { persist_hashes(hash_store, project, files, file_count, mode_skipped, mode_skipped_count); + /* Coverage rows (#963): re-write the merged set into the rebuilt DB + * (AFTER hashes, so the deleted-file prune sees the live file set). */ + if (cov_count > 0 && + cbm_store_coverage_replace(hash_store, project, cov, cov_count) != CBM_STORE_OK) { + cbm_log_error("incremental.err", "msg", "persist_coverage", "project", project); + } + /* FTS5 rebuild after incremental dump. The btree dump path bypasses * any triggers that could have kept nodes_fts synchronized, so we * rebuild from the nodes table here. See the full-dump path in @@ -729,6 +736,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_store_free_file_hashes(stored, stored_count); + /* Coverage rows (#963): the dump below rebuilds the DB file, wiping the + * separate index_coverage table — capture the previous rows now (store + * still open) so entries for files NOT re-extracted this run survive. */ + cbm_coverage_row_t *old_cov = NULL; + int old_cov_count = 0; + (void)cbm_store_coverage_get(store, project, &old_cov, &old_cov_count); + /* Build list of changed files */ cbm_file_info_t *changed_files = (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL; @@ -762,6 +776,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } free(deleted); free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_free_coverage(old_cov, old_cov_count); cbm_store_close(store); return CBM_NOT_FOUND; } @@ -840,13 +855,42 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } run_extract_resolve(&ctx, changed_files, ci); - /* Stamp parse-partial File nodes (#963) for the changed slice. A changed - * file that parses cleanly now got fresh "{}" props above, so a stale - * marker from a previous run cannot survive the reindex. */ - cbm_pipeline_stamp_parse_partial(p, existing); cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); + /* Coverage rows (#963): merge = previous rows for files NOT re-extracted + * this run + this run's fresh entries (changed files replace their old + * rows — a file that parses cleanly now simply contributes nothing, so + * its stale flag dies here). Rows for deleted files are pruned against + * file_hashes inside the replace. Borrowed strings: old_cov and the + * pipeline own them past the dump_and_persist call below. */ + cbm_file_error_t *run_errs = NULL; + int run_err_count = 0; + cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); + cbm_coverage_row_t *cov = NULL; + int cov_n = 0; + if (old_cov_count + run_err_count > 0) { + cov = (cbm_coverage_row_t *)malloc((size_t)(old_cov_count + run_err_count) * sizeof(*cov)); + } + if (cov) { + CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); + for (int i = 0; i < ci; i++) { + cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]); + } + for (int i = 0; i < old_cov_count; i++) { + if (old_cov[i].rel_path && !cbm_ht_get(changed_set, old_cov[i].rel_path)) { + cov[cov_n++] = old_cov[i]; + } + } + cbm_ht_free(changed_set); + for (int i = 0; i < run_err_count; i++) { + cov[cov_n].rel_path = run_errs[i].path; + cov[cov_n].kind = run_errs[i].phase; + cov[cov_n].detail = run_errs[i].reason; + cov_n++; + } + } + free(changed_files); cbm_registry_free(registry); cbm_path_alias_collection_free(path_aliases); @@ -871,7 +915,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); dump_and_persist(existing, db_path, project, files, file_count, mode_skipped, - mode_skipped_count, cbm_pipeline_repo_path(p)); + mode_skipped_count, cbm_pipeline_repo_path(p), cov, cov_n); + free(cov); + cbm_store_free_coverage(old_cov, old_cov_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 1648bddfe..a3d806558 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -520,14 +520,6 @@ void cbm_pipeline_create_route_nodes(cbm_gbuf_t *gb); /* ── Pass function prototypes ────────────────────────────────────── */ -/* Stamp File nodes for files recorded under phase "parse_partial" (#963): - * sets properties {"extension":...,"parse_incomplete":true,"error_ranges":...} - * on the (already-existing) File node so the coverage signal is queryable in - * the graph. Call after the extraction pass has merged its per-file errors — - * both the full path (pipeline.c) and the incremental path - * (pipeline_incremental.c) must call it. Implementation: pipeline.c. */ -void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf); - int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); diff --git a/src/store/store.c b/src/store/store.c index 33a9453c2..3e1735cac 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -269,6 +269,19 @@ static int init_schema(cbm_store_t *s) { " source_hash TEXT NOT NULL," " created_at TEXT NOT NULL," " updated_at TEXT NOT NULL" + ");" + /* Best-effort indexing-coverage signal (#963). One row per file the + * indexer could not fully cover: kind "parse_partial" (indexed, but the + * parse tree had ERROR/MISSING regions — detail = 1-based line ranges) + * or a skip phase ("read"/"extract"/"oversized" — detail = reason). + * Deliberately SEPARATE from the graph tables: coverage is metadata + * about the graph, not part of it. */ + "CREATE TABLE IF NOT EXISTS index_coverage (" + " project TEXT NOT NULL," + " rel_path TEXT NOT NULL," + " kind TEXT NOT NULL," + " detail TEXT DEFAULT ''," + " PRIMARY KEY (project, rel_path, kind)" ");"; int rc = exec_sql(s, ddl); @@ -1833,6 +1846,248 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project) { return CBM_STORE_OK; } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project) { + snprintf(dst, dstsz, "%s::coverage", project); +} + +/* Minimal JSON string escape (quotes, backslashes, control chars → space). */ +static void cov_json_escape(char *dst, size_t dstsz, const char *src) { + size_t o = 0; + for (const char *p = src; *p && o + 2 < dstsz; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') { + dst[o++] = '\\'; + dst[o++] = (char)c; + } else if (c < 0x20) { + dst[o++] = ' '; + } else { + dst[o++] = (char)c; + } + } + dst[o] = '\0'; +} + +/* Rebuild the derived miss-GRAPH view under the shadow project + * "::coverage": ONLY the not-fully-indexed files, laid out as the + * file structure (Project → Folder chain → File), each File carrying + * {kind, detail}. Queryable via query_graph(graph="coverage") with zero + * cypher-engine changes; the real project's graph gains no rows. Derived + * data — rebuilt from the authoritative (post-prune) table contents. */ +static void cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) { + char covproj[CBM_SZ_512]; + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + + /* Wipe the previous view (edges first: no FK pragma guarantee). */ + static const char *wipes[] = {"DELETE FROM edges WHERE project = ?1;", + "DELETE FROM nodes WHERE project = ?1;"}; + for (int w = 0; w < 2; w++) { + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, wipes[w], CBM_NOT_FOUND, &del, NULL) == SQLITE_OK) { + bind_text(del, SKIP_ONE, covproj); + (void)sqlite3_step(del); + sqlite3_finalize(del); + } + } + + cbm_coverage_row_t *rows = NULL; + int count = 0; + if (cbm_store_coverage_get(s, project, &rows, &count) != CBM_STORE_OK || count == 0) { + cbm_store_free_coverage(rows, count); + return; + } + + /* nodes.project has an FK to projects(name) (enforced: foreign_keys=ON), + * so the shadow project needs its row. Invisible to list_projects, which + * scans the cache directory for .db files, not this table. */ + if (cbm_store_upsert_project(s, covproj, "") != CBM_STORE_OK) { + cbm_store_free_coverage(rows, count); + return; + } + + cbm_node_t root = {.project = covproj, + .label = "Project", + .name = project, + .qualified_name = covproj, + .properties_json = "{}"}; + int64_t root_id = cbm_store_upsert_node(s, &root); + + for (int i = 0; i < count; i++) { + const char *rel = rows[i].rel_path; + if (!rel || !rel[0] || root_id <= 0) { + continue; + } + char pathbuf[CBM_SZ_1K]; + snprintf(pathbuf, sizeof(pathbuf), "%s", rel); + + int64_t parent = root_id; + for (char *p = pathbuf; *p; p++) { + if (*p != '/') { + continue; + } + /* Truncate at this slash → pathbuf is the directory prefix; the + * upsert binds copies, so restore the slash right after. */ + *p = '\0'; + const char *seg = strrchr(pathbuf, '/'); + cbm_node_t folder = {.project = covproj, + .label = "Folder", + .name = seg ? seg + 1 : pathbuf, + .qualified_name = pathbuf, + .file_path = pathbuf, + .properties_json = "{}"}; + int64_t fid = cbm_store_upsert_node(s, &folder); + *p = '/'; + if (fid > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = fid, + .type = "CONTAINS_FOLDER", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + parent = fid; + } + } + + const char *base = strrchr(rel, '/'); + char props[CBM_SZ_2K]; + char detail_esc[CBM_SZ_1K]; + cov_json_escape(detail_esc, sizeof(detail_esc), rows[i].detail ? rows[i].detail : ""); + snprintf(props, sizeof(props), "{\"kind\":\"%s\",\"detail\":\"%s\"}", + rows[i].kind ? rows[i].kind : "", detail_esc); + cbm_node_t file = {.project = covproj, + .label = "File", + .name = base ? base + 1 : rel, + .qualified_name = rel, + .file_path = rel, + .properties_json = props}; + int64_t file_id = cbm_store_upsert_node(s, &file); + if (file_id > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = file_id, + .type = "CONTAINS_FILE", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + } + } + cbm_store_free_coverage(rows, count); +} + +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count) { + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND, + &del, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage delete prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + bind_text(del, SKIP_ONE, project); + int rc = sqlite3_step(del); + sqlite3_finalize(del); + if (rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage delete"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_stmt *ins = NULL; + if (sqlite3_prepare_v2( + s->db, + "INSERT OR REPLACE INTO index_coverage (project, rel_path, kind, detail) " + "VALUES (?1, ?2, ?3, ?4);", + CBM_NOT_FOUND, &ins, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage insert prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + for (int i = 0; i < count; i++) { + if (!rows[i].rel_path || !rows[i].kind) { + continue; + } + bind_text(ins, SKIP_ONE, project); + bind_text(ins, ST_COL_2, rows[i].rel_path); + bind_text(ins, ST_COL_3, rows[i].kind); + bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : ""); + if (sqlite3_step(ins) != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage insert"); + sqlite3_finalize(ins); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_reset(ins); + } + sqlite3_finalize(ins); + /* Prune rows for files no longer known to the index (deleted from the + * repo): file_hashes is the authoritative live-file set after persist. */ + sqlite3_stmt *prune = NULL; + if (sqlite3_prepare_v2(s->db, + "DELETE FROM index_coverage WHERE project = ?1 AND rel_path NOT IN " + "(SELECT rel_path FROM file_hashes WHERE project = ?1);", + CBM_NOT_FOUND, &prune, NULL) == SQLITE_OK) { + bind_text(prune, SKIP_ONE, project); + (void)sqlite3_step(prune); + sqlite3_finalize(prune); + } + /* Rebuild the derived miss-graph view from the now-authoritative table + * contents (same transaction — the table and its view stay in step). */ + cov_rebuild_shadow_graph(s, project); + return exec_sql(s, "COMMIT;"); +} + +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, + "SELECT rel_path, kind, detail FROM index_coverage " + "WHERE project = ?1 ORDER BY rel_path, kind;", + CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage get prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + int cap = ST_INIT_CAP_16; + int n = 0; + cbm_coverage_row_t *arr = malloc(cap * sizeof(cbm_coverage_row_t)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + if (n >= cap) { + cap *= ST_GROWTH; + arr = safe_realloc(arr, cap * sizeof(cbm_coverage_row_t)); + } + arr[n].rel_path = heap_strdup((const char *)sqlite3_column_text(stmt, 0)); + arr[n].kind = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE)); + arr[n].detail = heap_strdup((const char *)sqlite3_column_text(stmt, CBM_SZ_2)); + n++; + } + sqlite3_finalize(stmt); + *out = arr; + *count = n; + return CBM_STORE_OK; +} + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count) { + if (!rows) { + return; + } + for (int i = 0; i < count; i++) { + free((char *)rows[i].rel_path); + free((char *)rows[i].kind); + free((char *)rows[i].detail); + } + free(rows); +} + /* ── FindNodesByFileOverlap ─────────────────────────────────────── */ int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, const char *file_path, diff --git a/src/store/store.h b/src/store/store.h index a4541e2df..e83be73e4 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -398,6 +398,40 @@ int cbm_store_delete_file_hash(cbm_store_t *s, const char *project, const char * int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project); +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* One best-effort coverage row: a file the indexer could not fully cover. + * kind "parse_partial" = indexed but the parse tree had ERROR/MISSING regions + * (detail = 1-based line ranges "12-40,88-90"); skip kinds "read"/"extract"/ + * "oversized" = not indexed at all (detail = reason). Stored in the separate + * index_coverage table — coverage is metadata ABOUT the graph, never mixed + * into the graph itself. */ +typedef struct { + const char *rel_path; + const char *kind; + const char *detail; +} cbm_coverage_row_t; + +/* Replace the project's coverage rows in one transaction, then prune rows for + * files absent from file_hashes (deleted from the repo). Call AFTER hashes + * were persisted for the run. */ +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count); + +/* Fetch all coverage rows (ordered by rel_path). Caller frees via + * cbm_store_free_coverage. */ +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count); + +/* Name of the derived miss-graph shadow project ("::coverage"). + * cbm_store_coverage_replace materializes the coverage rows as a file- + * structure graph (Project → Folder → File{kind, detail}) under this project + * name — queryable via the normal cypher path without touching the real + * project's graph. */ +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project); + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count); + /* ── Search ─────────────────────────────────────────────────────── */ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out); diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index db8cc709f..a16c7fac1 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -19,6 +19,11 @@ #include #include #include +#include + +/* Sleep before rewriting a fixture so its mtime_ns strictly increases and the + * incremental change classifier reliably sees the edit (10 ms). */ +#define INCR_FIX_SLEEP_NS 10000000L /* ── Local helpers ──────────────────────────────────────────────── */ @@ -358,35 +363,54 @@ TEST(index_parse_partial_reported) { ASSERT_NOT_NULL(strstr(logtext, "split.c")); free(logtext); - /* The File node carries the queryable marker. */ - cbm_node_t *nodes = NULL; - int count = 0; - int rc = cbm_store_find_nodes_by_qn_suffix(store, lp.project, "__file__", &nodes, &count); - ASSERT_EQ(rc, CBM_STORE_OK); + /* The signal is persisted in the SEPARATE index_coverage table (never + * mixed into the graph tables) and queryable via get_index_coverage, + * exactly as the index_repository tool description advertises. */ + cbm_coverage_row_t *rows = NULL; + int cov_count = 0; + ASSERT_EQ(cbm_store_coverage_get(store, lp.project, &rows, &cov_count), CBM_STORE_OK); int marked = 0; - for (int i = 0; i < count; i++) { - if (nodes[i].file_path && strstr(nodes[i].file_path, "split.c")) { - ASSERT_NOT_NULL(nodes[i].properties_json); - ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"parse_incomplete\":true")); - ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"error_ranges\":\"")); + for (int i = 0; i < cov_count; i++) { + if (rows[i].rel_path && strstr(rows[i].rel_path, "split.c")) { + ASSERT_NOT_NULL(rows[i].kind); + ASSERT_STR_EQ("parse_partial", rows[i].kind); + ASSERT_NOT_NULL(rows[i].detail); + ASSERT_GT((int)strlen(rows[i].detail), 0); marked = 1; } } - cbm_store_free_nodes(nodes, count); + cbm_store_free_coverage(rows, cov_count); ASSERT_TRUE(marked); - /* The marker is queryable exactly as the index_repository tool description - * advertises (agents are pointed at this query to find unindexed code). */ char qargs[900]; - snprintf(qargs, sizeof(qargs), - "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.parse_incomplete = true " - "RETURN f.file_path, f.error_ranges\"}", - lp.project); - char *qresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *qresp = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); ASSERT_NOT_NULL(qresp); ASSERT_NOT_NULL(strstr(qresp, "split.c")); + ASSERT_NOT_NULL(strstr(qresp, "parse_partial")); free(qresp); + /* The miss GRAPH: the same signal is queryable as a file-structure graph + * via query_graph(graph="coverage") — exactly the query the tool + * description advertises — and it lives under the shadow project, so the + * REAL code graph gained no coverage rows. */ + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"graph\":\"coverage\",\"query\":\"MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail\"}", + lp.project); + char *gresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(gresp); + ASSERT_NOT_NULL(strstr(gresp, "split.c")); + free(gresp); + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.kind = " + "\\\"parse_partial\\\" RETURN f.file_path\"}", + lp.project); + char *cresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(cresp); + ASSERT_NULL(strstr(cresp, "split.c")); /* code graph: no coverage rows */ + free(cresp); + /* Clean neighbors still extract. */ int funcs = rh_count_label(store, lp.project, "Function"); ASSERT_GTE(funcs, 1); @@ -397,8 +421,70 @@ TEST(index_parse_partial_reported) { PASS(); } +/* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must + * stay FRESH — after the broken file is fixed and the project re-indexed + * (incremental route: the DB already exists), its parse_partial row is gone + * and get_index_coverage reports it no longer. A stale flag on a fixed file + * would make the whole signal untrustworthy. */ +TEST(index_parse_partial_clears_on_fix) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + + ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef broken(:\n pass\n"); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char *resp = NULL; + cbm_store_t *store = ri_index_capture(&lp, &resp); + if (!resp) { + FAIL("no MCP response"); + } + free(resp); + if (!store) { + FAIL("store did not open"); + } + cbm_store_close(store); + + /* Flagged after the first (full) index. */ + char qargs[900]; + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *cov1 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + ASSERT_NOT_NULL(cov1); + ASSERT_NOT_NULL(strstr(cov1, "flaky.py")); + free(cov1); + + /* Fix the file; ensure a newer mtime so change detection can't miss it. */ + struct timespec ts = {0, INCR_FIX_SLEEP_NS}; + nanosleep(&ts, NULL); + ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef fixed():\n return 2\n"); + + /* Re-index WITHOUT deleting the DB → routes through the incremental path. */ + char iargs[700]; + snprintf(iargs, sizeof(iargs), "{\"repo_path\":\"%s\"}", lp.tmpdir); + char *resp2 = cbm_mcp_handle_tool(lp.srv, "index_repository", iargs); + ASSERT_NOT_NULL(resp2); + free(resp2); + + char *cov2 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + ASSERT_NOT_NULL(cov2); + ASSERT_NULL(strstr(cov2, "flaky.py")); + free(cov2); + + store = cbm_store_open_path(lp.dbpath); + if (!store) { + FAIL("store did not reopen"); + } + rh_cleanup(&lp, store); + PASS(); +} + SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); RUN_TEST(index_parse_partial_reported); + RUN_TEST(index_parse_partial_clears_on_fix); } diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 5b30ef9f5..46a864841 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1593,7 +1593,70 @@ TEST(store_count_nodes_unknown_project) { PASS(); } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* Round-trip + deleted-file prune + shadow miss-graph materialization + + * empty-replace wipe. The prune keys off file_hashes (the live-file set), so + * a row for a file with no hash row must not survive the replace. */ +TEST(store_coverage_roundtrip_prune_shadow) { + cbm_store_t *s = cbm_store_open_memory(); + cbm_store_upsert_project(s, "test", "/tmp/test"); + cbm_store_upsert_file_hash(s, "test", "src/a.py", "", 1, 10); + + cbm_coverage_row_t rows[] = { + {.rel_path = "src/a.py", .kind = "parse_partial", .detail = "4-7"}, + {.rel_path = "gone.py", .kind = "oversized", .detail = "too big"}, + }; + ASSERT_EQ(cbm_store_coverage_replace(s, "test", rows, 2), CBM_STORE_OK); + + cbm_coverage_row_t *got = NULL; + int n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); /* gone.py pruned — no file_hashes row */ + ASSERT_STR_EQ(got[0].rel_path, "src/a.py"); + ASSERT_STR_EQ(got[0].kind, "parse_partial"); + ASSERT_STR_EQ(got[0].detail, "4-7"); + cbm_store_free_coverage(got, n); + + /* Shadow miss-graph materialized under "test::coverage": + * Project → Folder(src) → File(a.py){kind,detail}. */ + cbm_node_t *nodes = NULL; + int nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].file_path, "src/a.py"); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"kind\":\"parse_partial\"")); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"detail\":\"4-7\"")); + cbm_store_free_nodes(nodes, nc); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "Folder", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].name, "src"); + cbm_store_free_nodes(nodes, nc); + + /* Empty replace clears the table AND wipes the shadow graph. */ + ASSERT_EQ(cbm_store_coverage_replace(s, "test", NULL, 0), CBM_STORE_OK); + got = NULL; + n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + free(got); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 0); + cbm_store_free_nodes(nodes, nc); + + cbm_store_close(s); + PASS(); +} + SUITE(store_nodes) { + RUN_TEST(store_coverage_roundtrip_prune_shadow); RUN_TEST(store_open_memory); RUN_TEST(store_close_null); RUN_TEST(store_open_memory_twice); From 664313e4ac79f185723cd74c1d6ba3d19a4587e8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 8 Jul 2026 19:13:25 +0200 Subject: [PATCH 3/8] feat(index): missed graph in UI + read-time coverage hook + recovery subtraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four refinements to the coverage signal: - extraction: subtract DEFINITE recovery before flagging. Tree-sitter error recovery plus the ERROR-descending def walker often re-extract constructs inside a failed region (verified: `def broken(:` comes back as a def); a region whose every line is covered by definitions that START inside it is not a miss, and a fully recovered file is not flagged at all. Container defs (Module/Package) don't count as evidence, and partially covered regions stay flagged — the #ifdef-split case keeps its flag because the first branch's function is genuinely lost - naming: the query_graph option is graph="missed" (the graph shows ONLY misses — "coverage" was misleading); shadow project renamed to "::missed"; tool descriptions updated so agents discover both the option and its semantics - hook: the CLI-installed PreToolUse augmenter now also matches Read and injects a coverage note when the file being read is listed as not fully indexed ("line ranges X-Y could not be parsed — the file content you are reading is ground truth"). Safe against the old issue-362 hazard: the augmenter is structurally non-blocking (always exit 0, additionalContext only), mirroring the Gemini matcher that already includes read_file; matcher upgrade bookkeeping updated - ui: "Missed files" toggle in the graph sidebar renders the miss graph as a second graph option — /api/layout gains graph=missed (same db file, shadow-project scoping; base project name validated as before) Tests: recovery-subtraction cases (recovered def unflagged, garbage region flagged, trailing recovered defs keep the flag), CLI matcher tests updated, e2e resilience fixtures switched to an unrecovered miss; verified end-to-end: hook emits the note for a flagged file and stays silent otherwise, /api/layout?graph=missed serves the miss graph and the code graph is unchanged, frontend build + tests green. Refs #963 Signed-off-by: Martin Vogel --- graph-ui/src/components/FilterPanel.tsx | 26 ++++ graph-ui/src/components/GraphTab.tsx | 11 +- graph-ui/src/hooks/useGraphData.ts | 16 ++- internal/cbm/cbm.c | 102 +++++++++++--- src/cli/cli.c | 13 +- src/cli/hook_augment.c | 180 +++++++++++++++++++++++- src/mcp/mcp.c | 24 ++-- src/store/store.c | 6 +- src/store/store.h | 2 +- src/ui/http_server.c | 18 ++- tests/test_cli.c | 24 ++-- tests/test_index_resilience.c | 30 +++- tests/test_parse_coverage.c | 92 +++++++++--- tests/test_store_nodes.c | 10 +- 14 files changed, 464 insertions(+), 90 deletions(-) diff --git a/graph-ui/src/components/FilterPanel.tsx b/graph-ui/src/components/FilterPanel.tsx index 214a2b835..db2d0a513 100644 --- a/graph-ui/src/components/FilterPanel.tsx +++ b/graph-ui/src/components/FilterPanel.tsx @@ -22,6 +22,9 @@ interface FilterPanelProps { onToggleShowOnlyDead: () => void; onToggleHideEntryPoints: () => void; onToggleHideTests: () => void; + /* Missed graph (#963): file structure of not-fully-indexed files */ + missedView: boolean; + onToggleMissedView: () => void; } /* Checkbox row matching the existing "Show labels" toggle style */ @@ -76,6 +79,8 @@ export function FilterPanel({ onToggleShowOnlyDead, onToggleHideEntryPoints, onToggleHideTests, + missedView, + onToggleMissedView, }: FilterPanelProps) { const { labelCounts, edgeTypeCounts, statusCounts } = useMemo(() => { const lc = new Map(); @@ -163,6 +168,27 @@ export function FilterPanel({ + {/* Missed graph (#963): swaps the layout to the file structure of files + the indexer could not fully cover (best-effort signal). */} +
+
+ + Graph + +
+ + {missedView && ( +

+ Best-effort: files with unparseable regions or skips. Empty = no + known misses (not a completeness guarantee). +

+ )} +
+ {/* Dead-code view */}
diff --git a/graph-ui/src/components/GraphTab.tsx b/graph-ui/src/components/GraphTab.tsx index 75f32a700..56f549cd2 100644 --- a/graph-ui/src/components/GraphTab.tsx +++ b/graph-ui/src/components/GraphTab.tsx @@ -102,6 +102,11 @@ export function GraphTab({ project }: GraphTabProps) { const [enabledLabels, setEnabledLabels] = useState>(new Set()); const [enabledEdgeTypes, setEnabledEdgeTypes] = useState>(new Set()); + /* Missed graph (#963): render the file structure of files the indexer + * could not fully cover instead of the code graph. Server-side option + * (`graph=missed` on /api/layout) — toggling refetches. */ + const [missedView, setMissedView] = useState(false); + /* Dead-code view: recolor by status + status-based filters */ const [deadCodeView, setDeadCodeView] = useState(false); const [showOnlyDead, setShowOnlyDead] = useState(false); @@ -184,11 +189,11 @@ export function GraphTab({ project }: GraphTabProps) { /* …and fetch only once budget and project agree (one fetch per change). */ useEffect(() => { if (project && budget.project === project) { - fetchOverview(project, budget.value); + fetchOverview(project, budget.value, missedView ? "missed" : "code"); setHighlightedIds(null); setSelectedPath(null); } - }, [project, budget, fetchOverview]); + }, [project, budget, missedView, fetchOverview]); /* Fetch git remote metadata for GitHub deep-links */ useEffect(() => { @@ -351,6 +356,8 @@ export function GraphTab({ project }: GraphTabProps) { onToggleShowOnlyDead={() => setShowOnlyDead((v) => !v)} onToggleHideEntryPoints={() => setHideEntryPoints((v) => !v)} onToggleHideTests={() => setHideTests((v) => !v)} + missedView={missedView} + onToggleMissedView={() => setMissedView((v) => !v)} /> void; + fetchOverview: ( + project: string, + maxNodes?: number, + graph?: "code" | "missed", + ) => void; fetchDetail: (project: string, centerNode: string) => void; } @@ -32,12 +36,18 @@ export function clampNodeBudget(value: number): number { return stepped; } +/** Which graph to lay out: the code graph (default) or the missed graph — + * only files the indexer could not fully cover, as their file structure. */ +export type GraphVariant = "code" | "missed"; + export async function fetchLayout( project: string, maxNodes = GRAPH_RENDER_NODE_LIMIT, onProgress?: (progress: LoadProgress) => void, + graph: GraphVariant = "code", ): Promise { const params = new URLSearchParams({ project, max_nodes: String(maxNodes) }); + if (graph === "missed") params.set("graph", "missed"); const res = await fetch(`/api/layout?${params}`); if (!res.ok) { @@ -83,12 +93,12 @@ export function useGraphData(): UseGraphDataResult { const [progress, setProgress] = useState(NO_PROGRESS); const fetchOverview = useCallback( - async (project: string, maxNodes?: number) => { + async (project: string, maxNodes?: number, graph: GraphVariant = "code") => { setLoading(true); setError(null); setProgress(NO_PROGRESS); try { - const result = await fetchLayout(project, maxNodes, setProgress); + const result = await fetchLayout(project, maxNodes, setProgress, graph); setData(result); } catch (e) { setError(e instanceof Error ? e.message : "Failed to fetch layout"); diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index c1becbfe2..5e1dd6b05 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -725,6 +725,72 @@ static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) { } } +/* Recovery subtraction (#963): tree-sitter error recovery plus the + * ERROR-descending def walker often still extract constructs INSIDE a failed + * region (verified: a function in an #ifdef-split ERROR region and even a + * `def broken(:` both came back as defs). A region whose every line is + * covered by definitions that START inside it is definitely recovered — its + * constructs ARE in the graph — so flagging it would be a false miss. + * Container defs (Module/Package) are ignored: a file-spanning Module node is + * not evidence the region's constructs survived. Conservative: partially + * covered regions stay flagged. */ +static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray *defs) { + enum { MAX_COVER_DEFS = 256 }; + uint32_t starts[MAX_COVER_DEFS]; + uint32_t ends[MAX_COVER_DEFS]; + int n = 0; + for (int i = 0; i < defs->count && n < MAX_COVER_DEFS; i++) { + const CBMDefinition *d = &defs->items[i]; + if (!d->label || strcmp(d->label, "Module") == 0 || strcmp(d->label, "Package") == 0) { + continue; + } + if (d->start_line < rs || d->start_line > re) { + continue; /* recovery evidence must originate inside the region */ + } + starts[n] = d->start_line; + ends[n] = d->end_line < d->start_line ? d->start_line : d->end_line; + n++; + } + if (n == 0) { + return false; + } + /* Insertion-sort by start, then sweep for gaps in [rs, re]. */ + for (int i = 1; i < n; i++) { + uint32_t s = starts[i]; + uint32_t e = ends[i]; + int j = i - 1; + while (j >= 0 && starts[j] > s) { + starts[j + 1] = starts[j]; + ends[j + 1] = ends[j]; + j--; + } + starts[j + 1] = s; + ends[j + 1] = e; + } + uint32_t covered_to = rs - 1; + for (int i = 0; i < n; i++) { + if (starts[i] > covered_to + 1) { + return false; /* uncovered gap */ + } + if (ends[i] > covered_to) { + covered_to = ends[i]; + } + } + return covered_to >= re; +} + +static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs) { + int kept = 0; + for (int i = 0; i < regs->count; i++) { + if (!cbm_region_is_recovered(regs->starts[i], regs->ends[i], defs)) { + regs->starts[kept] = regs->starts[i]; + regs->ends[kept] = regs->ends[i]; + kept++; + } + } + regs->count = kept; +} + /* Serialize collected regions as "start-end,start-end,..." into the arena. */ static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) { if (regs->count <= 0) { @@ -843,22 +909,6 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, TSNode root = ts_tree_root_node(tree); - /* Best-effort parse-coverage signal (#963): flag files whose tree contains - * ERROR/MISSING nodes — constructs inside those regions are silently absent - * from the graph. Detection aid only: the absence of this flag is NOT a - * completeness guarantee. */ - if (ts_node_has_error(root)) { - result->parse_incomplete = true; - cbm_error_regions_t regs = {{0}, {0}, 0}; - if (strcmp(ts_node_type(root), "ERROR") == 0) { - cbm_error_regions_push(®s, root); /* whole file unparseable */ - } else { - cbm_collect_error_regions(root, ®s); - } - result->error_region_count = regs.count; - result->error_ranges = cbm_error_ranges_str(a, ®s); - } - // Compute module QN. Java/Go derive the module from the CONTAINING // DIRECTORY (package semantics) rather than baking the filename stem in, // so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all @@ -1108,6 +1158,26 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, uint64_t t2 = now_ns(); + /* Best-effort parse-coverage signal (#963): flag files whose tree contains + * ERROR/MISSING regions. Computed AFTER extraction so definite recovery is + * subtracted first — a region fully re-extracted as definitions is not a + * miss, and a fully recovered file is not flagged at all. Detection aid + * only: the absence of this flag is NOT a completeness guarantee. */ + if (ts_node_has_error(root)) { + cbm_error_regions_t regs = {{0}, {0}, 0}; + if (strcmp(ts_node_type(root), "ERROR") == 0) { + cbm_error_regions_push(®s, root); /* whole file unparseable */ + } else { + cbm_collect_error_regions(root, ®s); + } + cbm_subtract_recovered_regions(®s, &result->defs); + if (regs.count > 0) { + result->parse_incomplete = true; + result->error_region_count = regs.count; + result->error_ranges = cbm_error_ranges_str(a, ®s); + } + } + result->imports_count = result->imports.count; // Accumulate profiling counters diff --git a/src/cli/cli.c b/src/cli/cli.c index a33c4b333..ebfc6c079 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1789,10 +1789,13 @@ int cbm_remove_junie_mcp(const char *config_path) { /* ── Claude Code pre-tool hooks ───────────────────────────────── */ -/* Matcher intentionally excludes Read: gating Read breaks Claude Code's - * read-before-edit invariant (issue #362). The hook is a non-blocking - * augmenter, never a gate. */ -#define CMM_HOOK_MATCHER "Grep|Glob" +/* Matcher includes Read for the indexing-coverage note (#963): when the agent + * reads a file the indexer could not fully cover, the hook injects a warning + * as additionalContext. The issue-#362 hazard (a GATING hook denying Read and + * breaking the read-before-edit invariant) cannot recur: the augmenter is + * structurally non-blocking — it always exits 0 and only ever ADDS context — + * mirroring the Gemini matcher, which already includes read_file. */ +#define CMM_HOOK_MATCHER "Grep|Glob|Read" /* Basename only; the full command path is resolved at install time via * cbm_resolve_hook_command so $CLAUDE_CONFIG_DIR is honored. */ #define CMM_HOOK_GATE_SCRIPT "cbm-code-discovery-gate" @@ -1805,7 +1808,7 @@ int cbm_remove_junie_mcp(const char *config_path) { * Per-agent lists (no shared global): each caller passes its own. */ static const char *const cmm_claude_old_matchers[] = { "Grep|Glob|Read|Search", - "Grep|Glob|Read", + "Grep|Glob", /* pre-#963 matcher — Read re-added for the coverage note */ NULL, }; static const char *const cmm_gemini_old_matchers[] = { diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index 31fcaec0d..dd212caf7 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -225,6 +225,153 @@ static char *ha_format_context(const char *envelope, const char *token, bool *is return text; } +/* ── Read coverage note (#963) ──────────────────────────────────── + * For Read calls: if the file being read is listed in the project's + * index_coverage table (parse_partial or a skip), inject a note so the agent + * knows the knowledge graph may under-report this file. Best-effort and + * non-blocking like everything else here — no entry, no output. */ + +/* Parse a get_index_coverage envelope and return a note when `rel` is listed. + * *is_error is set for MCP errors (project not indexed) → caller climbs. */ +static char *ha_coverage_context(const char *envelope, const char *rel, bool *is_error) { + *is_error = false; + yyjson_doc *edoc = yyjson_read(envelope, strlen(envelope), 0); + if (!edoc) { + return NULL; + } + yyjson_val *eroot = yyjson_doc_get_root(edoc); + yyjson_val *err = yyjson_obj_get(eroot, "isError"); + if (err && yyjson_is_true(err)) { + *is_error = true; + yyjson_doc_free(edoc); + return NULL; + } + yyjson_val *content = yyjson_obj_get(eroot, "content"); + yyjson_val *item0 = (content && yyjson_is_arr(content)) ? yyjson_arr_get(content, 0) : NULL; + const char *inner = ha_obj_str(item0, "text"); + if (!inner) { + yyjson_doc_free(edoc); + return NULL; + } + yyjson_doc *idoc = yyjson_read(inner, strlen(inner), 0); + if (!idoc) { + yyjson_doc_free(edoc); + return NULL; + } + yyjson_val *iroot = yyjson_doc_get_root(idoc); + char *text = NULL; + + yyjson_val *pp = yyjson_obj_get(iroot, "parse_partial"); + yyjson_val *files = pp ? yyjson_obj_get(pp, "files") : NULL; + size_t idx; + size_t maxn; + yyjson_val *fe; + if (files && yyjson_is_arr(files)) { + yyjson_arr_foreach(files, idx, maxn, fe) { + const char *fp = ha_obj_str(fe, "path"); + if (fp && strcmp(fp, rel) == 0) { + const char *ranges = ha_obj_str(fe, "error_ranges"); + text = malloc(1024); + if (text) { + snprintf(text, 1024, + "[codebase-memory] Coverage note: this file was only PARTIALLY " + "indexed — line range(s) %s could not be parsed, so constructs " + "there may be missing from the knowledge graph. The file content " + "you are reading is ground truth; graph queries may under-report " + "this file. (best-effort signal)", + ranges && ranges[0] ? ranges : "?"); + } + break; + } + } + } + if (!text) { + yyjson_val *sk = yyjson_obj_get(iroot, "skipped"); + files = sk ? yyjson_obj_get(sk, "files") : NULL; + if (files && yyjson_is_arr(files)) { + yyjson_arr_foreach(files, idx, maxn, fe) { + const char *fp = ha_obj_str(fe, "path"); + if (fp && strcmp(fp, rel) == 0) { + const char *phase = ha_obj_str(fe, "phase"); + const char *reason = ha_obj_str(fe, "reason"); + text = malloc(1024); + if (text) { + snprintf(text, 1024, + "[codebase-memory] Coverage note: this file was NOT indexed " + "(%s%s%s) — the knowledge graph has no data for it. " + "(best-effort signal)", + phase ? phase : "skipped", reason && reason[0] ? ": " : "", + reason ? reason : ""); + } + break; + } + } + } + } + yyjson_doc_free(idoc); + yyjson_doc_free(edoc); + return text; +} + +/* Strip the last path component in place. Returns false at a filesystem or + * drive root (nothing left to strip). */ +static bool ha_strip_last_component(char *dir) { + char *slash = strrchr(dir, '/'); + if (!slash || slash == dir) { + return false; /* POSIX root "/" */ + } + if (slash == dir + 2 && dir[1] == ':') { + return false; /* Windows drive root "X:/" — don't strip to "X:" */ + } + *slash = '\0'; + return true; +} + +/* Walk up from the file's parent directory to find the indexed project, then + * check whether the file (repo-relative) is listed in its coverage report. + * Mirrors ha_resolve_and_query: an MCP error means "not indexed here" → + * climb; a valid project with no entry for this file → stop, no output. */ +static char *ha_resolve_coverage(cbm_mcp_server_t *srv, const char *file_path) { + char dir[4096]; + snprintf(dir, sizeof(dir), "%s", file_path); + if (!ha_strip_last_component(dir)) { + return NULL; /* file directly at a root — nothing to resolve against */ + } + + for (int level = 0; level < HA_MAX_WALKUP && cbm_hook_path_is_abs(dir); level++) { + char *project = cbm_project_name_from_path(dir); + if (project) { + yyjson_mut_doc *adoc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *aroot = yyjson_mut_obj(adoc); + yyjson_mut_doc_set_root(adoc, aroot); + yyjson_mut_obj_add_str(adoc, aroot, "project", project); + char *args = yyjson_mut_write(adoc, 0, NULL); + yyjson_mut_doc_free(adoc); + free(project); + if (args) { + char *res = cbm_mcp_handle_tool(srv, "get_index_coverage", args); + free(args); + if (res) { + bool is_error = false; + const char *rel = file_path + strlen(dir) + 1; + char *ctx = ha_coverage_context(res, rel, &is_error); + free(res); + if (ctx) { + return ctx; /* listed → note */ + } + if (!is_error) { + return NULL; /* indexed project, file not listed → stop */ + } + } + } + } + if (!ha_strip_last_component(dir)) { + break; + } + } + return NULL; +} + /* Emit the PreToolUse additionalContext payload to stdout (exactly once). */ static void ha_emit(const char *text) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -314,13 +461,44 @@ int cbm_cmd_hook_augment(void) { yyjson_val *root = yyjson_doc_get_root(doc); const char *tool = ha_obj_str(root, "tool_name"); - if (!tool || (strcmp(tool, "Grep") != 0 && strcmp(tool, "Glob") != 0)) { + if (!tool || + (strcmp(tool, "Grep") != 0 && strcmp(tool, "Glob") != 0 && strcmp(tool, "Read") != 0)) { yyjson_doc_free(doc); free(input); return 0; } yyjson_val *tin = yyjson_obj_get(root, "tool_input"); + + /* Read → coverage note (#963): warn when the file being read is listed as + * not fully indexed. Independent of the Grep/Glob symbol augment below. */ + if (strcmp(tool, "Read") == 0) { + const char *fp = ha_obj_str(tin, "file_path"); + char fpbuf[4096]; + if (fp) { + snprintf(fpbuf, sizeof(fpbuf), "%s", fp); + for (char *p = fpbuf; *p; p++) { + if (*p == '\\') { + *p = '/'; + } + } + } + if (fp && cbm_hook_path_is_abs(fpbuf)) { + cbm_mcp_server_t *rsrv = cbm_mcp_server_new(NULL); + if (rsrv) { + char *note = ha_resolve_coverage(rsrv, fpbuf); + if (note) { + ha_emit(note); + free(note); + } + cbm_mcp_server_free(rsrv); + } + } + yyjson_doc_free(doc); + free(input); + return 0; + } + const char *pattern = ha_obj_str(tin, "pattern"); char token[HA_MAX_TOKEN + 1]; if (!ha_extract_token(pattern, token, sizeof(token))) { diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 613a133b6..37d7ad7f5 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -321,7 +321,7 @@ static const tool_def_t TOOLS[] = { "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " "constructs inside the listed line ranges could not be parsed and MAY be missing from " "the graph). Query the persisted signal any time via the get_index_coverage tool or " - "structurally via query_graph(graph=\"coverage\"). Both signals are best-effort: absence " + "structurally via query_graph(graph=\"missed\"). Both signals are best-effort: absence " "of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," @@ -395,8 +395,8 @@ static const tool_def_t TOOLS[] = { "Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE " "f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, " "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. " - "COVERAGE GRAPH: pass graph=\"coverage\" to query the best-effort miss graph instead — " - "the file structure of files the indexer could NOT fully cover (Project → Folder → " + "MISSED GRAPH: pass graph=\"missed\" to query the best-effort miss graph instead — the " + "file structure of ONLY the files the indexer could NOT fully index (Project → Folder → " "File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind " "(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; " "or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE " @@ -404,9 +404,9 @@ static const tool_def_t TOOLS[] = { "NOT a completeness guarantee.", "{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher " "query\"},\"project\":{\"type\":\"string\"}," - "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"coverage\"],\"default\":\"code\"," + "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"missed\"],\"default\":\"code\"," "\"description\":\"Which graph to query: the code knowledge graph (default) or the " - "indexing-coverage miss graph (files not fully indexed, as a file-structure graph).\"}," + "missed graph (only files not fully indexed, laid out as their file structure).\"}," "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Optional row limit. Default: unlimited up to a 100k row " @@ -507,7 +507,7 @@ static const tool_def_t TOOLS[] = { "completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). " "IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks " "what the indexer can detect. For structural queries over the misses use " - "query_graph(graph=\"coverage\").", + "query_graph(graph=\"missed\").", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, @@ -2087,20 +2087,20 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { cbm_store_t *store = resolve_store(srv, project); int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0); - /* graph="coverage" (#963): run the SAME cypher against the derived - * miss-graph view (shadow project "::coverage") instead of the + /* graph="missed" (#963): run the SAME cypher against the derived + * miss-graph view (shadow project "::missed") instead of the * code graph — file structure of not-fully-indexed files only. */ char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); - bool coverage_graph = graph_arg && strcmp(graph_arg, "coverage") == 0; + bool missed_graph = graph_arg && strcmp(graph_arg, "missed") == 0; free(graph_arg); if (!query) { free(project); return cbm_mcp_text_result("query is required", true); } - if (coverage_graph && !project) { + if (missed_graph && !project) { free(query); - return cbm_mcp_text_result("project is required when graph=\"coverage\"", true); + return cbm_mcp_text_result("project is required when graph=\"missed\"", true); } if (!store) { char *_err = build_project_list_error("project not found or not indexed"); @@ -2120,7 +2120,7 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { char covproj[CBM_SZ_512]; const char *cypher_project = project; - if (coverage_graph) { + if (missed_graph) { cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); cypher_project = covproj; } diff --git a/src/store/store.c b/src/store/store.c index 3e1735cac..f10ebee83 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1849,7 +1849,7 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project) { /* ── Index coverage (#963) ──────────────────────────────────────── */ void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project) { - snprintf(dst, dstsz, "%s::coverage", project); + snprintf(dst, dstsz, "%s::missed", project); } /* Minimal JSON string escape (quotes, backslashes, control chars → space). */ @@ -1870,9 +1870,9 @@ static void cov_json_escape(char *dst, size_t dstsz, const char *src) { } /* Rebuild the derived miss-GRAPH view under the shadow project - * "::coverage": ONLY the not-fully-indexed files, laid out as the + * "::missed": ONLY the not-fully-indexed files, laid out as the * file structure (Project → Folder chain → File), each File carrying - * {kind, detail}. Queryable via query_graph(graph="coverage") with zero + * {kind, detail}. Queryable via query_graph(graph="missed") with zero * cypher-engine changes; the real project's graph gains no rows. Derived * data — rebuilt from the authoritative (post-prune) table contents. */ static void cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) { diff --git a/src/store/store.h b/src/store/store.h index e83be73e4..6603bd234 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -423,7 +423,7 @@ int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_co int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, int *count); -/* Name of the derived miss-graph shadow project ("::coverage"). +/* Name of the derived miss-graph shadow project ("::missed"). * cbm_store_coverage_replace materializes the coverage rows as a file- * structure graph (Project → Folder → File{kind, detail}) under this project * name — queryable via the normal cypher path without touching the real diff --git a/src/ui/http_server.c b/src/ui/http_server.c index e08ed8dbe..939c1d76e 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -1369,6 +1369,7 @@ static double layout_radius(const cbm_layout_result_t *r) { static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { char project[256] = {0}; char max_str[32] = {0}; + char graph_str[32] = {0}; if (!cbm_http_query_param(req->query, "project", project, (int)sizeof(project)) || project[0] == '\0') { @@ -1383,6 +1384,21 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { max_nodes = v; } + /* graph=missed (#963): lay out the derived miss graph (shadow project + * "::missed" inside the SAME db file) instead of the code graph — + * only files the indexer could not fully cover, as their file structure. + * The db file still resolves from the validated base project name. */ + bool missed_graph = false; + if (cbm_http_query_param(req->query, "graph", graph_str, (int)sizeof(graph_str))) { + missed_graph = strcmp(graph_str, "missed") == 0; + } + char scoped_project[320]; + if (missed_graph) { + cbm_store_coverage_shadow_project(scoped_project, sizeof(scoped_project), project); + } else { + snprintf(scoped_project, sizeof(scoped_project), "%s", project); + } + char db_path[1024]; db_path_for_project(project, db_path, sizeof(db_path)); @@ -1398,7 +1414,7 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { } cbm_layout_result_t *layout = - cbm_layout_compute(store, project, CBM_LAYOUT_OVERVIEW, NULL, 0, max_nodes); + cbm_layout_compute(store, scoped_project, CBM_LAYOUT_OVERVIEW, NULL, 0, max_nodes); /* Find linked projects from CROSS_* edges. Keep `store` open through the * linked-projects loop below so we can resolve target Route QNs against diff --git a/tests/test_cli.c b/tests/test_cli.c index ea663fe33..0618fd292 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -2425,11 +2425,10 @@ TEST(cli_upsert_claude_hook_fresh) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); ASSERT(strstr(data, "PreToolUse") != NULL); - /* Matcher excludes Read per issue #362 (gating Read breaks the - * read-before-edit invariant). Assert exact matcher value AND that no - * Read-chained matcher slipped back in. */ - ASSERT(strstr(data, "\"Grep|Glob\"") != NULL); - ASSERT(strstr(data, "Glob|Read") == NULL); + /* Matcher includes Read for the coverage note (#963). Safe against the + * issue-#362 gate hazard: the augmenter is structurally non-blocking + * (always exit 0, additionalContext only). */ + ASSERT(strstr(data, "\"Grep|Glob|Read\"") != NULL); ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); @@ -2499,9 +2498,8 @@ TEST(cli_upsert_claude_hook_existing) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); - /* Our hook added with the non-blocking matcher (issue #362). */ - ASSERT(strstr(data, "\"Grep|Glob\"") != NULL); - ASSERT(strstr(data, "Glob|Read") == NULL); + /* Our hook added with the current matcher (Read included for #963). */ + ASSERT(strstr(data, "\"Grep|Glob|Read\"") != NULL); /* Existing hook preserved */ ASSERT(strstr(data, "Bash") != NULL); ASSERT(strstr(data, "firewall") != NULL); @@ -2518,9 +2516,9 @@ TEST(cli_upsert_claude_hook_replace) { char settingspath[512]; snprintf(settingspath, sizeof(settingspath), "%s/settings.json", tmpdir); - /* Pre-existing CMM hook with old message */ + /* Pre-existing CMM hook with an OLD matcher (pre-#963) + old message */ write_test_file(settingspath, - "{\"hooks\":{\"PreToolUse\":[{\"matcher\":\"Grep|Glob|Read\"," + "{\"hooks\":{\"PreToolUse\":[{\"matcher\":\"Grep|Glob\"," "\"hooks\":[{\"type\":\"command\",\"command\":\"echo old-cmm-message\"}]}]}}"); int rc = cbm_upsert_claude_hooks(settingspath); @@ -3049,9 +3047,9 @@ TEST(cli_sha256_file_matches_known_vector) { "", 0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); ASSERT_TRUE(sha256_vector_ok( "abc", 3, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")); - ASSERT_TRUE(sha256_vector_ok( - "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56, - "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")); + ASSERT_TRUE( + sha256_vector_ok("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56, + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")); PASS(); } diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index a16c7fac1..101d87eeb 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -390,12 +390,12 @@ TEST(index_parse_partial_reported) { ASSERT_NOT_NULL(strstr(qresp, "parse_partial")); free(qresp); - /* The miss GRAPH: the same signal is queryable as a file-structure graph - * via query_graph(graph="coverage") — exactly the query the tool + /* The missed GRAPH: the same signal is queryable as a file-structure + * graph via query_graph(graph="missed") — exactly the query the tool * description advertises — and it lives under the shadow project, so the * REAL code graph gained no coverage rows. */ snprintf(qargs, sizeof(qargs), - "{\"project\":\"%s\",\"graph\":\"coverage\",\"query\":\"MATCH (f:File) WHERE " + "{\"project\":\"%s\",\"graph\":\"missed\",\"query\":\"MATCH (f:File) WHERE " "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail\"}", lp.project); char *gresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); @@ -435,7 +435,19 @@ TEST(index_parse_partial_clears_on_fix) { } rh_to_fwd_slashes(lp.tmpdir); - ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef broken(:\n pass\n"); + /* The #ifdef-split-brace pattern: an UNRECOVERED miss (the first branch's + * `guarded` never becomes a def), so the flag survives recovery + * subtraction. A `def broken(:`-style fixture would NOT work here — its + * def is recovered and the region is dropped. */ + ri_write_text(lp.tmpdir, "flaky.c", + "void ok_before(void) { }\n" + "#ifdef A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n"); ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); char *resp = NULL; @@ -454,13 +466,17 @@ TEST(index_parse_partial_clears_on_fix) { snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); char *cov1 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); ASSERT_NOT_NULL(cov1); - ASSERT_NOT_NULL(strstr(cov1, "flaky.py")); + ASSERT_NOT_NULL(strstr(cov1, "flaky.c")); free(cov1); /* Fix the file; ensure a newer mtime so change detection can't miss it. */ struct timespec ts = {0, INCR_FIX_SLEEP_NS}; nanosleep(&ts, NULL); - ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef fixed():\n return 2\n"); + ri_write_text(lp.tmpdir, "flaky.c", + "void ok_before(void) { }\n" + "static int fixed(int x) {\n" + " return x + 1;\n" + "}\n"); /* Re-index WITHOUT deleting the DB → routes through the incremental path. */ char iargs[700]; @@ -471,7 +487,7 @@ TEST(index_parse_partial_clears_on_fix) { char *cov2 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); ASSERT_NOT_NULL(cov2); - ASSERT_NULL(strstr(cov2, "flaky.py")); + ASSERT_NULL(strstr(cov2, "flaky.c")); free(cov2); store = cbm_store_open_path(lp.dbpath); diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c index 3b11cfc01..7d3645ffb 100644 --- a/tests/test_parse_coverage.c +++ b/tests/test_parse_coverage.c @@ -76,14 +76,27 @@ static const char *C_CLEAN = "#include \n" " return x + 1;\n" "}\n"; -static const char *PY_BROKEN = "def ok():\n" - " return 1\n" - "\n" - "def broken(:\n" - " pass\n" - "\n" - "def ok2():\n" - " return 2\n"; +/* `def broken(:` parses with an ERROR region, but tree-sitter error recovery + * still yields the `broken` function def — a DEFINITELY RECOVERED miss. */ +static const char *PY_BROKEN_RECOVERED = "def ok():\n" + " return 1\n" + "\n" + "def broken(:\n" + " pass\n" + "\n" + "def ok2():\n" + " return 2\n"; + +/* Pure operator garbage between defs: an ERROR region no def walker can + * recover anything from — a genuine, unrecovered miss. */ +static const char *PY_GARBAGE = "def ok():\n" + " return 1\n" + "\n" + "%%% ((( garbage ))) %%%\n" + "??? !!!\n" + "\n" + "def ok2():\n" + " return 2\n"; static const char *PY_CLEAN = "def ok():\n" " return 1\n" @@ -150,8 +163,8 @@ TEST(c_clean_file_not_flagged) { PASS(); } -TEST(py_syntax_error_sets_parse_incomplete) { - CBMFileResult *r = do_extract(PY_BROKEN, CBM_LANG_PYTHON, "broken.py"); +TEST(py_unrecovered_garbage_sets_parse_incomplete) { + CBMFileResult *r = do_extract(PY_GARBAGE, CBM_LANG_PYTHON, "garbage.py"); ASSERT_NOT_NULL(r); ASSERT_TRUE(r->parse_incomplete); ASSERT_GTE(r->error_region_count, 1); @@ -161,6 +174,20 @@ TEST(py_syntax_error_sets_parse_incomplete) { PASS(); } +TEST(py_recovered_def_not_flagged) { + /* Recovery subtraction: `def broken(:` produces an ERROR region, but the + * def walker still recovers `broken` covering the whole region — the + * construct IS in the graph, so flagging it would be a false miss. */ + CBMFileResult *r = do_extract(PY_BROKEN_RECOVERED, CBM_LANG_PYTHON, "broken.py"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "broken")); /* the recovery that justifies unflagging */ + ASSERT_FALSE(r->parse_incomplete); + ASSERT_EQ(r->error_region_count, 0); + ASSERT_NULL(r->error_ranges); + cbm_free_result(r); + PASS(); +} + TEST(py_clean_file_not_flagged) { CBMFileResult *r = do_extract(PY_CLEAN, CBM_LANG_PYTHON, "clean.py"); ASSERT_NOT_NULL(r); @@ -172,17 +199,17 @@ TEST(py_clean_file_not_flagged) { } TEST(error_region_cap_is_honored) { - /* Pathological input: many separate broken defs interleaved with valid - * ones. The collector must stay bounded by its 64-region cap (matches - * CBM_MAX_ERROR_REGIONS in cbm.c) — pathological input can't blow up the - * report, and the flag itself still fires. */ - enum { BROKEN_DEFS = 200, LINE_CAP = 64 }; - char *src = (char *)malloc(BROKEN_DEFS * 96 + 1); + /* Pathological input: many separate unrecoverable garbage blocks + * interleaved with valid defs. The collector must stay bounded by its + * 64-region cap (matches CBM_MAX_ERROR_REGIONS in cbm.c) — pathological + * input can't blow up the report, and the flag itself still fires. */ + enum { GARBAGE_BLOCKS = 200, LINE_CAP = 64 }; + char *src = (char *)malloc(GARBAGE_BLOCKS * 96 + 1); ASSERT_NOT_NULL(src); size_t off = 0; - for (int i = 0; i < BROKEN_DEFS; i++) { - off += (size_t)snprintf(src + off, 96, - "def ok%d():\n return %d\ndef broken%d(:\n pass\n", i, i, i); + for (int i = 0; i < GARBAGE_BLOCKS; i++) { + off += (size_t)snprintf( + src + off, 96, "def ok%d():\n return %d\n%%%%%% garbage%d ((( %%%%%%\n", i, i, i); } CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "many_errors.py"); free(src); @@ -195,6 +222,29 @@ TEST(error_region_cap_is_honored) { PASS(); } +/* Trailing recovered functions AFTER the failed #ifdef region must not + * unflag it: recovery evidence must originate INSIDE the region, and the + * unrecovered lines (the first branch's `guarded`) keep it flagged. */ +TEST(c_trailing_recovered_defs_keep_flag) { + const char *src = "void ok_before(void) { }\n" + "#ifdef A\n" + "static int guarded(int x) {\n" + "#else\n" + "static int guarded_alt(int x) {\n" + "#endif\n" + " return x + 1;\n" + "}\n" + "void ok_after(void) { }\n" + "static int nested_ok(int y) { return y; }\n"; + CBMFileResult *r = do_extract(src, CBM_LANG_C, "probe.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_def(r, "guarded_alt")); /* partial recovery inside the region */ + ASSERT_TRUE(r->parse_incomplete); /* ...but `guarded` is still lost */ + ASSERT_GTE(r->error_region_count, 1); + cbm_free_result(r); + PASS(); +} + /* ── Suite ────────────────────────────────────────────────────────────────── */ SUITE(parse_coverage) { @@ -202,7 +252,9 @@ SUITE(parse_coverage) { RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted); RUN_TEST(c_error_range_points_at_failed_region); RUN_TEST(c_clean_file_not_flagged); - RUN_TEST(py_syntax_error_sets_parse_incomplete); + RUN_TEST(py_unrecovered_garbage_sets_parse_incomplete); + RUN_TEST(py_recovered_def_not_flagged); RUN_TEST(py_clean_file_not_flagged); RUN_TEST(error_region_cap_is_honored); + RUN_TEST(c_trailing_recovered_defs_keep_flag); } diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 46a864841..7cc1a019f 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1618,12 +1618,11 @@ TEST(store_coverage_roundtrip_prune_shadow) { ASSERT_STR_EQ(got[0].detail, "4-7"); cbm_store_free_coverage(got, n); - /* Shadow miss-graph materialized under "test::coverage": + /* Shadow miss-graph materialized under "test::missed": * Project → Folder(src) → File(a.py){kind,detail}. */ cbm_node_t *nodes = NULL; int nc = 0; - ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), - CBM_STORE_OK); + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "File", &nodes, &nc), CBM_STORE_OK); ASSERT_EQ(nc, 1); ASSERT_STR_EQ(nodes[0].file_path, "src/a.py"); ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"kind\":\"parse_partial\"")); @@ -1631,7 +1630,7 @@ TEST(store_coverage_roundtrip_prune_shadow) { cbm_store_free_nodes(nodes, nc); nodes = NULL; nc = 0; - ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "Folder", &nodes, &nc), + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "Folder", &nodes, &nc), CBM_STORE_OK); ASSERT_EQ(nc, 1); ASSERT_STR_EQ(nodes[0].name, "src"); @@ -1646,8 +1645,7 @@ TEST(store_coverage_roundtrip_prune_shadow) { free(got); nodes = NULL; nc = 0; - ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), - CBM_STORE_OK); + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::missed", "File", &nodes, &nc), CBM_STORE_OK); ASSERT_EQ(nc, 0); cbm_store_free_nodes(nodes, nc); From afc649f4e807acdb4c5eabe8181ce414a179efff Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 8 Jul 2026 20:07:01 +0200 Subject: [PATCH 4/8] feat(ui): missed skeleton beside the code galaxy with click-to-focus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render the miss graph side by side with the code graph instead of swapping layouts: - /api/layout (code graph) now also attaches "missed_graph": {nodes, edges, offset} — the shadow-project layout placed below the primary cluster (same satellite pattern as linked_projects; -Y slot so cross-repo satellites collide last). graph=missed stays the isolated view for API users - the UI paints the skeleton white and ghostly beside the galaxy, auto-frames BOTH clusters on load, and navigates naturally: clicking the skeleton flies the camera into it (file labels + detail panel), clicking a code node flies back to the code side, and clicking empty space while the skeleton has focus returns to the overview (the galaxy can be entirely off-screen at that point) - the sidebar toggle now shows/hides the skeleton (with a missed-file count) rather than swapping the layout Verified in the browser: overview composition, skeleton focus with split.c labeled, and the empty-space return flight; ui/httpd suites and frontend build + tests green. Refs #963 Signed-off-by: Martin Vogel --- graph-ui/src/components/FilterPanel.tsx | 29 ++++++---- graph-ui/src/components/GraphScene.tsx | 34 +++++++++++ graph-ui/src/components/GraphTab.tsx | 77 ++++++++++++++++++++++--- graph-ui/src/lib/types.ts | 10 ++++ src/ui/http_server.c | 67 ++++++++++++++++++++- 5 files changed, 195 insertions(+), 22 deletions(-) diff --git a/graph-ui/src/components/FilterPanel.tsx b/graph-ui/src/components/FilterPanel.tsx index db2d0a513..b2e221531 100644 --- a/graph-ui/src/components/FilterPanel.tsx +++ b/graph-ui/src/components/FilterPanel.tsx @@ -22,8 +22,9 @@ interface FilterPanelProps { onToggleShowOnlyDead: () => void; onToggleHideEntryPoints: () => void; onToggleHideTests: () => void; - /* Missed graph (#963): file structure of not-fully-indexed files */ + /* Missed skeleton (#963): white satellite of not-fully-indexed files */ missedView: boolean; + missedCount: number; onToggleMissedView: () => void; } @@ -80,6 +81,7 @@ export function FilterPanel({ onToggleHideEntryPoints, onToggleHideTests, missedView, + missedCount, onToggleMissedView, }: FilterPanelProps) { const { labelCounts, edgeTypeCounts, statusCounts } = useMemo(() => { @@ -168,25 +170,30 @@ export function FilterPanel({
- {/* Missed graph (#963): swaps the layout to the file structure of files - the indexer could not fully cover (best-effort signal). */} + {/* Missed skeleton (#963): white satellite cluster of files the indexer + could not fully cover, shown beside the code galaxy. Click it to + focus; click the code galaxy to come back. */}
- Graph + Missed files + {missedCount > 0 && ( + + {missedCount.toLocaleString()} files + + )}
- {missedView && ( -

- Best-effort: files with unparseable regions or skips. Empty = no - known misses (not a completeness guarantee). -

- )} +

+ {missedCount > 0 + ? "White satellite = files not fully indexed (best-effort). Click it to focus, click the galaxy to return." + : "No known misses (best-effort — not a completeness guarantee)."} +

{/* Dead-code view */} diff --git a/graph-ui/src/components/GraphScene.tsx b/graph-ui/src/components/GraphScene.tsx index 663c2ad32..cad93d926 100644 --- a/graph-ui/src/components/GraphScene.tsx +++ b/graph-ui/src/components/GraphScene.tsx @@ -112,22 +112,31 @@ function IdleAutoRotate({ interface GraphSceneProps { data: GraphData; + /* Missed skeleton (#963): pre-offset, pre-painted white nodes + edges of + * the not-fully-indexed files, rendered as a ghost cluster beside the + * galaxy. null hides it. */ + missed?: { nodes: GraphNode[]; edges: GraphData["edges"] } | null; highlightedIds: Set | null; cameraTarget: CameraTarget | null; showLabels: boolean; display?: DisplaySettings; onNodeClick: (node: GraphNode) => void; + /* Fired when a click hits empty space (no node). Used to fly back to the + * overview after focusing the missed skeleton. */ + onBackgroundClick?: () => void; } export type { CameraTarget }; export function GraphScene({ data, + missed = null, highlightedIds, cameraTarget, showLabels, display = DEFAULT_DISPLAY_SETTINGS, onNodeClick, + onBackgroundClick, }: GraphSceneProps) { const [hovered, setHovered] = useState(null); const controlsRef = useRef(null); @@ -151,6 +160,7 @@ export function GraphScene({ alpha: false, powerPreference: "high-performance", }} + onPointerMissed={onBackgroundClick} > @@ -176,6 +186,30 @@ export function GraphScene({ /> {showLabels && } + {/* Missed skeleton (#963): white ghost of the not-fully-indexed files. + * Clicks route through the same handler — GraphTab re-centers the + * camera on the whole skeleton cluster. */} + {missed && missed.nodes.length > 0 && ( + + + + {showLabels && } + + )} + {/* Satellite galaxies for cross-repo linked projects */} {data.linked_projects?.map((lp: LinkedProject) => { const offsetNodes = lp.nodes.map((n) => ({ diff --git a/graph-ui/src/components/GraphTab.tsx b/graph-ui/src/components/GraphTab.tsx index 56f549cd2..1144b9f64 100644 --- a/graph-ui/src/components/GraphTab.tsx +++ b/graph-ui/src/components/GraphTab.tsx @@ -102,10 +102,11 @@ export function GraphTab({ project }: GraphTabProps) { const [enabledLabels, setEnabledLabels] = useState>(new Set()); const [enabledEdgeTypes, setEnabledEdgeTypes] = useState>(new Set()); - /* Missed graph (#963): render the file structure of files the indexer - * could not fully cover instead of the code graph. Server-side option - * (`graph=missed` on /api/layout) — toggling refetches. */ - const [missedView, setMissedView] = useState(false); + /* Missed skeleton (#963): the file structure of files the indexer could + * not fully cover, shown as a white satellite cluster beside the code + * galaxy. Toggle only hides/shows it — the data rides along with every + * code-graph layout. */ + const [showMissedSkeleton, setShowMissedSkeleton] = useState(true); /* Dead-code view: recolor by status + status-based filters */ const [deadCodeView, setDeadCodeView] = useState(false); @@ -189,11 +190,53 @@ export function GraphTab({ project }: GraphTabProps) { /* …and fetch only once budget and project agree (one fetch per change). */ useEffect(() => { if (project && budget.project === project) { - fetchOverview(project, budget.value, missedView ? "missed" : "code"); + fetchOverview(project, budget.value); setHighlightedIds(null); setSelectedPath(null); } - }, [project, budget, missedView, fetchOverview]); + }, [project, budget, fetchOverview]); + + /* Missed skeleton: offset into place and paint white — a ghost of the + * files the graph could not fully cover, sitting beside the galaxy. */ + const missedSkeleton = useMemo(() => { + const mg = data?.missed_graph; + if (!mg || mg.nodes.length === 0) return null; + const nodes = mg.nodes.map((n) => ({ + ...n, + x: n.x + mg.offset.x, + y: n.y + mg.offset.y, + z: n.z + mg.offset.z, + color: "#e9eef5", + })); + return { nodes, edges: mg.edges, ids: new Set(nodes.map((n) => n.id)) }; + }, [data]); + + /* Overview framing: both clusters (galaxy + skeleton) in one shot. */ + const overviewTarget = useMemo(() => { + if (!data) return null; + const all = missedSkeleton ? [...data.nodes, ...missedSkeleton.nodes] : data.nodes; + return computeCameraTarget(all, new Set(all.map((n) => n.id))); + }, [data, missedSkeleton]); + + /* With a skeleton beside the galaxy, auto-frame BOTH clusters on load so + * the side-by-side composition is visible without manual zooming. */ + useEffect(() => { + if (missedSkeleton && overviewTarget) { + setCameraTarget(overviewTarget); + } + }, [missedSkeleton, overviewTarget]); + + /* Clicking empty space while the skeleton has focus flies back to the + * overview (the galaxy may be entirely off-screen at that point, so there + * is no code node to click). No-op during normal galaxy exploration. */ + const handleBackgroundClick = useCallback(() => { + if (selectedNode && missedSkeleton?.ids.has(selectedNode.id) && overviewTarget) { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + setCameraTarget(overviewTarget); + } + }, [selectedNode, missedSkeleton, overviewTarget]); /* Fetch git remote metadata for GitHub deep-links */ useEffect(() => { @@ -231,6 +274,19 @@ export function GraphTab({ project }: GraphTabProps) { const handleNodeClick = useCallback( (node: GraphNode) => { if (!filteredData) return; + + /* Clicking the missed skeleton re-centers the camera on that whole + * cluster (it's small — the natural focus unit is the skeleton, not a + * single node); clicking any code node flies back to the code galaxy + * via the normal per-node focus below. */ + if (missedSkeleton?.ids.has(node.id)) { + setSelectedNode(node); + setHighlightedIds(null); + setSelectedPath(node.file_path ?? null); + setCameraTarget(computeCameraTarget(missedSkeleton.nodes, missedSkeleton.ids)); + return; + } + setSelectedNode(node); /* Highlight the node and its direct connections */ @@ -243,7 +299,7 @@ export function GraphTab({ project }: GraphTabProps) { setSelectedPath(node.file_path ?? null); setCameraTarget(computeCameraTarget(filteredData.nodes, connectedIds)); }, - [filteredData], + [filteredData, missedSkeleton], ); const handleNavigateToNode = useCallback( @@ -356,8 +412,9 @@ export function GraphTab({ project }: GraphTabProps) { onToggleShowOnlyDead={() => setShowOnlyDead((v) => !v)} onToggleHideEntryPoints={() => setHideEntryPoints((v) => !v)} onToggleHideTests={() => setHideTests((v) => !v)} - missedView={missedView} - onToggleMissedView={() => setMissedView((v) => !v)} + missedView={showMissedSkeleton} + missedCount={data?.missed_graph?.nodes.filter((n) => n.label === "File").length ?? 0} + onToggleMissedView={() => setShowMissedSkeleton((v) => !v)} /> diff --git a/graph-ui/src/lib/types.ts b/graph-ui/src/lib/types.ts index a10b53d95..4d8ffa638 100644 --- a/graph-ui/src/lib/types.ts +++ b/graph-ui/src/lib/types.ts @@ -50,11 +50,21 @@ export interface LinkedProject { cross_edges: GraphEdge[]; } +/* Missed-graph skeleton (#963): the file structure of files the indexer + * could not fully cover, laid out as a satellite cluster beside the code + * galaxy (server-computed offset, same shape as LinkedProject's). */ +export interface MissedGraph { + nodes: GraphNode[]; + edges: GraphEdge[]; + offset: { x: number; y: number; z: number }; +} + export interface GraphData { nodes: GraphNode[]; edges: GraphEdge[]; total_nodes: number; linked_projects?: LinkedProject[]; + missed_graph?: MissedGraph; } export interface Project { diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 939c1d76e..4d26f0561 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -1366,6 +1366,63 @@ static double layout_radius(const cbm_layout_result_t *r) { return sqrt(max_r2); } +/* Attach the missed-graph skeleton (#963) to the primary layout doc as + * "missed_graph": {"nodes":[...], "edges":[...], "offset":{x,y,z}} + * — the file structure of files the indexer could not fully cover, laid out + * as a satellite cluster beside the code galaxy (the UI renders it as a white + * skeleton; clicking it re-centers the camera there). The offset sits on the + * -Y side: linked-project satellites spread counter-clockwise from +X, so + * this slot collides last. Returns true when a non-empty skeleton was + * attached; no-op when the project has no missed files. */ +static bool attach_missed_graph(yyjson_mut_doc *mdoc, yyjson_mut_val *mroot, cbm_store_t *store, + const char *project, double primary_radius) { + char covproj[512]; + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + cbm_layout_result_t *ml = cbm_layout_compute(store, covproj, CBM_LAYOUT_OVERVIEW, NULL, 0, 0); + if (!ml) { + return false; + } + if (ml->node_count == 0) { + cbm_layout_free(ml); + return false; + } + double miss_radius = layout_radius(ml); + char *mjson = cbm_layout_to_json(ml); + cbm_layout_free(ml); + if (!mjson) { + return false; + } + yyjson_doc *mldoc = yyjson_read(mjson, strlen(mjson), 0); + free(mjson); + if (!mldoc) { + return false; + } + yyjson_mut_val *entry = yyjson_mut_obj(mdoc); + yyjson_val *mlroot = yyjson_doc_get_root(mldoc); + yyjson_val *mn = yyjson_obj_get(mlroot, "nodes"); + yyjson_val *me = yyjson_obj_get(mlroot, "edges"); + if (mn) { + yyjson_mut_obj_add_val(mdoc, entry, "nodes", yyjson_val_mut_copy(mdoc, mn)); + } + if (me) { + yyjson_mut_obj_add_val(mdoc, entry, "edges", yyjson_val_mut_copy(mdoc, me)); + } + yyjson_doc_free(mldoc); + + double dist = primary_radius + miss_radius + LAYOUT_GALAXY_PAD; + if (dist < LAYOUT_GALAXY_SPACING) { + dist = LAYOUT_GALAXY_SPACING; + } + yyjson_mut_val *offset = yyjson_mut_obj(mdoc); + yyjson_mut_obj_add_real(mdoc, offset, "x", 0.0); + yyjson_mut_obj_add_real(mdoc, offset, "y", -dist); + yyjson_mut_obj_add_real(mdoc, offset, "z", 0.0); + yyjson_mut_obj_add_val(mdoc, entry, "offset", offset); + + yyjson_mut_obj_add_val(mdoc, mroot, "missed_graph", entry); + return true; +} + static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { char project[256] = {0}; char max_str[32] = {0}; @@ -1440,14 +1497,16 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { return; } - if (linked_count == 0) { + /* Fast path: no satellites to attach. The missed skeleton only decorates + * the CODE graph — a graph=missed request already IS the miss graph. */ + if (linked_count == 0 && missed_graph) { cbm_store_close(store); cbm_http_replyf(c, 200, g_cors_json, "%s", primary_json); free(primary_json); return; } - /* Parse primary JSON and append linked_projects array */ + /* Parse primary JSON and append missed_graph + linked_projects */ yyjson_doc *pdoc = yyjson_read(primary_json, strlen(primary_json), 0); free(primary_json); if (!pdoc) { @@ -1460,6 +1519,10 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { yyjson_doc_free(pdoc); yyjson_mut_val *mroot = yyjson_mut_doc_get_root(mdoc); + if (!missed_graph) { + (void)attach_missed_graph(mdoc, mroot, store, project, primary_radius); + } + yyjson_mut_val *lp_arr = yyjson_mut_arr(mdoc); for (int li = 0; li < linked_count; li++) { From c97f6828f8e884e18e21a9e27082b9f7f0999947 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 8 Jul 2026 23:25:39 +0200 Subject: [PATCH 5/8] test(smoke): matcher guard tracks Read inclusion for the coverage note Check 8d still locked the matcher to the pre-#963 'Grep|Glob' and failed the pr-smoke leg on all platforms. The matcher now includes Read (the augmenter injects the coverage note when a not-fully-indexed file is read; structurally non-blocking, so the issue-#362 gate hazard cannot recur). The guard now pins the exact new matcher and still rejects Search/catch-all creep. Refs #963 Signed-off-by: Martin Vogel --- scripts/smoke-test.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 34bdc3d6c..6d9675f13 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -929,21 +929,24 @@ if ! path_match "$CMD" "$SELF_PATH"; then fi echo "OK 8c: Claude Code MCP (.claude/.mcp.json)" -# 8d: Claude Code hooks — matcher must be exactly "Grep|Glob" (no Read, no Search). -# Gating Read breaks Claude Code's read-before-edit invariant (issue #362), so -# this assertion locks in the matcher to prevent regressions. +# 8d: Claude Code hooks — matcher must be exactly "Grep|Glob|Read" (no Search). +# Read is matched for the indexing-coverage note (#963); safe against the old +# issue-#362 gate hazard because the augmenter is structurally non-blocking +# (always exit 0, additionalContext only). This assertion locks in the exact +# matcher to prevent both regressions (Read dropped again) and creep (Search +# or catch-all matchers sneaking back). if ! cat "$FAKE_HOME/.claude/settings.json" 2>/dev/null | python3 -c " import json, sys d = json.load(sys.stdin) hooks = d.get('hooks', {}).get('PreToolUse', []) -ok = any(h.get('matcher') == 'Grep|Glob' for h in hooks) -bad = any('Read' in str(h.get('matcher', '')) for h in hooks) +ok = any(h.get('matcher') == 'Grep|Glob|Read' for h in hooks) +bad = any('Search' in str(h.get('matcher', '')) for h in hooks) sys.exit(0 if (ok and not bad) else 1) " 2>/dev/null; then - echo "FAIL 8d: PreToolUse hook matcher is not exactly 'Grep|Glob' (or still contains Read)" + echo "FAIL 8d: PreToolUse hook matcher is not exactly 'Grep|Glob|Read'" exit 1 fi -echo "OK 8d: Claude Code PreToolUse hook (matcher=Grep|Glob, Read excluded)" +echo "OK 8d: Claude Code PreToolUse hook (matcher=Grep|Glob|Read)" # 8e: Claude Code shim script — must be non-blocking augmenter, not a gate. if [ "$(uname -s)" != "MINGW64_NT" ] 2>/dev/null; then From 545971cd88369da48a3e08f5fd13c7cda948f082 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 9 Jul 2026 00:34:46 +0200 Subject: [PATCH 6/8] feat(mcp): fold coverage into index_status; coverage note on get_code_snippet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: no dedicated coverage tool. - index_status now carries the coverage report (parse_partial + skipped lists from the separate index_coverage table, plus the best-effort coverage_note); get_index_coverage is removed. Tool description updated so agents learn the coverage semantics from index_status; index_repository and the response note now point there - the read hook resolves coverage through index_status (same envelope fields, unchanged note wording); verified end-to-end against a real index (flagged file -> note with exact ranges, clean file -> silence) - get_code_snippet responses from a parse_partial file now carry a correlated coverage_note ("line range(s) X could not be parsed ... the source above is ground truth") — the result names its file, so the warning is precisely anchored. Entirely-skipped files cannot appear here (no nodes), so no uncorrelated noise is possible; clean files carry no note (asserted both ways in the e2e test) Refs #963 Signed-off-by: Martin Vogel --- src/cli/hook_augment.c | 5 +- src/mcp/mcp.c | 211 ++++++++++++++++++---------------- tests/test_index_resilience.c | 26 ++++- 3 files changed, 136 insertions(+), 106 deletions(-) diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index dd212caf7..35ff15f78 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -231,7 +231,8 @@ static char *ha_format_context(const char *envelope, const char *token, bool *is * knows the knowledge graph may under-report this file. Best-effort and * non-blocking like everything else here — no entry, no output. */ -/* Parse a get_index_coverage envelope and return a note when `rel` is listed. +/* Parse an index_status envelope (which carries the coverage report) and + * return a note when `rel` is listed. * *is_error is set for MCP errors (project not indexed) → caller climbs. */ static char *ha_coverage_context(const char *envelope, const char *rel, bool *is_error) { *is_error = false; @@ -349,7 +350,7 @@ static char *ha_resolve_coverage(cbm_mcp_server_t *srv, const char *file_path) { yyjson_mut_doc_free(adoc); free(project); if (args) { - char *res = cbm_mcp_handle_tool(srv, "get_index_coverage", args); + char *res = cbm_mcp_handle_tool(srv, "index_status", args); free(args); if (res) { bool is_error = false; diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 37d7ad7f5..2a2d0885d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -320,7 +320,7 @@ static const tool_def_t TOOLS[] = { "COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not " "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " "constructs inside the listed line ranges could not be parsed and MAY be missing from " - "the graph). Query the persisted signal any time via the get_index_coverage tool or " + "the graph). Query the persisted signal any time via index_status or " "structurally via query_graph(graph=\"missed\"). Both signals are best-effort: absence " "of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" @@ -437,7 +437,10 @@ static const tool_def_t TOOLS[] = { {"get_code_snippet", "Get code snippet", "Read source code for a function/class/symbol. IMPORTANT: First call search_graph to find the " "exact qualified_name, then pass it here. This is a read tool, not a search tool. Accepts " - "full qualified_name (exact match) or short function name (returns suggestions if ambiguous).", + "full qualified_name (exact match) or short function name (returns suggestions if ambiguous). " + "If the response carries a 'coverage_note', the file was only partially indexed — constructs " + "in the noted line ranges may be missing from the graph (best-effort signal); prefer grep " + "there and treat the returned source as ground truth.", "{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\",\"description\":" "\"Full qualified_name from search_graph, or short function name\"},\"project\":{" "\"type\":\"string\"},\"include_neighbors\":{" @@ -495,18 +498,15 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, - {"index_status", "Index status", "Get the indexing status of a project", - "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" - "\"project\"]}"}, - - {"get_index_coverage", "Index coverage", - "Report which files the indexer could NOT fully cover (best-effort signal, #963): " - "'parse_partial' files WERE indexed but contain line ranges tree-sitter could not parse — " - "constructs there MAY be missing from the graph (some are still recovered); 'skipped' files " - "were not indexed at all (oversized/read/parse failure). Use this before trusting graph " - "completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). " - "IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks " - "what the indexer can detect. For structural queries over the misses use " + {"index_status", "Index status", + "Get the indexing status of a project: node/edge counts, root path, git context, and the " + "indexing-COVERAGE report — which files the indexer could NOT fully cover (best-effort " + "signal): 'parse_partial' files WERE indexed but contain line ranges tree-sitter could not " + "parse — constructs there MAY be missing from the graph (some are still recovered); " + "'skipped' files were not indexed at all (oversized/read/parse failure). Use this before " + "trusting graph completeness on a file: if a file is listed, ALSO grep it (especially the " + "flagged ranges). IMPORTANT: absence from these lists is NOT a completeness guarantee — the " + "signal only marks what the indexer can detect. For structural queries over the misses use " "query_graph(graph=\"missed\").", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, @@ -2178,6 +2178,68 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return res; } +/* Indexing-coverage report (#963), attached to index_status: the best-effort + * signal from the separate index_coverage table (coverage is metadata ABOUT + * the graph, stored outside it). Full per-project list, capped generously. */ +enum { COVERAGE_FILE_CAP = 500 }; + +static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const char *project) { + cbm_coverage_row_t *rows = NULL; + int count = 0; + (void)cbm_store_coverage_get(store, project, &rows, &count); + + yyjson_mut_val *pp_files = yyjson_mut_arr(doc); + yyjson_mut_val *sk_files = yyjson_mut_arr(doc); + int pp_n = 0; + int sk_n = 0; + for (int i = 0; i < count; i++) { + bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0; + if (partial) { + if (pp_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_arr_add_val(pp_files, fe); + } + pp_n++; + } else { + if (sk_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "reason", rows[i].detail ? rows[i].detail : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : ""); + yyjson_mut_arr_add_val(sk_files, fe); + } + sk_n++; + } + } + cbm_store_free_coverage(rows, count); + + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, pp, "files", pp_files); + yyjson_mut_obj_add_int(doc, pp, "count", pp_n); + yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); + + yyjson_mut_val *sk = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, sk, "files", sk_files); + yyjson_mut_obj_add_int(doc, sk, "count", sk_n); + yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "skipped", sk); + + if (pp_n > 0 || sk_n > 0) { + yyjson_mut_obj_add_str( + doc, root, "coverage_note", + "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " + "but constructs inside the listed line ranges (1-based) MAY be missing from the graph " + "(tree-sitter error recovery still salvages some). skipped files were not indexed at " + "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " + "are NOT guaranteed to be fully indexed."); + } +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); @@ -2203,6 +2265,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { safe_str_free(&proj_info.indexed_at); safe_str_free(&proj_info.root_path); } + add_coverage_report(doc, root, store, project); if (nodes == 0) { yyjson_mut_obj_add_str( doc, root, "hint", @@ -2221,87 +2284,6 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { return result; } -/* get_index_coverage (#963): report the best-effort indexing-coverage signal - * from the separate index_coverage table (coverage is metadata ABOUT the - * graph, stored outside it). Full per-project list, capped generously. */ -enum { COVERAGE_FILE_CAP = 500 }; - -static char *handle_get_index_coverage(cbm_mcp_server_t *srv, const char *args) { - char *project = get_project_arg(args); - cbm_store_t *store = resolve_store(srv, project); - REQUIRE_STORE(store, project); - - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root); - - if (project) { - yyjson_mut_obj_add_str(doc, root, "project", project); - cbm_coverage_row_t *rows = NULL; - int count = 0; - (void)cbm_store_coverage_get(store, project, &rows, &count); - - yyjson_mut_val *pp_files = yyjson_mut_arr(doc); - yyjson_mut_val *sk_files = yyjson_mut_arr(doc); - int pp_n = 0; - int sk_n = 0; - for (int i = 0; i < count; i++) { - bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0; - if (partial) { - if (pp_n < COVERAGE_FILE_CAP) { - yyjson_mut_val *fe = yyjson_mut_obj(doc); - yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); - yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", - rows[i].detail ? rows[i].detail : ""); - yyjson_mut_arr_add_val(pp_files, fe); - } - pp_n++; - } else { - if (sk_n < COVERAGE_FILE_CAP) { - yyjson_mut_val *fe = yyjson_mut_obj(doc); - yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); - yyjson_mut_obj_add_strcpy(doc, fe, "reason", - rows[i].detail ? rows[i].detail : ""); - yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : ""); - yyjson_mut_arr_add_val(sk_files, fe); - } - sk_n++; - } - } - cbm_store_free_coverage(rows, count); - - yyjson_mut_val *pp = yyjson_mut_obj(doc); - yyjson_mut_obj_add_val(doc, pp, "files", pp_files); - yyjson_mut_obj_add_int(doc, pp, "count", pp_n); - yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP); - yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); - - yyjson_mut_val *sk = yyjson_mut_obj(doc); - yyjson_mut_obj_add_val(doc, sk, "files", sk_files); - yyjson_mut_obj_add_int(doc, sk, "count", sk_n); - yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP); - yyjson_mut_obj_add_val(doc, root, "skipped", sk); - - yyjson_mut_obj_add_str( - doc, root, "note", - "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " - "but constructs inside the listed line ranges (1-based) MAY be missing from the graph " - "(tree-sitter error recovery still salvages some). skipped files were not indexed at " - "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " - "are NOT guaranteed to be fully indexed."); - } else { - yyjson_mut_obj_add_str(doc, root, "status", "no_project"); - } - - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); - free(project); - - char *result = cbm_mcp_text_result(json, false); - free(json); - return result; -} - /* delete_project: just erase the .db file (and WAL/SHM). */ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { char *name = get_project_arg(args); @@ -3520,7 +3502,7 @@ static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, "not be parsed and MAY be missing from the graph (tree-sitter error " "recovery still salvages some). Prefer text search (grep) for those " "regions. Files absent from this list are NOT guaranteed to be fully " - "indexed. Query the persisted signal via get_index_coverage."); + "indexed. Query the persisted signal via index_status."); yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); } @@ -4427,6 +4409,38 @@ static void add_string_array(yyjson_mut_doc *doc, yyjson_mut_val *obj, const cha yyjson_mut_obj_add_val(doc, obj, key, arr); } +/* get_code_snippet coverage note (#963): if the resolved node's file is + * flagged parse_partial, warn that the graph may under-report this file. + * Correlated by construction — the result names its file. (An entirely- + * skipped file cannot appear here: it has no nodes to resolve a snippet + * from.) */ +static void add_snippet_coverage_note(yyjson_mut_doc *doc, yyjson_mut_val *root_obj, + cbm_store_t *store, const cbm_node_t *node) { + if (!node->file_path || !node->file_path[0] || !node->project) { + return; + } + cbm_coverage_row_t *rows = NULL; + int count = 0; + if (cbm_store_coverage_get(store, node->project, &rows, &count) != CBM_STORE_OK) { + return; + } + for (int i = 0; i < count; i++) { + if (rows[i].rel_path && strcmp(rows[i].rel_path, node->file_path) == 0 && rows[i].kind && + strcmp(rows[i].kind, "parse_partial") == 0) { + char note[CBM_SZ_1K]; + snprintf(note, sizeof(note), + "This file was only PARTIALLY indexed — line range(s) %s could not be " + "parsed, so constructs there may be missing from the graph (callers/callees " + "and search results can under-report this file). The source above is ground " + "truth. (best-effort signal)", + rows[i].detail && rows[i].detail[0] ? rows[i].detail : "?"); + yyjson_mut_obj_add_strcpy(doc, root_obj, "coverage_note", note); + break; + } + } + cbm_store_free_coverage(rows, count); +} + static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, const char *match_method, bool include_neighbors, cbm_node_t *alternatives, int alt_count) { @@ -4484,6 +4498,8 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_int(doc, root_obj, "callers", in_deg); yyjson_mut_obj_add_int(doc, root_obj, "callees", out_deg); + add_snippet_coverage_note(doc, root_obj, store, node); + char **nb_callers = NULL; int nb_caller_count = 0; char **nb_callees = NULL; @@ -5978,9 +5994,6 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch if (strcmp(tool_name, "query_graph") == 0) { return handle_query_graph(srv, args_json); } - if (strcmp(tool_name, "get_index_coverage") == 0) { - return handle_get_index_coverage(srv, args_json); - } if (strcmp(tool_name, "index_status") == 0) { return handle_index_status(srv, args_json); } diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index 101d87eeb..c5838a283 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -364,7 +364,7 @@ TEST(index_parse_partial_reported) { free(logtext); /* The signal is persisted in the SEPARATE index_coverage table (never - * mixed into the graph tables) and queryable via get_index_coverage, + * mixed into the graph tables) and reported by index_status, * exactly as the index_repository tool description advertises. */ cbm_coverage_row_t *rows = NULL; int cov_count = 0; @@ -384,12 +384,28 @@ TEST(index_parse_partial_reported) { char qargs[900]; snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); - char *qresp = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + char *qresp = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); ASSERT_NOT_NULL(qresp); ASSERT_NOT_NULL(strstr(qresp, "split.c")); ASSERT_NOT_NULL(strstr(qresp, "parse_partial")); free(qresp); + /* get_code_snippet on a symbol from the flagged file carries the + * correlated coverage note (the result names its file, so the warning is + * precisely anchored); the note never fires for clean files. */ + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\",\"qualified_name\":\"ok_before\"}", + lp.project); + char *sresp = cbm_mcp_handle_tool(lp.srv, "get_code_snippet", qargs); + ASSERT_NOT_NULL(sresp); + ASSERT_NOT_NULL(strstr(sresp, "coverage_note")); + ASSERT_NOT_NULL(strstr(sresp, "PARTIALLY indexed")); + free(sresp); + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\",\"qualified_name\":\"alpha\"}", lp.project); + char *cresp2 = cbm_mcp_handle_tool(lp.srv, "get_code_snippet", qargs); + ASSERT_NOT_NULL(cresp2); + ASSERT_NULL(strstr(cresp2, "coverage_note")); /* good.py: no note */ + free(cresp2); + /* The missed GRAPH: the same signal is queryable as a file-structure * graph via query_graph(graph="missed") — exactly the query the tool * description advertises — and it lives under the shadow project, so the @@ -424,7 +440,7 @@ TEST(index_parse_partial_reported) { /* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must * stay FRESH — after the broken file is fixed and the project re-indexed * (incremental route: the DB already exists), its parse_partial row is gone - * and get_index_coverage reports it no longer. A stale flag on a fixed file + * and index_status reports it no longer. A stale flag on a fixed file * would make the whole signal untrustworthy. */ TEST(index_parse_partial_clears_on_fix) { RProj lp; @@ -464,7 +480,7 @@ TEST(index_parse_partial_clears_on_fix) { /* Flagged after the first (full) index. */ char qargs[900]; snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); - char *cov1 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + char *cov1 = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); ASSERT_NOT_NULL(cov1); ASSERT_NOT_NULL(strstr(cov1, "flaky.c")); free(cov1); @@ -485,7 +501,7 @@ TEST(index_parse_partial_clears_on_fix) { ASSERT_NOT_NULL(resp2); free(resp2); - char *cov2 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + char *cov2 = cbm_mcp_handle_tool(lp.srv, "index_status", qargs); ASSERT_NOT_NULL(cov2); ASSERT_NULL(strstr(cov2, "flaky.c")); free(cov2); From 660b90d94afc07047f583a5ae2d76ad970a6694c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 9 Jul 2026 02:04:58 +0200 Subject: [PATCH 7/8] feat(ui): report-an-edge-case callout when the missed skeleton is selected Clicking a node of the missed skeleton now opens a dedicated right-panel callout instead of the standard node panel (code snippet and callers/callees are meaningless for a not-fully-indexed file): - explains the gap in plain words (best-effort detection; the file content itself is ground truth) - asks the user to have their agent summarize what fails to parse and report it upstream so the edge case can be handled - two working actions: a prefilled upstream GitHub issue (title 'Indexing gap: '; body carries ONLY the file path and project name, with an explicit add-snippets-only-if-shareable note) and a copy-to-clipboard agent prompt (index_status -> summarize flagged ranges -> file the issue) with visible copied feedback Verified in the browser: skeleton click opens the callout with both actions; frontend build + tests green. Refs #963 Signed-off-by: Martin Vogel --- graph-ui/src/components/GraphTab.tsx | 42 +++++--- graph-ui/src/components/MissedCallout.tsx | 122 ++++++++++++++++++++++ 2 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 graph-ui/src/components/MissedCallout.tsx diff --git a/graph-ui/src/components/GraphTab.tsx b/graph-ui/src/components/GraphTab.tsx index 1144b9f64..bc05d5c14 100644 --- a/graph-ui/src/components/GraphTab.tsx +++ b/graph-ui/src/components/GraphTab.tsx @@ -22,6 +22,7 @@ import { import { Sidebar } from "./Sidebar"; import { FilterPanel } from "./FilterPanel"; import { NodeDetailPanel } from "./NodeDetailPanel"; +import { MissedCallout } from "./MissedCallout"; import { ResizeHandle } from "./ResizeHandle"; import { ErrorBoundary } from "./ErrorBoundary"; import type { GraphNode, GraphData, RepoInfo } from "../lib/types"; @@ -556,19 +557,34 @@ export function GraphTab({ project }: GraphTabProps) { className="border-l border-border shrink-0 h-full overflow-hidden" style={{ width: rightWidth, maxHeight: "100%" }} > - { - setSelectedNode(null); - setHighlightedIds(null); - setSelectedPath(null); - }} - onNavigate={handleNavigateToNode} - /> + {missedSkeleton?.ids.has(selectedNode.id) ? ( + /* Skeleton node: the standard panel (code snippet, callers) is + * meaningless for a not-fully-indexed file — show the coverage + * callout with its report-the-edge-case actions instead. */ + { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + }} + /> + ) : ( + { + setSelectedNode(null); + setHighlightedIds(null); + setSelectedPath(null); + }} + onNavigate={handleNavigateToNode} + /> + )}
)} diff --git a/graph-ui/src/components/MissedCallout.tsx b/graph-ui/src/components/MissedCallout.tsx new file mode 100644 index 000000000..86da6f257 --- /dev/null +++ b/graph-ui/src/components/MissedCallout.tsx @@ -0,0 +1,122 @@ +import { useEffect, useState } from "react"; +import type { GraphNode } from "../lib/types"; + +/* Upstream tracker for indexing gaps — issues land on the codebase-memory-mcp + * project itself (the tool that failed to index), never the user's repo. */ +const UPSTREAM_ISSUES_URL = "https://github.com/DeusData/codebase-memory-mcp/issues/new"; + +interface MissedCalloutProps { + node: GraphNode; + project: string | null; + onClose: () => void; +} + +function buildIssueUrl(path: string, project: string | null): string { + const title = `Indexing gap: ${path}`; + const body = [ + "## Not fully indexed (best-effort coverage signal)", + "", + `- **File:** \`${path}\``, + `- **Project:** \`${project ?? "unknown"}\``, + "", + "", + "", + "_Reported from the graph UI's missed-coverage view._", + ].join("\n"); + return `${UPSTREAM_ISSUES_URL}?title=${encodeURIComponent(title)}&body=${encodeURIComponent(body)}`; +} + +function buildAgentPrompt(path: string, project: string | null): string { + return ( + `codebase-memory-mcp could not fully index \`${path}\`` + + (project ? ` (project \`${project}\`)` : "") + + " — best-effort coverage signal. Please: " + + "1) call the index_status MCP tool and note this file's flagged line ranges under parse_partial; " + + "2) read those ranges in the file and summarize which construct fails to parse; " + + `3) file a GitHub issue at ${UPSTREAM_ISSUES_URL} titled "Indexing gap: ${path}" ` + + "with the summary — include a minimal reproducible snippet ONLY if the code is shareable." + ); +} + +/* Right-panel callout shown when a missed-skeleton node is selected: explains + * the gap and offers two working actions — a prefilled upstream issue and a + * ready-made agent prompt (clipboard, with visible feedback). */ +export function MissedCallout({ node, project, onClose }: MissedCalloutProps) { + const path = node.file_path || node.name; + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) return; + const t = setTimeout(() => setCopied(false), 2000); + return () => clearTimeout(t); + }, [copied]); + + const copyPrompt = async () => { + try { + await navigator.clipboard.writeText(buildAgentPrompt(path, project)); + setCopied(true); + } catch { + /* clipboard unavailable (permissions/insecure context) — leave the + * button state unchanged so the failure is visible, not silent */ + } + }; + + return ( +
+
+
+

+ Not fully indexed +

+

{path}

+

{node.label}

+
+ +
+ +

+ We did not manage to fully index this part of your code — constructs here may be + missing from the graph (best-effort detection; the file content itself is ground + truth). +

+

+ Help us handle this edge case too: let your agent summarize what fails to parse + here and file a GitHub issue for the codebase-memory-mcp project. +

+ +
+ + + File a GitHub issue (prefilled) ↗ + +
+ +

+ The prefilled issue contains only the file path and project name — add code + snippets only if they are shareable. +

+
+ ); +} From 3eb294fa11d2e4de3a061bd52cdd05d0faab02ed Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 9 Jul 2026 03:38:34 +0200 Subject: [PATCH 8/8] fix(ui): serve the upstream issues URL from the backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UI security audit (security-ui.sh, layer A1) forbids hardcoded external URLs in graph-ui source — the callout's GitHub issues link tripped it on all pr-smoke legs. Follow the established pattern for external targets (/api/repo-info deep-links): /api/ui-config now carries upstream_issues_url and the callout consumes it, rendering the issue button only when the backend provides an https URL. The protocol check uses a regex literal on purpose: a bare protocol string in source also aborts the audit's URL extraction. Verified: security-ui.sh passes locally; /api/ui-config serves the URL; the callout opens with the prefilled link in the browser; frontend build + tests green. Refs #963 Signed-off-by: Martin Vogel --- graph-ui/src/components/MissedCallout.tsx | 63 +++++++++++++++++------ src/ui/http_server.c | 8 ++- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/graph-ui/src/components/MissedCallout.tsx b/graph-ui/src/components/MissedCallout.tsx index 86da6f257..1156d19b9 100644 --- a/graph-ui/src/components/MissedCallout.tsx +++ b/graph-ui/src/components/MissedCallout.tsx @@ -1,9 +1,24 @@ import { useEffect, useState } from "react"; import type { GraphNode } from "../lib/types"; -/* Upstream tracker for indexing gaps — issues land on the codebase-memory-mcp - * project itself (the tool that failed to index), never the user's repo. */ -const UPSTREAM_ISSUES_URL = "https://github.com/DeusData/codebase-memory-mcp/issues/new"; +/* Upstream tracker for indexing gaps. The URL is served by the backend + * (/api/ui-config) — the UI security audit forbids hardcoded external URLs in + * graph-ui source, so external targets come from an auditable backend + * response (same pattern as the /api/repo-info deep-links). */ +let issuesUrlRequest: Promise | null = null; + +function fetchIssuesUrl(): Promise { + issuesUrlRequest ??= fetch("/api/ui-config") + .then((r) => (r.ok ? r.json() : null)) + .then((cfg) => { + const url: unknown = cfg?.upstream_issues_url; + /* Accept only an https URL (regex literal on purpose — the UI security + * audit greps source for protocol strings). */ + return typeof url === "string" && /^https:\/\//.test(url) ? url : null; + }) + .catch(() => null); + return issuesUrlRequest; +} interface MissedCalloutProps { node: GraphNode; @@ -11,7 +26,7 @@ interface MissedCalloutProps { onClose: () => void; } -function buildIssueUrl(path: string, project: string | null): string { +function buildIssueUrl(base: string, path: string, project: string | null): string { const title = `Indexing gap: ${path}`; const body = [ "## Not fully indexed (best-effort coverage signal)", @@ -25,17 +40,20 @@ function buildIssueUrl(path: string, project: string | null): string { "", "_Reported from the graph UI's missed-coverage view._", ].join("\n"); - return `${UPSTREAM_ISSUES_URL}?title=${encodeURIComponent(title)}&body=${encodeURIComponent(body)}`; + return `${base}?title=${encodeURIComponent(title)}&body=${encodeURIComponent(body)}`; } -function buildAgentPrompt(path: string, project: string | null): string { +function buildAgentPrompt(issuesUrl: string | null, path: string, project: string | null): string { + const where = issuesUrl + ? `file a GitHub issue at ${issuesUrl}` + : "file a GitHub issue on the codebase-memory-mcp project"; return ( `codebase-memory-mcp could not fully index \`${path}\`` + (project ? ` (project \`${project}\`)` : "") + " — best-effort coverage signal. Please: " + "1) call the index_status MCP tool and note this file's flagged line ranges under parse_partial; " + "2) read those ranges in the file and summarize which construct fails to parse; " + - `3) file a GitHub issue at ${UPSTREAM_ISSUES_URL} titled "Indexing gap: ${path}" ` + + `3) ${where}, titled "Indexing gap: ${path}", ` + "with the summary — include a minimal reproducible snippet ONLY if the code is shareable." ); } @@ -46,6 +64,17 @@ function buildAgentPrompt(path: string, project: string | null): string { export function MissedCallout({ node, project, onClose }: MissedCalloutProps) { const path = node.file_path || node.name; const [copied, setCopied] = useState(false); + const [issuesUrl, setIssuesUrl] = useState(null); + + useEffect(() => { + let cancelled = false; + fetchIssuesUrl().then((url) => { + if (!cancelled) setIssuesUrl(url); + }); + return () => { + cancelled = true; + }; + }, []); useEffect(() => { if (!copied) return; @@ -55,7 +84,7 @@ export function MissedCallout({ node, project, onClose }: MissedCalloutProps) { const copyPrompt = async () => { try { - await navigator.clipboard.writeText(buildAgentPrompt(path, project)); + await navigator.clipboard.writeText(buildAgentPrompt(issuesUrl, path, project)); setCopied(true); } catch { /* clipboard unavailable (permissions/insecure context) — leave the @@ -103,14 +132,16 @@ export function MissedCallout({ node, project, onClose }: MissedCalloutProps) { > {copied ? "✓ Copied — paste it to your agent" : "Copy agent prompt"} - - File a GitHub issue (prefilled) ↗ - + {issuesUrl && ( + + File a GitHub issue (prefilled) ↗ + + )}

diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 4d26f0561..844896ce0 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -116,7 +116,13 @@ static void handle_ui_config(cbm_http_conn_t *c, const cbm_http_req_t *req) { if (cfg) { cbm_config_close(cfg); } - cbm_http_replyf(c, 200, g_cors_json, "{\"lang\":\"%s\"}", lang_buf); + /* upstream_issues_url: where the missed-coverage callout (#963) sends + * edge-case reports. Served from the backend on purpose — the UI security + * audit forbids hardcoded external URLs in graph-ui source (external + * targets must come from an auditable backend response, same pattern as + * the /api/repo-info deep-links). */ + cbm_http_replyf(c, 200, g_cors_json, "{\"lang\":\"%s\",\"upstream_issues_url\":\"%s\"}", + lang_buf, "https://github.com/DeusData/codebase-memory-mcp/issues/new"); } /* ── Server state ─────────────────────────────────────────────── */