From b9ac434a871b62df48157d17f581e0a42352c804 Mon Sep 17 00:00:00 2001 From: "Tapabrata \"Topo\" Pal" Date: Wed, 14 Oct 2026 21:52:29 -0400 Subject: [PATCH 1/3] java: error recovery never yields keyword-named members; orphaned members rejoin their class An unterminated char literal mid-edit made tree-sitter recover `if (x) {` as a constructor named "if" and `return bean;` as a field of type "return". A constructor must be named after its class, a method not after a keyword, and a field's type is never a keyword. Members recovered without an enclosing type are attached to the innermost type whose span contains them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KXa6Cv1ZUXcdMqXcJfQFhX --- strategies/extractors.go | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/strategies/extractors.go b/strategies/extractors.go index afa6598..9364702 100644 --- a/strategies/extractors.go +++ b/strategies/extractors.go @@ -1860,9 +1860,45 @@ func qualLast(path string) string { func extractJavaNodes(root *sitter.Node, filePath, blobSHA string, src []byte, imports []string) []astkit.Symbol { var out []astkit.Symbol javaVisit(root, filePath, blobSHA, src, imports, "", &out) + javaReattachOrphans(out) return out } +// javaReattachOrphans gives a member with no enclosing type the innermost +// type whose span contains it. Java has no top-level methods or fields: a +// parentless one comes from error recovery splitting a class body around a +// broken statement mid-edit (15 BeanDeserializer methods lost their class +// after an unterminated char literal, 2026-10-14). +func javaReattachOrphans(out []astkit.Symbol) { + for i := range out { + m := &out[i] + if m.ParentName != "" || (m.Kind != astkit.KindMethod && m.Kind != astkit.KindConstructor && m.Kind != astkit.KindField) { + continue + } + best := -1 + for j := range out { + t := out[j] + switch t.Kind { + case astkit.KindClass, astkit.KindInterface, astkit.KindEnum: + default: + continue + } + if t.Span.Start > m.Span.Start || t.Span.End < m.Span.End { + continue + } + if best < 0 || t.Span.End-t.Span.Start < out[best].Span.End-out[best].Span.Start { + best = j + } + } + if best < 0 { + continue + } + owner := out[best] + m.ParentName = owner.Name + m.QualifiedName = owner.QualifiedName + "." + m.Name + } +} + func javaVisit(node *sitter.Node, filePath, blobSHA string, src []byte, imports []string, parentClass string, out *[]astkit.Symbol) { for i := 0; i < int(node.ChildCount()); i++ { n := node.Child(i) @@ -1882,11 +1918,20 @@ func javaVisit(node *sitter.Node, filePath, blobSHA string, src []byte, imports if parentClass == "" { continue } + if name := n.ChildByFieldName("name"); name == nil || javaReservedWord[name.Content(src)] { + continue // recovered from statements, not a declaration + } javaMethodDecl(n, astkit.KindMethod, filePath, blobSHA, src, imports, parentClass, out) case "constructor_declaration": if parentClass == "" { continue } + // A constructor is named after its class. Error recovery reads + // `if (x) {` after an unterminated literal as a constructor + // named "if" (jackson BeanDeserializer edit, 2026-10-14). + if name := n.ChildByFieldName("name"); name == nil || name.Content(src) != parentClass[strings.LastIndexByte(parentClass, '.')+1:] { + continue + } javaMethodDecl(n, astkit.KindConstructor, filePath, blobSHA, src, imports, parentClass, out) case "compact_constructor_declaration": if parentClass == "" { @@ -2265,7 +2310,24 @@ func javaLambdaDecl(n *sitter.Node, filePath, blobSHA string, src []byte, import javaNestedExecutables(n, filePath, blobSHA, src, imports, parentClass, out) } +// javaReservedWord holds Java keywords and literals that can never be a +// declared name or a field's type. Error recovery around a broken statement +// (`return bean;`, `if (...) {`) produces such "declarations". +var javaReservedWord = map[string]bool{ + "abstract": true, "assert": true, "break": true, "case": true, "catch": true, "class": true, + "const": true, "continue": true, "default": true, "do": true, "else": true, "enum": true, + "extends": true, "final": true, "finally": true, "for": true, "goto": true, "if": true, + "implements": true, "import": true, "instanceof": true, "interface": true, "native": true, + "new": true, "package": true, "private": true, "protected": true, "public": true, + "return": true, "static": true, "strictfp": true, "super": true, "switch": true, + "synchronized": true, "this": true, "throw": true, "throws": true, "transient": true, + "try": true, "volatile": true, "while": true, "true": true, "false": true, "null": true, +} + func javaFieldDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports []string, parentClass string, out *[]astkit.Symbol) { + if t := n.ChildByFieldName("type"); t != nil && javaReservedWord[strings.TrimSpace(t.Content(src))] { + return // `return bean;` recovered as a field of type "return" + } raw := n.Content(src) // Full header, not FirstLine: an annotation on its own line before the // field made FirstLine return just "@Deprecated" as the signature. From 9f363b33cdf2b9178a7f339d07e85081712a6b85 Mon Sep 17 00:00:00 2001 From: "Tapabrata \"Topo\" Pal" Date: Wed, 14 Oct 2026 21:55:42 -0400 Subject: [PATCH 2/3] textmask: lex JSX elements; regex after an if/while/for condition JSX text children were lexed as code: `src/*.ts` opened a block comment to the end of the file, a backtick a template literal. grove masks callers' bodies to find local declarations and default exports, so TSX files got phantom callers (a locally shadowed helper bound to the imported one) and lost a default-export caller. Where an expression starts in TSX/JavaScript, ` Claude-Session: https://claude.ai/code/session_01KXa6Cv1ZUXcdMqXcJfQFhX --- textmask/textmask.go | 142 +++++++++++++++++++++++++++++++++++--- textmask/textmask_test.go | 23 ++++++ 2 files changed, 157 insertions(+), 8 deletions(-) diff --git a/textmask/textmask.go b/textmask/textmask.go index 724a7bd..b0c75e6 100644 --- a/textmask/textmask.go +++ b/textmask/textmask.go @@ -24,13 +24,13 @@ // Known gaps, all heuristic boundaries a lexer cannot settle without a // parser: // - JavaScript/TypeScript regex literals are told from division by the -// previous token. `)` and `]` mean division (so `if (x) /re/.test(s)` is -// misread), `}` means regex, except `{x} />` (a JSX self-closing tag). -// A TypeScript non-null `x! / 2` is division. -// - JSX/TSX text between tags is lexed as code: an apostrophe in -// `

Don't

` opens a quote that runs to the end of that line, and -// `//` in `http://x` starts a comment. A quoted string right -// after `=` may span lines (a JSX attribute value). +// previous token. `]` means division, `)` too unless it closes an +// if/while/for/with condition, `}` means regex, except `{x} />` (a JSX +// self-closing tag). A TypeScript non-null `x! / 2` is division. +// - JSX elements are recognised where an expression starts, in TSX and +// JavaScript (not plain TypeScript, where `x` is a type assertion); +// their text children are literal text. JSX that does not close +// masks to the end of the source. // - Swift regex literals (`/.../`, `#/.../#`) are not recognized. // - C/C++ strings inside `#define` bodies are masked although tree-sitter // treats a macro body as opaque text; `#error`/`#warning` text is kept. @@ -416,6 +416,11 @@ func (s *scanner) code(i int, closer byte, fmtStop bool) int { if s.fam == famJS { s.regexOK = true } + // condParens marks each open '(' that follows if/while/for/with: after + // its ')' a statement starts, so `/` opens a regex (`if (x) /re/.test(s)` + // was read as division). + var condParens []bool + lastWord := "" for i < n { c := s.src[i] switch c { @@ -424,6 +429,10 @@ func (s *scanner) code(i int, closer byte, fmtStop bool) int { continue case '(', '[', '{': depth++ + if c == '(' { + condParens = append(condParens, lastWord == "if" || lastWord == "while" || lastWord == "for" || lastWord == "with") + } + lastWord = "" i++ s.regexOK = true continue @@ -434,8 +443,14 @@ func (s *scanner) code(i int, closer byte, fmtStop bool) int { if depth > 0 { depth-- } + cond := false + if c == ')' && len(condParens) > 0 { + cond = condParens[len(condParens)-1] + condParens = condParens[:len(condParens)-1] + } + lastWord = "" i++ - s.regexOK = c == '}' + s.regexOK = c == '}' || cond continue case ':': if fmtStop && depth == 0 && !s.at(i+1, ":") && !s.at(i-1, ":") { @@ -459,12 +474,14 @@ func (s *scanner) code(i int, closer byte, fmtStop bool) int { if s.fam == famJS { s.regexOK = jsRegexKeyword(s.src[i:k]) } + lastWord = s.src[i:k] i = k continue } if isDigit(c) { i = s.number(i) s.regexOK = false + lastWord = "" continue } j := s.token(i) @@ -474,9 +491,11 @@ func (s *scanner) code(i int, closer byte, fmtStop bool) int { if j > i { i = j s.regexOK = false + lastWord = "" continue } // Plain punctuation. + lastWord = "" if s.fam == famJS { s.regexOK = jsRegexAfter(c) if c == '!' && i > 0 && (isIdentByte(s.src[i-1]) || s.src[i-1] == ')' || s.src[i-1] == ']') { @@ -641,6 +660,15 @@ func (s *scanner) token(i int) int { if s.regexOK && !(next == '>' && s.prevByte(i) == '}') { return s.jsRegex(i) } + case '<': + // Where an expression starts, `<` is never a comparison: in + // TSX/JSX it opens an element, whose text children are prose + // (`src/*.ts` opened a block comment to EOF, an + // apostrophe or backtick a literal). Plain TypeScript reads + // `x` as a type assertion, so it is excluded. + if s.regexOK && s.lang != "typescript" && s.jsxTagStart(i) { + return s.jsxElement(i) + } } case famPython: switch c { @@ -1561,3 +1589,101 @@ func (s *scanner) jcl() { ls = le + 1 } } + +// jsxTagStart reports whether `<` at i opens a JSX element or fragment: `<>` +// or a tag name followed by whitespace, `>`, `/` or an attribute — never a +// generic parameter list (`(x) =>`). +func (s *scanner) jsxTagStart(i int) bool { + k := i + 1 + if k < len(s.src) && s.src[k] == '>' { + return true // fragment + } + if k >= len(s.src) || !isIdentStart(s.src[k]) { + return false + } + for k < len(s.src) && (isIdentByte(s.src[k]) || s.src[k] == '.' || s.src[k] == '-' || s.src[k] == ':') { + k++ + } + if k >= len(s.src) { + return false + } + switch s.src[k] { + case ' ', '\t', '\n', '\r', '>', '/': + return true + } + return false +} + +// jsxTag lexes an opening or self-closing tag from `<` at i: attribute +// strings are literals, `{...}` attribute values are code. It returns the +// offset after the tag and whether it closed itself. +func (s *scanner) jsxTag(i int) (int, bool) { + n := len(s.src) + k := i + 1 + for k < n { + switch c := s.src[k]; c { + case '"', '\'': + k = s.quoted(k, k, c, false, true) + case '{': + k = s.hole(k+1, '}') + case '/': + if k+1 < n && s.src[k+1] == '>' { + return k + 2, true + } + k++ + case '>': + return k + 1, false + default: + k++ + } + } + return n, false +} + +// jsxElement lexes a JSX element from `<` at i through its closing tag: +// text children are literal text, `{...}` children are code, nested +// elements are followed. It returns the offset after the element. +func (s *scanner) jsxElement(i int) int { + n := len(s.src) + k, self := s.jsxTag(i) + if self { + return k + } + depth, seg := 1, k + for k < n { + switch s.src[k] { + case '{': + s.lit(seg, k) + k = s.hole(k+1, '}') + seg = k + case '<': + s.lit(seg, k) + if k+1 < n && s.src[k+1] == '/' { + j := strings.IndexByte(s.src[k:], '>') + if j < 0 { + return n + } + k += j + 1 + seg = k + if depth--; depth == 0 { + return k + } + continue + } + if !s.jsxTagStart(k) { + k++ // a stray `<` in text + seg = k - 1 + continue + } + j, closed := s.jsxTag(k) + k, seg = j, j + if !closed { + depth++ + } + default: + k++ + } + } + s.lit(seg, n) + return n +} diff --git a/textmask/textmask_test.go b/textmask/textmask_test.go index 492bad8..2a32f37 100644 --- a/textmask/textmask_test.go +++ b/textmask/textmask_test.go @@ -489,3 +489,26 @@ func BenchmarkMask(b *testing.B) { }) } } + +// JSX text is prose: `/*`, a backtick or an apostrophe in it opens nothing, +// and the code after the element is lexed as code. A regex after the `)` of +// an if condition is a regex, not division (grove phantom callers, +// 2026-10-14). +func TestMaskJSXTextAndConditionRegex(t *testing.T) { + for _, tc := range []struct{ lang, src, keep string }{ + {"tsx", "const a =

src/*.ts

\nfunction helper() {}\n", "function helper() {}"}, + {"tsx", "const a =

use `x` here

\nfunction helper() {}\n", "function helper() {}"}, + {"tsx", "const a =

Don't {name} go

\nfunction helper() {}\n", "function helper() {}"}, + {"javascript", "return (

text /* not */
)\nfunction helper() {}\n", "function helper() {}"}, + {"tsx", "const id = (x: T) => x\nfunction helper() {}\n", "function helper() {}"}, + {"typescript", "if (x) /`/.test(s)\nfunction helper() {}\n", "function helper() {}"}, + } { + got := Mask(tc.lang, tc.src) + if !strings.Contains(got, tc.keep) { + t.Errorf("%s %q: code after it was masked:\n%q", tc.lang, tc.src, got) + } + } + if got := Mask("tsx", "const a =

{name}

\n"); !strings.Contains(got, "{name}") { + t.Errorf("a {expr} child is code: %q", got) + } +} From d9d35ce0c931782e091cd68a2370f132fb09465f Mon Sep 17 00:00:00 2001 From: "Tapabrata \"Topo\" Pal" Date: Wed, 14 Oct 2026 22:12:42 -0400 Subject: [PATCH 3/3] rust: mark string-literal and range-index receivers of method calls `"test".as_bytes()` and `doc[start..].find('}')` arrived as bare calls ("as_bytes", "find"), which the graph resolved like free functions to every same-named function and method. The receiver is now kept as a marker, or , also for `"x".parse()` inside macros. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KXa6Cv1ZUXcdMqXcJfQFhX --- strategies/metadata.go | 36 ++++++++++++++++++++++++++++++++++-- strategies/metadata_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/strategies/metadata.go b/strategies/metadata.go index 04f8631..247e2b2 100644 --- a/strategies/metadata.go +++ b/strategies/metadata.go @@ -772,10 +772,14 @@ func rustCallSites(body *sitter.Node, src []byte) []astkit.CallSite { // SearcherTester::new(...).line_number(false) gives // line_number the "new()" call-result qualifier // instead of arriving bare. - qual := qualifierName(fn.ChildByFieldName("value"), src, + value := fn.ChildByFieldName("value") + qual := qualifierName(value, src, []string{"identifier", "self"}, map[string]string{"field_expression": "field", "scoped_identifier": "name"}, map[string]string{"call_expression": "function"}) + if qual == "" { + qual = rustExprReceiver(value) + } return joinQualified(qual, string(field.Content(src))) } case "scoped_identifier": @@ -845,7 +849,8 @@ func rustMacroCallSites(body *sitter.Node, src []byte) []astkit.CallSite { if tt := internalast.FindChildByType(n, "token_tree"); tt != nil { // Comments and string/char literals in the token tree // are prose, not calls: `assert!(ok, "see bar(1)")`. - text := textmask.Mask("rust", tt.Content(src)) + raw := tt.Content(src) + text := textmask.Mask("rust", raw) line := int(n.StartPoint().Row) + 1 for _, m := range rustMacroCallRe.FindAllStringSubmatchIndex(text, -1) { path := text[m[2]:m[3]] @@ -857,6 +862,8 @@ func rustMacroCallSites(body *sitter.Node, src []byte) []astkit.CallSite { } else if i := strings.LastIndexByte(path, '.'); i >= 0 { name = path[i+1:] qual = rustLastSegment(path[:i]) + } else if m[0] >= 2 && raw[m[0]-1] == '.' && raw[m[0]-2] == '"' { + qual = rustStrReceiver // `"match:fg:magenta".parse()` in vec![...] } else if q := rustMacroChainQualifier(text, m[0]); q != "" { // `ig.matched("", false).is_ignore()`: the regex // scan (no nested-structure awareness) matches @@ -1852,3 +1859,28 @@ func findChildByType(n *sitter.Node, typeName string) *sitter.Node { } return nil } + +// Receivers that are not a path: a string literal is a str and a range +// index (`doc[start..]`) a slice, so the call is a std method unless the +// repository implements one on those types. Left bare, the call matched +// every same-named function and method (ripgrep: "test".as_bytes(), +// doc[start..].find(...)). +const ( + rustStrReceiver = "" + rustSliceReceiver = "" +) + +func rustExprReceiver(value *sitter.Node) string { + if value == nil { + return "" + } + switch value.Type() { + case "string_literal", "raw_string_literal": + return rustStrReceiver + case "index_expression": + if internalast.FindChildByType(value, "range_expression") != nil { + return rustSliceReceiver + } + } + return "" +} diff --git a/strategies/metadata_test.go b/strategies/metadata_test.go index d691a82..3899e2f 100644 --- a/strategies/metadata_test.go +++ b/strategies/metadata_test.go @@ -1479,3 +1479,27 @@ func TestJava_AnonymousClassesAsLambdaBodiesAndEnumArguments(t *testing.T) { t.Errorf("anonymous=%d getUrl=%d lambdas=%d; want 3, 1, 3", anon, getURL, lambdas) } } + +// A method call on a string literal or a range index keeps a receiver +// marker instead of arriving bare: `"x".as_bytes()` is a str method and +// `doc[start..].find(..)` a slice method, not any same-named function. +func TestRust_LiteralAndSliceReceiversAreMarked(t *testing.T) { + src := `fn f(doc: &str, start: usize) { + let a = "test".as_bytes(); + let b = doc[start..].find('}'); + let c = vec!["x".parse().unwrap()]; +} +` + syms, _ := extract(t, astkit.LangRust, src) + got := map[string]bool{} + for _, s := range syms { + for _, cs := range s.CallSites { + got[cs.Callee] = true + } + } + for _, want := range []string{".as_bytes", ".find", ".parse"} { + if !got[want] { + t.Errorf("missing %q in %v", want, got) + } + } +}