diff --git a/eval/go.mod b/eval/go.mod index 1a74724..53ca54e 100644 --- a/eval/go.mod +++ b/eval/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/provasign/astkit v0.15.4 // indirect + github.com/provasign/astkit v0.16.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 // indirect github.com/sourcegraph/beaut v0.0.0-20240611013027-627e4c25335a // indirect diff --git a/eval/go.sum b/eval/go.sum index 707da78..743ac8c 100644 --- a/eval/go.sum +++ b/eval/go.sum @@ -18,8 +18,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/provasign/astkit v0.15.4 h1:Zdo9Dmu6hOcXh8iAijK9nUUIrsbLSdcZjobaprqdX1c= -github.com/provasign/astkit v0.15.4/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= +github.com/provasign/astkit v0.16.0 h1:0CGyrZex64wlqK9z6wiPyOLEk26Mhfi+yjWN1xGUaJk= +github.com/provasign/astkit v0.16.0/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/scip-code/scip/bindings/go/scip v0.8.1 h1:ZkyT9RDgkbd6ffvMbi9nYdwh7+gLoz8Z9ftFMxgUiVs= diff --git a/go.mod b/go.mod index 06eb117..43d65e9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26 require modernc.org/sqlite v1.51.0 require ( - github.com/provasign/astkit v0.15.4 + github.com/provasign/astkit v0.16.0 github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 golang.org/x/sys v0.46.0 ) diff --git a/go.sum b/go.sum index 25a4148..80ba73b 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/provasign/astkit v0.15.4 h1:Zdo9Dmu6hOcXh8iAijK9nUUIrsbLSdcZjobaprqdX1c= -github.com/provasign/astkit v0.15.4/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= +github.com/provasign/astkit v0.16.0 h1:0CGyrZex64wlqK9z6wiPyOLEk26Mhfi+yjWN1xGUaJk= +github.com/provasign/astkit v0.16.0/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= diff --git a/internal/graph/cfamilylocaltypes.go b/internal/graph/cfamilylocaltypes.go index 0ff5b73..91b113d 100644 --- a/internal/graph/cfamilylocaltypes.go +++ b/internal/graph/cfamilylocaltypes.go @@ -50,11 +50,9 @@ func cFamilyLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]str default: continue } - body := cls.RawText - if i := strings.IndexByte(body, '{'); i >= 0 { - body = body[i+1:] - } - for _, m := range cppLocalDeclRe.FindAllStringSubmatch(body, -1) { + // Top-level members only (classBody): an inline method's + // local or a commented-out declaration is not a field. + for _, m := range cppLocalDeclRe.FindAllStringSubmatch(classBody(idx, cls), -1) { if t := cFamilyBareType(m[1]); t != "" { if _, exists := out[m[2]]; !exists { record(m[2], t) @@ -69,14 +67,20 @@ func cFamilyLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]str } } - // Parameters. - for name, typ := range cFamilyParamTypes(symbol.Signature, symbol.RawText) { + // Parameters. Read on masked text: a symbol whose signature has no + // parameter list falls back to its body, and the first `(` there + // could be in a comment (`// ghost() { helper(1) }`), which hid the + // real parameters. + body := "" + if symbol.RawText != "" { + body = maskCode(symbol.Language, symbol.RawText) + } + for name, typ := range cFamilyParamTypes(maskCode(symbol.Language, symbol.Signature), body) { record(name, typ) } // Body locals (highest precedence): new-expressions. - if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + if body != "" { for _, m := range cppLocalDeclRe.FindAllStringSubmatch(body, -1) { if t := cFamilyBareType(m[1]); t != "" { record(m[2], t) diff --git a/internal/graph/changeimpact.go b/internal/graph/changeimpact.go index d9d072c..f4afe78 100644 --- a/internal/graph/changeimpact.go +++ b/internal/graph/changeimpact.go @@ -739,10 +739,7 @@ func declaredSuperNames(s *core.SymbolRecord) []string { if s.Kind != core.KindClass && s.Kind != core.KindInterface { return nil } - text := s.Signature - if text == "" { - text = firstLine(s.RawText) - } + text := declHeader(s) text = stripAngleBrackets(text) names := matchNameList(extendsRe, text) return append(names, matchNameList(implementsRe, text)...) @@ -753,29 +750,20 @@ func declaredSuperNames(s *core.SymbolRecord) []string { if s.Kind != core.KindClass && s.Kind != core.KindInterface && s.Kind != core.KindStruct { return nil } - text := s.Signature - if text == "" { - text = firstLine(s.RawText) - } + text := declHeader(s) return csharpBaseNames(stripAngleBrackets(text)) case "swift": if s.Kind != core.KindClass && s.Kind != core.KindStruct && s.Kind != core.KindEnum && s.Kind != core.KindInterface { return nil } - text := s.Signature - if text == "" { - text = firstLine(s.RawText) - } + text := declHeader(s) return swiftBaseNames(text) case "kotlin": if s.Kind != core.KindClass && s.Kind != core.KindInterface { return nil } - text := s.Signature - if text == "" { - text = firstLine(s.RawText) - } + text := declHeader(s) var out []string for _, raw := range kotlinBaseNames(text) { if name, _ := kotlinBaseNameAndCtor(raw); name != "" { @@ -787,10 +775,7 @@ func declaredSuperNames(s *core.SymbolRecord) []string { if s.Kind != core.KindClass && s.Kind != core.KindInterface { return nil } - text := s.Signature - if text == "" { - text = firstLine(s.RawText) - } + text := declHeader(s) var out []string if m := objcSuperclassRe.FindStringSubmatch(text); len(m) == 2 { out = append(out, m[1]) @@ -800,17 +785,7 @@ func declaredSuperNames(s *core.SymbolRecord) []string { if s.Kind != core.KindClass { return nil } - m := pythonClassBase.FindStringSubmatch(firstLine(s.RawText)) - if len(m) < 2 { - return nil - } - var out []string - for _, base := range splitTrim(m[1], ',') { - if base = stripPythonBase(base); base != "" { - out = append(out, base) - } - } - return out + return pythonClassBases(s) } // Go and Rust have NO declared-supertype clause on the type: Go interface // satisfaction is structural (derived by method-set inclusion, and only @@ -900,7 +875,7 @@ func (g *CodeGraph) typeDeclaresMember(t *core.SymbolRecord, method string) bool default: return false } - body := stripCommentsAndStrings(t.RawText) + body := maskCode(t.Language, t.RawText) if i := strings.IndexByte(body, '{'); i >= 0 { body = body[i+1:] } @@ -1357,7 +1332,7 @@ func paramTypesOf(s *core.SymbolRecord) []string { if !strings.Contains(src, ")") { src = s.RawText } - inner := tsDeclParams(src) + inner := tsDeclParams(s.Language, src) if s.Language == "go" { gi, ok := goDeclParamsOK(src) if !ok { @@ -1383,7 +1358,7 @@ func paramTypesOf(s *core.SymbolRecord) []string { return pyParamTypeTokens(inner) } var out []string - for _, gr := range splitTopLevel(inner, ',') { + for _, gr := range splitParams(s.Language, inner) { token := bareTypeToken(gr) switch s.Language { case "typescript", "tsx", "javascript": @@ -1405,7 +1380,7 @@ func paramTypesOf(s *core.SymbolRecord) []string { // so one unannotated parameter makes the whole list neutral (nil). func pyParamTypeTokens(inner string) []string { var out []string - for i, gr := range splitTopLevel(inner, ',') { + for i, gr := range splitParams("python", inner) { gr = strings.TrimSpace(gr) name, ann, annotated := strings.Cut(gr, ":") name = strings.TrimSpace(name) @@ -1443,7 +1418,7 @@ func pyParamTypeTokens(inner string) []string { // Returns nil (neutral evidence, like paramTypesOf) when a token cannot be // recovered. func goParamTokens(inner string) []string { - groups := splitTopLevel(inner, ',') + groups := splitParams("go", inner) n := len(groups) types := make([]string, n) named := false diff --git a/internal/graph/csharp_attribute_types_test.go b/internal/graph/csharp_attribute_types_test.go new file mode 100644 index 0000000..3347f69 --- /dev/null +++ b/internal/graph/csharp_attribute_types_test.go @@ -0,0 +1,36 @@ +package graph + +import ( + "reflect" + "testing" + + "github.com/provasign/grove/internal/core" +) + +func TestCSharpAttributeTypeNames(t *testing.T) { + idx := newEdgeIndex([]core.SymbolRecord{ + {ID: "a.cs::JsonPropertyAttribute", FilePath: "a.cs", Language: "csharp", Kind: core.KindClass, + Name: "JsonPropertyAttribute", QualifiedName: "JsonPropertyAttribute"}, + }) + for _, tc := range []struct { + ann string + want []string + }{ + {`[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]`, + []string{"JsonConverter", "StringEnumConverter", "CamelCaseNamingStrategy"}}, + {`[JsonProperty(NullValueHandling = NullValueHandling.Ignore, Order = 2)]`, + []string{"JsonPropertyAttribute", "NullValueHandling"}}, + {`[DataMember(Name = "first_name")]`, []string{"DataMember"}}, + {`[Obsolete("Use TypeNameAssemblyFormatHandling instead.")]`, []string{"Obsolete"}}, + {`[global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_Name", CanBeNull=false)]`, []string{"ColumnAttribute"}}, + {`[CompilationMapping(SourceConstructFlags.UnionCase, 1)]`, []string{"CompilationMapping", "SourceConstructFlags"}}, + {`[return: NotNull, Flags]`, []string{"NotNull", "Flags"}}, + {`[JsonConverter(typeof(Outer.Converter), new object[] { Mode.Fast /* Fake.Thing */ })]`, + []string{"JsonConverter", "Converter", "Mode"}}, + } { + s := core.SymbolRecord{Language: "csharp", Annotations: []string{tc.ann}} + if got := csharpAttributeTypeNames(idx, &s); !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s: got %v, want %v", tc.ann, got, tc.want) + } + } +} diff --git a/internal/graph/csharplocaltypes.go b/internal/graph/csharplocaltypes.go index fdbffac..96e1ff9 100644 --- a/internal/graph/csharplocaltypes.go +++ b/internal/graph/csharplocaltypes.go @@ -44,14 +44,9 @@ func csharpLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]stri } seen[className] = true for _, cls := range csharpTypeFragments(idx, className, dirOf(symbol.FilePath)) { - if cls.RawText == "" { - continue - } - body := cls.RawText - if i := strings.IndexByte(body, '{'); i >= 0 { - body = body[i+1:] - } - for _, m := range csharpFieldRe.FindAllStringSubmatch(body, -1) { + // Top-level members only (classBody): a method's local + // `Foo helper = ...` or a commented-out field is not one. + for _, m := range csharpFieldRe.FindAllStringSubmatch(classBody(idx, cls), -1) { if t := javaBareType(m[1]); t != "" && !csharpKeyword(t) { if _, exists := out[m[2]]; !exists { out[m[2]] = t @@ -66,8 +61,8 @@ func csharpLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]stri } // Parameters: "Type name" pairs from the declaration's paren group. - if params := tsDeclParams(symbol.RawText); params != "" { - for _, g := range splitTopLevel(params, ',') { + if params := tsDeclParams(symbol.Language, symbol.RawText); params != "" { + for _, g := range splitParams(symbol.Language, params) { fields := strings.Fields(strings.TrimSpace(g)) // Drop C# parameter modifiers and attributes. for len(fields) > 2 || (len(fields) == 2 && csharpParamModifier(fields[0])) { @@ -84,7 +79,7 @@ func csharpLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]stri // Body declarations (highest precedence). if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) if i := strings.IndexByte(body, '{'); i >= 0 { body = body[i+1:] } @@ -262,11 +257,11 @@ func csDeclSource(s *core.SymbolRecord) string { // csParamTypes parses a C# callable's parameter type tokens (normalized) // and whether the last one is a `params` array. func csParamTypes(s *core.SymbolRecord) (types []string, variadic bool) { - params := tsDeclParams(csDeclSource(s)) + params := tsDeclParams("csharp", csDeclSource(s)) if params == "" { return nil, false } - for _, g := range splitTopLevel(params, ',') { + for _, g := range splitParams("csharp", params) { g = strings.TrimSpace(g) if eq := strings.Index(g, "="); eq >= 0 { g = g[:eq] // default value @@ -314,8 +309,8 @@ func csharpArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string out[name] = typ } } - if params := tsDeclParams(csDeclSource(symbol)); params != "" { - for _, g := range splitTopLevel(params, ',') { + if params := tsDeclParams("csharp", csDeclSource(symbol)); params != "" { + for _, g := range splitParams("csharp", params) { g = strings.TrimSpace(g) if eq := strings.Index(g, "="); eq >= 0 { g = g[:eq] @@ -330,7 +325,7 @@ func csharpArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string } } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) if i := strings.IndexByte(body, '{'); i >= 0 { body = body[i+1:] } @@ -936,7 +931,8 @@ func csRecoverArgs(idx *edgeIndex, symbol *core.SymbolRecord, cs core.CallSite, if cs.Argc == 0 || cs.Line < symbol.Span.Start || symbol.RawText == "" { return nil } - lines := strings.Split(symbol.RawText, "\n") + // Masked: `Log("Foo(1, 2)"); Foo(x)` located the call inside the string. + lines := maskedSymbolLines(idx, symbol) off := cs.Line - symbol.Span.Start if off < 0 || off >= len(lines) { return nil @@ -992,16 +988,6 @@ func csRecoverArgs(idx *edgeIndex, symbol *core.SymbolRecord, cs core.CallSite, if depth == 0 { end = k } - case '"', '\'': - // Skip a literal: overload-relevant literals were already - // classified by the extractor, so any call reaching here has - // none at top level worth parsing exactly. - q := line[k] - for k++; k < len(line) && line[k] != q; k++ { - if line[k] == '\\' { - k++ - } - } } if end >= 0 { break diff --git a/internal/graph/csnamespace.go b/internal/graph/csnamespace.go index 570c357..eef758a 100644 --- a/internal/graph/csnamespace.go +++ b/internal/graph/csnamespace.go @@ -272,9 +272,9 @@ func csIsExtensionMethod(s *core.SymbolRecord) bool { if s.Language != "csharp" || s.Kind != core.KindMethod { return false } - params := tsDeclParams(s.Signature) + params := tsDeclParams(s.Language, s.Signature) if params == "" { - params = tsDeclParams(s.RawText) + params = tsDeclParams(s.Language, s.RawText) } first := strings.TrimSpace(strings.SplitN(params, ",", 2)[0]) return strings.HasPrefix(first, "this ") @@ -318,3 +318,124 @@ func csImplicitConstructorShadows(idx *edgeIndex, caller *core.SymbolRecord, cal } return false } + +// csharpAttributeTypeNames returns the type names a C# symbol's attribute +// sections ([JsonConverter(typeof(StringEnumConverter))], [JsonObject( +// MemberSerialization.OptIn)]) refer to: each attribute's class (Name + +// "Attribute" when the index holds that class, as C# resolves it), every +// typeof(T), and the type of each member-access value (Enum.Member, +// Ns.Enum.Member). The signature no longer carries attributes (astkit cuts +// them; they stay on Annotations), so buildUsesType reads them here, on +// masked text: a string argument or a named argument's name (`Name = "x"`) +// is never a type. +func csharpAttributeTypeNames(idx *edgeIndex, s *core.SymbolRecord) []string { + var out []string + for _, ann := range s.Annotations { + if !strings.HasPrefix(strings.TrimSpace(ann), "[") { + continue + } + out = csharpAttributeSectionTypes(idx, maskCode("csharp", ann), out) + } + return out +} + +func csharpAttributeSectionTypes(idx *edgeIndex, masked string, out []string) []string { + isIdent := func(c byte) bool { + return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + } + // dotted reads a (global::)Ns.Type.Member path at i: its segments and + // the index after it. + dotted := func(i int) ([]string, int) { + var segs []string + for { + for i < len(masked) && (masked[i] == ' ' || masked[i] == '\t' || masked[i] == '\n' || masked[i] == '\r') { + i++ + } + start := i + for i < len(masked) && isIdent(masked[i]) { + i++ + } + if start == i || masked[start] >= '0' && masked[start] <= '9' { + return segs, start + } + segs = append(segs, masked[start:i]) + j := i + for j < len(masked) && (masked[j] == ' ' || masked[j] == '\t') { + j++ + } + switch { + case strings.HasPrefix(masked[j:], "::"): + i = j + 2 + case j < len(masked) && masked[j] == '.': + i = j + 1 + default: + return segs, i + } + } + } + nextNonSpace := func(i int) byte { + for i < len(masked) && (masked[i] == ' ' || masked[i] == '\t' || masked[i] == '\n' || masked[i] == '\r') { + i++ + } + if i < len(masked) { + return masked[i] + } + return 0 + } + depth := 0 // parens inside the section + expectName := false // after '[' or a top-level ',': an attribute name + for i := 0; i < len(masked); { + c := masked[i] + switch { + case c == '[' && depth == 0: + expectName = true + i++ + case c == ',' && depth == 0: + expectName = true + i++ + case c == '(': + depth++ + i++ + case c == ')': + if depth > 0 { + depth-- + } + i++ + case isIdent(c) && (i == 0 || !isIdent(masked[i-1]) && masked[i-1] != '.'): + segs, end := dotted(i) + if end <= i { + i++ + continue + } + next := nextNonSpace(end) + switch { + case expectName && depth == 0 && next == ':' && !strings.HasPrefix(masked[end:], "::"): + // `[return: X]`: a target, the name follows. + end++ + case expectName && depth == 0: + name := segs[len(segs)-1] + if !strings.HasSuffix(name, "Attribute") && len(idx.byName[strings.ToLower(name+"Attribute")]) > 0 { + name += "Attribute" + } + out = append(out, name) + expectName = false + case len(segs) == 1 && segs[0] == "typeof" && next == '(': + open := strings.IndexByte(masked[end:], '(') + end + depth++ + end = open + 1 + if inner, after := dotted(end); len(inner) > 0 { + out = append(out, inner[len(inner)-1]) + end = after + } + case len(segs) >= 2 && next != '(': + // A member-access value: the segment before the member + // names its type (Enum.Member, Ns.Enum.Member). + out = append(out, segs[len(segs)-2]) + } + i = end + default: + i++ + } + } + return out +} diff --git a/internal/graph/edges.go b/internal/graph/edges.go index ffaa4b8..532c619 100644 --- a/internal/graph/edges.go +++ b/internal/graph/edges.go @@ -2,6 +2,7 @@ package graph import ( "fmt" + "github.com/provasign/astkit/textmask" "os" "path" "regexp" @@ -71,6 +72,11 @@ type edgeIndex struct { // walk once per call site otherwise. Values are READ-ONLY; nothing // mutates a returned map. qualifierFiles sync.Map // string("file\x00qualifier") → qualifierFileSet + // classBodies / maskedLines memoize classBody and maskedSymbolLines + // (localtypes.go) per symbol ID; classBodies also holds PHP trait rules + // (phpClassTraitRules) under a "php-traits\x00" key prefix. + classBodies sync.Map + maskedLines sync.Map // Rust crate topology: visibility in Rust is crate-wide (any item is // reachable through crate:: paths without a per-file use), so scope is @@ -351,7 +357,9 @@ func segmentSuffixIndex(m map[string][]string) map[string][]string { // candidates whose file lies under the last module segment before the // type (pcre2). Without a path nothing is preferred — an own-crate // preference was tried and lost 143 real cross-crate edges on ripgrep. -func rustPinByPath(idx *edgeIndex, symbol *core.SymbolRecord, cs core.CallSite, qualifier string, cands []*core.SymbolRecord) []*core.SymbolRecord { +// body is the caller's masked RawText (maskCode), so a path in a trailing +// comment on the call line pins nothing. +func rustPinByPath(idx *edgeIndex, symbol *core.SymbolRecord, body string, cs core.CallSite, qualifier string, cands []*core.SymbolRecord) []*core.SymbolRecord { if qualifier == "" || qualifier[0] < 'A' || qualifier[0] > 'Z' || cs.Line <= 0 || symbol.Span.Start <= 0 { return cands } @@ -361,7 +369,7 @@ func rustPinByPath(idx *edgeIndex, symbol *core.SymbolRecord, cs core.CallSite, } mod := "" off := cs.Line - symbol.Span.Start - lines := strings.Split(symbol.RawText, "\n") + lines := strings.Split(body, "\n") if off >= 0 && off < len(lines) { re := localFnRes.get(`((?:[a-z_][a-z0-9_]*::)+)` + regexp.QuoteMeta(qualifier) + `::` + regexp.QuoteMeta(name) + `\b`) if m := re.FindStringSubmatch(lines[off]); m != nil { @@ -588,17 +596,25 @@ func (idx *edgeIndex) buildRustInlineRefs(symbols []core.SymbolRecord) { if s.Language != "rust" || s.RawText == "" { continue } + // Comments and strings name paths too (`// Same contract as + // beta::normalize`); only code paths widen the scope. The body is + // masked only when the raw text mentions a workspace crate at all. + var segs []string for _, m := range rustInlinePathRe.FindAllStringSubmatch(s.RawText, -1) { - seg := m[1] - switch seg { - case "crate", "super", "self", "std", "core", "alloc": - continue + if idx.rustWorkspaceCrateSeg(m[1]) { + segs = append(segs, m[1]) } - if _, ok := idx.rustCrateByName[seg]; !ok { - if i := strings.LastIndexByte(seg, '_'); i < 0 || idx.rustCrateByName[seg[i+1:]] == "" { - continue - } + } + if len(segs) == 0 { + continue + } + segs = segs[:0] + for _, m := range rustInlinePathRe.FindAllStringSubmatch(maskCode(s.Language, s.RawText), -1) { + if idx.rustWorkspaceCrateSeg(m[1]) { + segs = append(segs, m[1]) } + } + for _, seg := range segs { if seen[s.FilePath] == nil { seen[s.FilePath] = map[string]bool{} } @@ -610,6 +626,21 @@ func (idx *edgeIndex) buildRustInlineRefs(symbols []core.SymbolRecord) { } } +// rustWorkspaceCrateSeg reports whether an inline path's leading segment +// names a workspace crate (or its last underscore token does: grep_regex -> +// regex). +func (idx *edgeIndex) rustWorkspaceCrateSeg(seg string) bool { + switch seg { + case "crate", "super", "self", "std", "core", "alloc": + return false + } + if _, ok := idx.rustCrateByName[seg]; ok { + return true + } + i := strings.LastIndexByte(seg, '_') + return i >= 0 && idx.rustCrateByName[seg[i+1:]] != "" +} + // buildPyModuleGlobals collects the name→class-type map from Python // module-level annotated variables (KindVariable symbols). A name declared // with two different types in different modules is ambiguous and dropped, so @@ -683,8 +714,10 @@ var pyModuleVarCtorRe = regexp.MustCompile(`^\s*([A-Za-z_]\w*)\s*=\s*(?:\w+\.)*( // creationSite reports whether the call site's source line instantiates // the callee (`new Name(` / `new Name<`), for the languages whose -// extractors emit the type name as the callee of an object creation. -func creationSite(symbol *core.SymbolRecord, cs core.CallSite) bool { +// extractors emit the type name as the callee of an object creation. body is +// the caller's masked RawText (maskCode): `new X(` in a comment or string on +// the call line is not a creation. +func creationSite(symbol *core.SymbolRecord, body string, cs core.CallSite) bool { switch symbol.Language { case "java", "csharp", "typescript", "tsx", "javascript", "php": default: @@ -697,7 +730,7 @@ func creationSite(symbol *core.SymbolRecord, cs core.CallSite) bool { if off < 0 { return false } - lines := strings.Split(symbol.RawText, "\n") + lines := strings.Split(body, "\n") if off >= len(lines) { return false } @@ -708,9 +741,10 @@ func creationSite(symbol *core.SymbolRecord, cs core.CallSite) bool { // function named name: JS/TS `function name(`, Python `def name(`, Go // `name := func`, Rust `fn name(`. The declaration line of the caller // itself (which also matches `def name(` when the caller IS name) is -// excluded by requiring the match to start after the first line. -func declaresLocalFunction(symbol *core.SymbolRecord, name string) bool { - body := symbol.RawText +// excluded by requiring the match to start after the first line. body is the +// caller's masked RawText (maskCode): a `def name(` in a docstring example, a +// comment or a template literal declares nothing. +func declaresLocalFunction(symbol *core.SymbolRecord, body, name string) bool { if body == "" || name == "" { return false } @@ -805,13 +839,13 @@ func sortedFileKeys(m map[string][]*core.SymbolRecord) []string { // because package names commonly prefix the directory name (grep-searcher // lives in crates/searcher). func (idx *edgeIndex) buildRustCrates() { - roots := map[string]bool{} + roots := map[string]struct{}{} for _, f := range sortedFileKeys(idx.byFile) { if !strings.HasSuffix(f, ".rs") { continue } if base := baseNameNoExt(f); base == "lib" || base == "main" { - roots[dirOf(f)] = true + roots[dirOf(f)] = struct{}{} } // Cargo's integration tests, benches and examples are crates of // their own: every file directly under tests/ is a crate root, @@ -819,7 +853,7 @@ func (idx *edgeIndex) buildRustCrates() { // Without a root they scoped to their own file only (fd's // TestEnv::new from tests/tests.rs resolved nothing). if d := baseOf(dirOf(f)); d == "tests" || d == "benches" || d == "examples" { - roots[dirOf(f)] = true + roots[dirOf(f)] = struct{}{} } } if len(roots) == 0 { @@ -834,7 +868,7 @@ func (idx *edgeIndex) buildRustCrates() { } root := dirOf(f) for d := root; ; { - if roots[d] { + if _, ok := roots[d]; ok { root = d break } @@ -854,13 +888,29 @@ func (idx *edgeIndex) buildRustCrates() { } return strings.ToLower(strings.ReplaceAll(name, "-", "_")) } - for root := range roots { + // Integration-test, bench and example targets are crates nothing can + // name: registering them put every `tests`/`benches` root under the + // one name, and a library's own `mod tests` paths (`tests::helper`) + // pulled whichever one won the map iteration into the library's scope. + // Names are registered in sorted root order so a clash between two + // real crates resolves the same way on every run. + var named []string + for _, root := range sortedKeys(roots) { + switch baseOf(root) { + case "tests", "benches", "examples": + continue + } + named = append(named, root) + } + for _, root := range named { if name := crateName(root); name != "" && name != "." { - idx.rustCrateByName[name] = root + if _, dup := idx.rustCrateByName[name]; !dup { + idx.rustCrateByName[name] = root + } } } // Token aliases never displace an exact crate name. - for root := range roots { + for _, root := range named { name := crateName(root) if i := strings.LastIndexByte(name, '_'); i >= 0 && i+1 < len(name) { if tok := name[i+1:]; idx.rustCrateByName[tok] == "" { @@ -1329,185 +1379,125 @@ func trimExt(path string) string { return path } -// stripCommentsAndStrings removes // line comments, /* */ block comments, -// # python comments, and string literals from a source body so that -// regex-based call matching does not produce false positives. -func stripCommentsAndStrings(src string) string { - var out strings.Builder - out.Grow(len(src)) - i, n := 0, len(src) - for i < n { - ch := src[i] - // Block comment - if ch == '/' && i+1 < n && src[i+1] == '*' { - end := strings.Index(src[i+2:], "*/") - if end < 0 { - return out.String() - } - i += end + 4 - continue - } - // Line comment // and # - if (ch == '/' && i+1 < n && src[i+1] == '/') || ch == '#' { - for i < n && src[i] != '\n' { - i++ - } - continue - } - // A slash after an expression opener is a JavaScript/TypeScript regex - // literal. Skip it as one unit so quotes inside /["']/ do not open a - // fake string that consumes the rest of the source. - if ch == '/' && looksLikeRegexLiteral(src, i) { - out.WriteByte(' ') - i++ - inClass := false - for i < n { - if src[i] == '\\' && i+1 < n { - i += 2 - continue - } - if src[i] == '[' { - inClass = true - } else if src[i] == ']' { - inClass = false - } else if src[i] == '/' && !inClass { - i++ - for i < n && isASCIIAlpha(src[i]) { - i++ - } - break - } - if src[i] == '\n' { - out.WriteByte('\n') - } - i++ - } - continue - } - // Rust lifetimes ('a, 'static) are identifiers, not character strings. - if ch == '\'' && rustLifetimeAt(src, i) { - out.WriteByte(ch) - i++ - continue - } - // String literal — preserve newlines and executable interpolation holes. - if ch == '"' || ch == '\'' || ch == '`' { - quote := ch - prefix := literalPrefix(src, i) - raw := strings.Contains(prefix, "@") || strings.ContainsAny(prefix, "rR") - interpolated := quote == '`' || strings.Contains(prefix, "$") || strings.ContainsAny(prefix, "fF") - out.WriteByte(' ') - i++ - for i < n { - if interpolated { - open := -1 - if quote == '`' && src[i] == '$' && i+1 < n && src[i+1] == '{' { - open = i + 1 - } else if quote != '`' && src[i] == '{' && (i+1 >= n || src[i+1] != '{') { - open = i - } - if open >= 0 { - if end := matchingBrace(src, open); end >= 0 { - out.WriteByte(' ') - out.WriteString(stripCommentsAndStrings(src[open+1 : end])) - out.WriteByte(' ') - i = end + 1 - continue - } - } - } - if src[i] == quote { - if raw && i+1 < n && src[i+1] == quote { - i += 2 // C# verbatim doubled quote - continue - } - i++ - break - } - if !raw && src[i] == '\\' && i+1 < n { - i += 2 - continue - } - if src[i] == '\n' { - out.WriteByte('\n') - } - i++ - } - continue - } - out.WriteByte(ch) - i++ - } - return out.String() -} - -func isASCIIAlpha(ch byte) bool { - return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' +// maskCode blanks comments and string literals in src (a symbol body in +// language lang) so regex scans only see code. Offsets and line breaks are +// kept, so a line index into the result is a line index into src. +// Interpolation holes stay as code. +func maskCode(lang, src string) string { + if !textmask.Supported(lang) { + lang = "c" // unknown languages get the generic C-style lexer + } + if lang == "cobol" { + // The parser normalizes COBOL symbol text: the sequence and + // indicator areas (columns 1-7) and the identification area are + // cut off, so a statement starts in column 1. textmask defaults + // to fixed format and would skip the first 7 bytes of every line + // (and open a literal on a quote it never saw start); lex the + // normalized text as free format instead. + const free = ">>SOURCE FORMAT FREE\n" + return textmask.Mask(lang, free+src)[len(free):] + } + return textmask.Mask(lang, src) } -func rustLifetimeAt(src string, quote int) bool { - if quote+1 >= len(src) || !(isASCIIAlpha(src[quote+1]) || src[quote+1] == '_') { - return false - } - i := quote + 2 - for i < len(src) && (isASCIIAlpha(src[i]) || src[i] == '_' || src[i] >= '0' && src[i] <= '9') { - i++ +// maskedLine returns line idx of maskCode(lang, src). Masking the whole body +// keeps multi-line comments and strings masked on their middle lines. +func maskedLine(lang, src string, idx int) string { + lines := strings.Split(maskCode(lang, src), "\n") + if idx < 0 || idx >= len(lines) { + return "" } - return i >= len(src) || src[i] != '\'' + return lines[idx] } -func literalPrefix(src string, quote int) string { - i := quote - for i > 0 { - ch := src[i-1] - if ch != '$' && ch != '@' && ch != 'f' && ch != 'F' && ch != 'r' && ch != 'R' { - break - } - i-- +// codeSignature returns s.Signature with comments and string literals +// blanked, for scans that parse a signature as code (base lists, type +// names). Extractors collapse some multi-line headers onto one line (Python +// and Kotlin class headers), which turns a `# ...` or `// ...` comment into +// one running to the end of the signature: masking that text on its own +// would hide the code after it. So the signature is aligned with the +// symbol's RawText, where the header keeps its line breaks, and masked by +// RawText's lexing. Only a signature that does not align is masked alone. +func codeSignature(s *core.SymbolRecord) string { + sig := s.Signature + if sig == "" { + return "" } - return src[i:quote] -} - -func matchingBrace(src string, open int) int { - depth := 0 - for i := open; i < len(src); i++ { - switch src[i] { - case '{': - depth++ - case '}': - depth-- - if depth == 0 { - return i - } + if s.RawText != "" && !strings.HasPrefix(s.RawText, sig) { + if masked, ok := maskAlignedSignature(s.Language, sig, s.RawText); ok { + return masked } } - return -1 + return maskCode(s.Language, sig) } -func looksLikeRegexLiteral(src string, slash int) bool { - prev := byte(0) - for i := slash - 1; i >= 0; i-- { - if src[i] != ' ' && src[i] != '\t' && src[i] != '\r' && src[i] != '\n' { - prev = src[i] - break +// maskAlignedSignature masks sig by the lexing of raw, the source text it +// was derived from. The two are aligned byte for byte ignoring whitespace, +// starting at the first occurrence in raw of sig's first token that aligns +// through the end of sig. ok is false when no alignment exists (the +// signature was synthesized or reordered). +func maskAlignedSignature(lang, sig, raw string) (string, bool) { + sig0 := strings.TrimLeft(sig, " \t\r\n") + if sig0 == "" { + return sig, true + } + lead := len(sig) - len(sig0) + first := sig0 + if i := strings.IndexAny(first, " \t\r\n"); i >= 0 { + first = first[:i] + } + pos := make([]int, len(sig)) + for from, tries := 0, 0; tries < 4; tries++ { + start := strings.Index(raw[from:], first) + if start < 0 { + return "", false + } + start += from + from = start + 1 + j := start + ok := true + for i := lead; i < len(sig); i++ { + c := sig[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + pos[i] = -1 + continue + } + for j < len(raw) && (raw[j] == ' ' || raw[j] == '\t' || raw[j] == '\n' || raw[j] == '\r') { + j++ + } + if j >= len(raw) || raw[j] != c { + ok = false + break + } + pos[i] = j + j++ } - } - if prev != 0 && !strings.ContainsRune("(=,:;!&|?{[", rune(prev)) { - return false - } - for i := slash + 1; i < len(src); i++ { - if src[i] == '\\' { - i++ + if !ok { continue } - if src[i] == '\n' { - return false - } - if src[i] == '/' { - return true + masked := maskCode(lang, raw[:j]) + out := []byte(sig) + for i := lead; i < len(sig); i++ { + if p := pos[i]; p >= 0 && masked[p] == ' ' { + out[i] = ' ' + } } + return string(out), true } - return false + return "", false +} + +// declHeader is the masked declaration header of a type: its signature, or +// the first line of its source when the extractor left no signature. +func declHeader(s *core.SymbolRecord) string { + if text := codeSignature(s); text != "" { + return text + } + return maskCode(s.Language, firstLine(s.RawText)) +} + +func isASCIIAlpha(ch byte) bool { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' } // buildDefinesAndImports emits "file → symbol" defines edges and @@ -1629,7 +1619,7 @@ func buildContains(idx *edgeIndex, symbols []core.SymbolRecord) []core.Edge { var ( extendsRe = regexp.MustCompile(`\bextends\s+([A-Za-z_][A-Za-z0-9_.]*(?:\s*,\s*[A-Za-z_][A-Za-z0-9_.]*)*)`) implementsRe = regexp.MustCompile(`\bimplements\s+([A-Za-z_][A-Za-z0-9_.]*(?:\s*,\s*[A-Za-z_][A-Za-z0-9_.]*)*)`) - pythonClassBase = regexp.MustCompile(`^\s*class\s+[A-Za-z_][A-Za-z0-9_]*\s*\(([^)]+)\)`) + pythonClassHead = regexp.MustCompile(`^\s*class\s+[A-Za-z_][A-Za-z0-9_]*\s*(?:\[[^\]]*\]\s*)?`) rustImplForRe = regexp.MustCompile(`\bimpl(?:\s*<[^>]+>)?\s+([A-Za-z_][A-Za-z0-9_:]*)(?:<[^{}]+>)?\s+for\s+&?\s*([A-Za-z_][A-Za-z0-9_:]*)(?:<[^{}]+>)?`) usesTypeIdent = regexp.MustCompile(`\b([A-Z][A-Za-z0-9_]+)\b`) goEmbeddedTypeRe = regexp.MustCompile(`^\s*\*?((?:[A-Za-z_][A-Za-z0-9_]*\.)?[A-Za-z_][A-Za-z0-9_]*)(?:\[[^]\n]+\])?\s*(?://.*)?$`) @@ -1671,10 +1661,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. if symbol.Kind != core.KindClass && symbol.Kind != core.KindInterface { continue } - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) // Drop balanced <...> sections so generic arguments and bounds // (`class Foo extends Baz`) neither confuse // the extends regex nor emit bogus edges to type parameters. @@ -1697,10 +1684,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. if symbol.Kind != core.KindClass && symbol.Kind != core.KindInterface { continue } - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) for i, name := range csharpBaseNames(text) { edgeType := core.EdgeImplements if symbol.Kind == core.KindClass && i == 0 && !strings.HasPrefix(name, "I") { @@ -1712,10 +1696,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. if symbol.Kind != core.KindClass && symbol.Kind != core.KindInterface && symbol.Kind != core.KindTrait { continue } - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) for _, name := range inheritanceClauseTypes(text, "extends", "implements") { edges = append(edges, resolveTypeEdges(idx, symbol, name, core.EdgeExtends, 0.85)...) } @@ -1753,16 +1734,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. if symbol.Kind != core.KindClass { continue } - line := firstLine(symbol.RawText) - matches := pythonClassBase.FindStringSubmatch(line) - if len(matches) < 2 { - continue - } - for _, base := range splitTrim(matches[1], ',') { - base = stripPythonBase(base) - if base == "" { - continue - } + for _, base := range pythonClassBases(&symbol) { edges = append(edges, resolveTypeEdges(idx, symbol, base, core.EdgeExtends, 0.85)...) } case "rust": @@ -1806,10 +1778,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. symbol.Kind != core.KindEnum && symbol.Kind != core.KindInterface { continue } - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) for i, name := range swiftBaseNames(text) { if name == "" { continue @@ -1828,10 +1797,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. if symbol.Kind != core.KindClass && symbol.Kind != core.KindInterface { continue } - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) for _, raw := range kotlinBaseNames(text) { name, isCtorCall := kotlinBaseNameAndCtor(raw) if name == "" { @@ -1853,10 +1819,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. // function treats one interface extending another elsewhere). switch symbol.Kind { case core.KindClass: - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) if m := objcSuperclassRe.FindStringSubmatch(text); len(m) == 2 { edges = append(edges, resolveTypeEdges(idx, symbol, m[1], core.EdgeExtends, 0.85)...) } @@ -1864,10 +1827,7 @@ func buildExtendsImplements(idx *edgeIndex, symbols []core.SymbolRecord) []core. edges = append(edges, resolveTypeEdges(idx, symbol, name, core.EdgeImplements, 0.85)...) } case core.KindInterface: - text := symbol.Signature - if text == "" { - text = firstLine(symbol.RawText) - } + text := declHeader(&symbol) for _, name := range objcProtocolNames(text) { edges = append(edges, resolveTypeEdges(idx, symbol, name, core.EdgeExtends, 0.85)...) } @@ -1960,7 +1920,13 @@ func buildUsesType(idx *edgeIndex, symbols []core.SymbolRecord) []core.Edge { continue } scope := idx.importedFiles(symbol.FilePath) - matches := usesTypeIdent.FindAllStringSubmatch(symbol.Signature, -1) + sig := codeSignature(&symbol) + matches := usesTypeIdent.FindAllStringSubmatch(sig, -1) + if symbol.Language == "csharp" && len(symbol.Annotations) > 0 { + for _, name := range csharpAttributeTypeNames(idx, &symbol) { + matches = append(matches, []string{name, name}) + } + } for _, m := range matches { candidateName := m[1] if candidateName == symbol.Name { @@ -1968,7 +1934,8 @@ func buildUsesType(idx *edgeIndex, symbols []core.SymbolRecord) []core.Edge { } var targets []*core.SymbolRecord for _, target := range idx.byName[strings.ToLower(candidateName)] { - if target.ID == symbol.ID || target.Name != candidateName { + if target.ID == symbol.ID || target.Name != candidateName || + !typeRefLanguagesCompatible(symbol.Language, target.Language) { continue } switch target.Kind { @@ -1982,7 +1949,7 @@ func buildUsesType(idx *edgeIndex, symbols []core.SymbolRecord) []core.Edge { targets = append(targets, target) } if symbol.Language == "cpp" && len(targets) > 1 { - qualified := localFnRes.get(`\b(?:[A-Za-z_]\w*::)+`+regexp.QuoteMeta(candidateName)+`\b`).FindAllString(symbol.Signature, -1) + qualified := localFnRes.get(`\b(?:[A-Za-z_]\w*::)+`+regexp.QuoteMeta(candidateName)+`\b`).FindAllString(sig, -1) var pinned []*core.SymbolRecord if len(qualified) > 0 { for _, target := range targets { @@ -2095,6 +2062,21 @@ func callLanguagesCompatible(caller, candidate string) bool { return tsFamilyLang(caller) && tsFamilyLang(candidate) } +// typeRefLanguagesCompatible reports whether a signature in language from +// can name a type declared in language to: the call-compatible pairs plus +// the interop families whose signatures share types (Java/Kotlin on the JVM; +// C, C++, Objective-C and Swift through Clang modules). Without it a Kotlin +// signature naming Gadget took a uses-type edge to a Python class Gadget in +// the same directory. +func typeRefLanguagesCompatible(from, to string) bool { + if callLanguagesCompatible(from, to) { + return true + } + jvm := func(l string) bool { return l == "java" || l == "kotlin" } + native := func(l string) bool { return l == "c" || l == "cpp" || l == "objc" || l == "swift" } + return jvm(from) && jvm(to) || native(from) && native(to) +} + func sameCSharpPrivateScope(callerOwner, candidateOwner string) bool { if callerOwner == "" || candidateOwner == "" { return false @@ -2117,6 +2099,11 @@ func graphCallableSymbol(symbol *core.SymbolRecord) bool { return false } text := symbol.Signature + "\n" + symbol.RawText + if !strings.Contains(text, "=>") && !strings.Contains(text, "function") { + return false + } + // `label = "a => b"` is a string field, not a callable one. + text = maskCode(symbol.Language, text) return strings.Contains(text, "=>") || strings.Contains(text, "function") } @@ -2329,6 +2316,17 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa } var javaArgTypeCache map[string]string var csArgTypeCache map[string]string + // codeBody is the caller's RawText with comments and strings + // masked, for the source-line checks below; masked once, on first + // use. + var maskedBody *string + codeBody := func() string { + if maskedBody == nil { + m := maskCode(symbol.Language, symbol.RawText) + maskedBody = &m + } + return *maskedBody + } var ktShapeCache map[string]string javaTypesFor := func(cands []*core.SymbolRecord, cs core.CallSite) map[string]string { if javaArgTypeCache == nil { @@ -2473,7 +2471,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa // receiver syntax, so the same source recovery applies: // `w.serializer.serialize(...)` → chain "w.serializer"). if (tsFamilyLang(symbol.Language) || symbol.Language == "java") && qualifier != "" && cs.Line > 0 { - if chain := tsReceiverChainAt(symbol.RawText, cs.Line-symbol.Span.Start, calleeName); chain != "" { + if chain := tsReceiverChainAt(idx, &symbol, cs.Line-symbol.Span.Start, calleeName); chain != "" { fullChain = chain } } @@ -2483,7 +2481,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa // constructor in scope. continue } - localDefinition := qualifier == "" && symbol.Name != "" && declaresLocalFunction(&symbol, calleeName) + localDefinition := qualifier == "" && symbol.Name != "" && declaresLocalFunction(&symbol, codeBody(), calleeName) if localDefinition && (symbol.Language != "python" || len(pythonLexicalChild(idx, &symbol, calleeName)) == 0) { // A function declared inside the caller's own body // (`function run(i)` nested in Socket.run, a nested `def`) @@ -2677,7 +2675,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa fmt.Fprintf(os.Stderr, "grove-trace %s: callee=%q qual=%q args=%v cands=%d capped=%v scope=%d first=%v\n", symbol.QualifiedName, cs.Callee, qualifier, cs.Args, len(cands), capped, len(scope), ids) } if symbol.Language == "python" && qualifier != "" && cs.Argc > 0 && len(cands) > 0 && - !pyCallHasSplat(&symbol, cs) { + !pyCallHasSplat(idx, &symbol, cs) { // Python has no overloading: a method whose parameters cannot // take this call's argument count is not its target (click: // ctx.invoke(other_cmd, arg=42) is not Command.invoke(self, ctx)). @@ -2898,7 +2896,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa cands = csNarrowOverloads(idx, cands, cs.Args, csArgTypeCache) } } - if creationSite(&symbol, cs) { + if creationSite(&symbol, codeBody(), cs) { // `new X(...)`: only a constructor can be the target. A // same-named method elsewhere (a test named List() for // `new List()`) is never it. @@ -3132,7 +3130,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa } } if symbol.Language == "rust" && qualifier != "" && len(cands) > 1 { - cands = rustPinByPath(idx, &symbol, cs, qualifier, cands) + cands = rustPinByPath(idx, &symbol, codeBody(), cs, qualifier, cands) } if symbol.Language == "rust" && qualifier == "" && len(cands) > 0 && !anyInFile(cands, symbol.FilePath) && rustImportedExternal(idx, &symbol, calleeName) { // `use regex_syntax::escape;` then `escape(pattern)`: the @@ -3217,7 +3215,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa if qualifier == "super()" || qualifier == "super" || (symbol.Language == "csharp" && qualifier == "base") || (symbol.Language == "php" && qualifier == "parent") { if traceCalls { - fmt.Fprintf(os.Stderr, "grove-trace %s: super bases=%v matched=%d\n", symbol.QualifiedName, baseClassesFor(idx, symbol.Language, symbol.ParentSymbol, dirOf(symbol.FilePath)), len(narrowBySuper(idx, &symbol, cands))) + fmt.Fprintf(os.Stderr, "grove-trace %s: super bases=%v matched=%d\n", symbol.QualifiedName, baseClassesInFileFor(idx, symbol.Language, symbol.ParentSymbol, symbol.FilePath), len(narrowBySuper(idx, &symbol, cands))) } viaSuper := narrowBySuper(idx, &symbol, cands) // `base.WriteValue(x)` binds ONE overload on the base type, @@ -3577,7 +3575,7 @@ func resolveCallEdges(idx *edgeIndex, symbol core.SymbolRecord, sat *interfaceSa if symbol.RawText == "" { return edges } - stripped := stripCommentsAndStrings(symbol.RawText) + stripped := maskCode(symbol.Language, symbol.RawText) if hasAnnotation(&symbol, "syntax-recovery") { stripped = stripRecoveredNestedDeclarations(symbol.Language, stripped) } @@ -3932,7 +3930,7 @@ func declParamCount(s *core.SymbolRecord) (int, bool, bool) { if s.Language == "go" { params, parsed = goDeclParamsOK(src) } else { - params = tsDeclParams(src) + params = tsDeclParams(s.Language, src) parsed = params != "" || strings.Contains(src, "()") } if params == "" { @@ -3941,7 +3939,7 @@ func declParamCount(s *core.SymbolRecord) (int, bool, bool) { } return 0, false, false } - groups := splitTopLevel(params, ',') + groups := splitParams(s.Language, params) if s.Language == "java" { groups = splitJavaParamGroups(params) } @@ -4333,10 +4331,69 @@ func splitTrim(s string, sep byte) []string { return out } +// pythonClassBases returns the simple names of a Python class's bases. The +// base list is read from the whole masked header, so a list spanning lines +// or carrying comments (`class C(Base, # the (old) Mixin`) keeps every +// base. The signature normally holds the full header; when it stops early, +// the header is read from the masked source. +func pythonClassBases(s *core.SymbolRecord) []string { + list, ok := pythonClassBaseList(s) + if !ok { + return nil + } + var out []string + for _, base := range splitTopLevel(list, ',') { + if base = stripPythonBase(base); base != "" { + out = append(out, base) + } + } + return out +} + +// pythonClassBaseList returns the masked text between the parentheses of a +// Python class header, read from the signature or, when the signature is cut +// off, from the class text. Comments in the header are masked. +func pythonClassBaseList(s *core.SymbolRecord) (string, bool) { + list, ok := pythonBaseList(codeSignature(s)) + if !ok && s.RawText != "" { + list, ok = pythonBaseList(maskCode(s.Language, s.RawText)) + } + return list, ok +} + +// pythonBaseList returns the text between the parentheses of a masked +// `class Name(...)` header. ok is false when the header is cut off before +// its closing parenthesis; a header with no parentheses has no bases. +func pythonBaseList(header string) (string, bool) { + loc := pythonClassHead.FindStringIndex(header) + if loc == nil { + return "", false + } + rest := header[loc[1]:] + if !strings.HasPrefix(rest, "(") { + return "", true + } + depth := 0 + for i := 0; i < len(rest); i++ { + switch rest[i] { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + if depth == 0 { + return rest[1:i], true + } + } + } + return "", false +} + func stripPythonBase(b string) string { b = strings.TrimSpace(b) // Strip default ABC bases that add noise (object, Generic[T], etc.) - if strings.HasPrefix(b, "metaclass=") || b == "object" { + if strings.Contains(b, "=") || strings.HasPrefix(b, "*") || b == "object" { + // Keyword arguments (metaclass=M, total=False) and unpacked + // base lists are not bases. return "" } if i := strings.Index(b, "["); i >= 0 { @@ -4516,7 +4573,10 @@ func objcProtocolNames(text string) []string { func goEmbeddedTypes(body string) []string { // Look at lines between the first `{` and the matching `}` of the struct // declaration. We consider each non-empty line consisting of a single - // identifier (or *Identifier) to be an embedded type. + // identifier (or *Identifier) to be an embedded type. Comments and + // struct tags are masked first: a lone name inside a /* */ block is + // prose, and a tagged embed (`Base `+"`json:\"b\"`"+`) is still an embed. + body = maskCode("go", body) open := strings.IndexByte(body, '{') if open < 0 { return nil diff --git a/internal/graph/edges_test.go b/internal/graph/edges_test.go index bc8cc3d..cb93bf2 100644 --- a/internal/graph/edges_test.go +++ b/internal/graph/edges_test.go @@ -74,21 +74,23 @@ func TestFanoutCapRunsAfterTypedReceiverNarrowing(t *testing.T) { } } -func TestStripCommentsAndStringsPreservesExecutableSyntax(t *testing.T) { +func TestMaskCodePreservesExecutableSyntax(t *testing.T) { for name, tc := range map[string]struct { + lang string input string want string }{ - "template interpolation": {"return `value=${g.label()}`;", "g.label()"}, - "python f-string": {"return f'value={g.label()}'", "g.label()"}, - "csharp interpolation": {"return $\"value={g.Label()}\";", "g.Label()"}, - "rust lifetime": {"fn f<'a>(x: &'a str) { target(); }", "target()"}, - "csharp verbatim": {"var p = @\"C:\\dir\\\"; target();", "target()"}, - "javascript regex": {"s.replace(/[\"']/g, ''); target();", "target()"}, + "template interpolation": {"typescript", "return `value=${g.label()}`;", "g.label()"}, + "python f-string": {"python", "return f'value={g.label()}'", "g.label()"}, + "csharp interpolation": {"csharp", "return $\"value={g.Label()}\";", "g.Label()"}, + "rust lifetime": {"rust", "fn f<'a>(x: &'a str) { target(); }", "target()"}, + "csharp verbatim": {"csharp", "var p = @\"C:\\dir\\\"; target();", "target()"}, + "javascript regex": {"javascript", "s.replace(/[\"']/g, ''); target();", "target()"}, + "python single quotes": {"python", "log('starting up')\nclient = Foo()", "client = Foo()"}, } { t.Run(name, func(t *testing.T) { - if got := stripCommentsAndStrings(tc.input); !strings.Contains(got, tc.want) { - t.Fatalf("stripped = %q, want it to retain %q", got, tc.want) + if got := maskCode(tc.lang, tc.input); !strings.Contains(got, tc.want) { + t.Fatalf("maskCode(%q) = %q, want it to keep %q", tc.input, got, tc.want) } }) } @@ -1013,3 +1015,23 @@ func TestMergeEdgesInterfaceDispatchKeepsImplementersOfThatInterface(t *testing. t.Fatal("a same-name method of a non-implementer survived interface dispatch") } } + +// codeSignature masks a signature the extractor collapsed onto one line by +// the lexing of RawText, so code after a line comment survives. +func TestCodeSignatureCollapsedHeader(t *testing.T) { + cases := []struct { + lang, sig, raw, want string + }{ + {"kotlin", "class K : KBase(), // the base KIface", "class K : KBase(), // the base\n KIface {\n}", "class K : KBase()," + strings.Repeat(" ", 13) + "KIface"}, + {"python", "class M( Base, # the (old) Mixin Mixin, ):", "class M(\n Base, # the (old) Mixin\n Mixin,\n):\n pass", "class M( Base," + strings.Repeat(" ", 19) + "Mixin, ):"}, + {"csharp", `[Description("class Y : Fake")] public class A`, `[Description("class Y : Fake")] public class A {}`, `[Description( )] public class A`}, + // No alignment (synthesized signature): masked on its own. + {"go", `func F() // x`, "func G() {}", "func F() "}, + } + for _, tc := range cases { + s := core.SymbolRecord{Language: tc.lang, Signature: tc.sig, RawText: tc.raw} + if got := codeSignature(&s); got != tc.want { + t.Errorf("%s: codeSignature = %q, want %q", tc.lang, got, tc.want) + } + } +} diff --git a/internal/graph/implicitsuper.go b/internal/graph/implicitsuper.go index bd8ac22..922a21e 100644 --- a/internal/graph/implicitsuper.go +++ b/internal/graph/implicitsuper.go @@ -26,10 +26,7 @@ func implicitSuperBases(idx *edgeIndex, ctor *core.SymbolRecord) []string { if cls.Kind != core.KindClass || cls.Name != ctor.ParentSymbol { continue } - sig := cls.Signature - if sig == "" { - sig = firstLine(cls.RawText) - } + sig := declHeader(cls) switch ctor.Language { case "java": return uniqueStrings(inheritanceClauseTypes(stripLeadingGenericParams(sig), "extends", "implements")) diff --git a/internal/graph/interfaces.go b/internal/graph/interfaces.go index 782f5da..70c72fa 100644 --- a/internal/graph/interfaces.go +++ b/internal/graph/interfaces.go @@ -85,7 +85,7 @@ func interfaceMethodNamesSeen(iface *core.SymbolRecord, idx *edgeIndex, visited re = csIfaceMethodRe } if re != nil { - body := stripCommentsAndStrings(iface.RawText) + body := maskCode(iface.Language, iface.RawText) // Drop the declaration line so "type Render interface {" can't // contribute "interface(" style artifacts on unusual formatting. if i := strings.IndexByte(body, '{'); i >= 0 { @@ -164,7 +164,7 @@ func interfaceMemberSignatures(iface *core.SymbolRecord, methodName string) []st default: return nil } - body := stripCommentsAndStrings(iface.RawText) + body := maskCode(iface.Language, iface.RawText) if i := strings.IndexByte(body, '{'); i >= 0 { body = body[i+1:] } @@ -215,6 +215,14 @@ func buildInterfaceSatisfaction(idx *edgeIndex, symbols []core.SymbolRecord) (*i // grafana-bigblast 1.00→0.02 regression). methodsByType := map[typeKey]map[string][]*core.SymbolRecord{} typeSymbols := map[typeKey]*core.SymbolRecord{} + // An anonymous class's name is unique only in its own file: key it by + // the file, or two files' `` pool their methods. + scopeOf := func(s *core.SymbolRecord, typeName string) string { + if fileLocalTypeName(typeName) { + return s.FilePath + } + return dirOf(s.FilePath) + } for i := range symbols { s := &symbols[i] switch s.Kind { @@ -222,14 +230,14 @@ func buildInterfaceSatisfaction(idx *edgeIndex, symbols []core.SymbolRecord) (*i if s.ParentSymbol == "" { continue } - key := typeKey{s.Language, dirOf(s.FilePath), s.ParentSymbol} + key := typeKey{s.Language, scopeOf(s, s.ParentSymbol), s.ParentSymbol} if methodsByType[key] == nil { methodsByType[key] = map[string][]*core.SymbolRecord{} } ln := strings.ToLower(s.Name) methodsByType[key][ln] = append(methodsByType[key][ln], s) case core.KindStruct, core.KindClass, core.KindType: - typeSymbols[typeKey{s.Language, dirOf(s.FilePath), s.Name}] = s + typeSymbols[typeKey{s.Language, scopeOf(s, s.Name), s.Name}] = s } } diff --git a/internal/graph/java_anonymous_test.go b/internal/graph/java_anonymous_test.go new file mode 100644 index 0000000..5d38e17 --- /dev/null +++ b/internal/graph/java_anonymous_test.go @@ -0,0 +1,97 @@ +package graph + +import ( + "reflect" + "testing" + + "github.com/provasign/grove/internal/core" +) + +// Two files of one package each declare an anonymous class at line 4, +// column 12, so both are named `` (guava had 4,308 anonymous +// classes under 2,289 names). +func anonymousTwins(aBase, bBase string, aMethods, bMethods []string) []core.SymbolRecord { + syms := []core.SymbolRecord{ + {ID: "p/Task.java::Task", FilePath: "p/Task.java", Language: "java", Kind: core.KindInterface, + Name: "Task", QualifiedName: "Task", Signature: "interface Task", + RawText: "interface Task {\n void run();\n void start();\n}"}, + {ID: "p/Task.java::Task.run", FilePath: "p/Task.java", Language: "java", Kind: core.KindMethod, + Name: "run", QualifiedName: "Task.run", ParentSymbol: "Task", Signature: "void run();"}, + {ID: "p/Task.java::Task.start", FilePath: "p/Task.java", Language: "java", Kind: core.KindMethod, + Name: "start", QualifiedName: "Task.start", ParentSymbol: "Task", Signature: "void start();"}, + } + add := func(file, owner, base string, methods []string) { + anon := "" + syms = append(syms, core.SymbolRecord{ + ID: file + "::" + owner + "." + anon, FilePath: file, Language: "java", Kind: core.KindClass, + Name: anon, QualifiedName: owner + "." + anon, ParentSymbol: owner, + Signature: "class " + anon + " extends " + base, Span: core.LineRange{Start: 4, End: 6}, + }) + for _, m := range methods { + syms = append(syms, core.SymbolRecord{ + ID: file + "::" + owner + "." + anon + "." + m, FilePath: file, Language: "java", Kind: core.KindMethod, + Name: m, QualifiedName: owner + "." + anon + "." + m, ParentSymbol: anon, + Signature: "public void " + m + "()", Span: core.LineRange{Start: 5, End: 5}, + }) + } + } + add("p/A.java", "A", aBase, aMethods) + add("p/B.java", "B", bBase, bMethods) + return syms +} + +func ids(syms []*core.SymbolRecord) []string { + var out []string + for _, s := range syms { + out = append(out, s.ID) + } + return out +} + +func TestSubclassOverrides_AnonymousClassesStayInTheirFile(t *testing.T) { + syms := anonymousTwins("Task", "Thread", []string{"run"}, []string{"run"}) + want := []string{"p/A.java::A..run"} + for _, order := range [][]core.SymbolRecord{syms, {syms[0], syms[1], syms[2], syms[5], syms[6], syms[3], syms[4]}} { + got := ids(subclassOverrides(newEdgeIndex(order), "java", "Task", "run", "")) + if !reflect.DeepEqual(got, want) { + t.Fatalf("Task.run overrides = %v, want %v", got, want) + } + } +} + +func TestInterfaceSatisfaction_AnonymousMethodSetsStayInTheirFile(t *testing.T) { + // Neither class has both of Task's methods; pooled by name they did. + syms := anonymousTwins("Thread", "Thread", []string{"run"}, []string{"start"}) + sat, _ := buildInterfaceSatisfaction(newEdgeIndex(syms), syms) + for _, m := range []string{"run", "start"} { + if got := sat.implementorsFor(&syms[0], m); len(got) > 0 { + t.Fatalf("Task.%s implementors = %v, want none", m, ids(got)) + } + } +} + +func TestJavaLocalTypes_AnonymousOwnerFieldsFromItsOwnFile(t *testing.T) { + // A's anonymous class extends Task (field helper: Helper), B's extends + // Thread (field helper: Wrong). A method of each sees its own base's. + syms := anonymousTwins("Task", "Thread", nil, nil) + syms = append(syms, + core.SymbolRecord{ID: "p/Task.java::Task.helper", FilePath: "p/Task.java", Language: "java", Kind: core.KindField, + Name: "helper", QualifiedName: "Task.helper", ParentSymbol: "Task", RawText: " Helper helper = null;"}, + core.SymbolRecord{ID: "p/Thread.java::Thread", FilePath: "p/Thread.java", Language: "java", Kind: core.KindClass, + Name: "Thread", QualifiedName: "Thread", Signature: "class Thread"}, + core.SymbolRecord{ID: "p/Thread.java::Thread.helper", FilePath: "p/Thread.java", Language: "java", Kind: core.KindField, + Name: "helper", QualifiedName: "Thread.helper", ParentSymbol: "Thread", RawText: " Wrong helper;"}, + ) + for file, want := range map[string]string{"p/A.java": "Helper", "p/B.java": "Wrong"} { + owner := file[2:3] + caller := core.SymbolRecord{ + ID: file + "::" + owner + "..go", FilePath: file, Language: "java", Kind: core.KindMethod, + Name: "go", QualifiedName: owner + "..go", ParentSymbol: "", + RawText: "public void go() {\n helper.run();\n}", Span: core.LineRange{Start: 5, End: 7}, + } + all := append(append([]core.SymbolRecord{}, syms...), caller) + if got := javaLocalTypes(newEdgeIndex(all), &caller)["helper"]; got != want { + t.Fatalf("%s: helper = %q, want %s (the field of its own anonymous class's base)", file, got, want) + } + } +} diff --git a/internal/graph/java_declsource_test.go b/internal/graph/java_declsource_test.go new file mode 100644 index 0000000..93e14ed --- /dev/null +++ b/internal/graph/java_declsource_test.go @@ -0,0 +1,37 @@ +package graph + +import ( + "reflect" + "testing" + + "github.com/provasign/grove/internal/core" +) + +func TestJavaParamTypes_CommentInsideLeadingAnnotation(t *testing.T) { + // jackson-databind BuilderCreatorSubtype4742Test, with a decoy comment + // in the annotation's argument list. + m := core.SymbolRecord{ + Language: "java", Kind: core.KindMethod, Name: "properties", + RawText: "@JsonSubTypes({ // don't 'quote \"unterminated (\n" + + " @JsonSubTypes.Type(name = \"bird\", value = Bird.class) /* ) */\n})\n" + + "public abstract Builder properties(AnimalProperties properties);", + } + if got := javaParamTypes(&m); !reflect.DeepEqual(got, []string{"AnimalProperties"}) { + t.Fatalf("param types = %v, want [AnimalProperties]", got) + } +} + +func TestJavaLocalTypes_LocalDeclaredWithoutInitializer(t *testing.T) { + caller := core.SymbolRecord{ + ID: "T.java::T.test", FilePath: "T.java", Language: "java", Kind: core.KindMethod, + Name: "test", QualifiedName: "T.test", ParentSymbol: "T", + RawText: "public void test() {\n Map map;\n try {\n map = make();\n } catch (RuntimeException e) {\n return;\n }\n // Wrong other;\n map.size();\n}", + } + types := javaLocalTypes(newEdgeIndex([]core.SymbolRecord{caller}), &caller) + if types["map"] != "Map" { + t.Fatalf("map = %q, want Map", types["map"]) + } + if _, ok := types["other"]; ok { + t.Fatalf("a commented-out declaration typed other = %q", types["other"]) + } +} diff --git a/internal/graph/java_receiver_test.go b/internal/graph/java_receiver_test.go new file mode 100644 index 0000000..de67ecf --- /dev/null +++ b/internal/graph/java_receiver_test.go @@ -0,0 +1,61 @@ +package graph + +import ( + "testing" + + "github.com/provasign/grove/internal/core" +) + +func TestJavaLocalTypes_GenericParametersWithSpacedArguments(t *testing.T) { + // guava TableCollectors.mergeTables: strings.Fields tore + // "Table table" into four tokens and the parameter went + // untyped (it used to be typed by accident, from another method's + // local read as a class field). + m := core.SymbolRecord{ + ID: "T.java::T.merge", FilePath: "T.java", Language: "java", Kind: core.KindMethod, + Name: "merge", QualifiedName: "T.merge", ParentSymbol: "T", + RawText: "private static void merge(\n Table table,\n @ParametricNullness R row,\n Map map) {\n table.get(row, row);\n}", + } + types := javaLocalTypes(newEdgeIndex([]core.SymbolRecord{m}), &m) + if types["table"] != "Table" || types["map"] != "Map" { + t.Fatalf("table = %q, map = %q; want Table, Map", types["table"], types["map"]) + } + if got := javaArgTypes(newEdgeIndex([]core.SymbolRecord{m}), &m)["table"]; got == "" { + t.Fatal("javaArgTypes: table untyped") + } +} + +func TestJavaChainReceiver_ExternalFieldTypeDispatches(t *testing.T) { + // jackson-databind TestUnknownPropertyDeserialization: + // `result.values.size()` where MapWithoutX.values is a java.util.Map. + // The in-repo Map implementation is a dispatch target, as for a + // single-hop `Map` receiver. + holder := core.SymbolRecord{ + ID: "p/T.java::T.MapWithoutX", FilePath: "p/T.java", Language: "java", Kind: core.KindClass, + Name: "MapWithoutX", QualifiedName: "T.MapWithoutX", ParentSymbol: "T", Signature: "static class MapWithoutX", + RawText: "static class MapWithoutX {\n public Map values;\n}", + } + field := core.SymbolRecord{ + ID: "p/T.java::T.MapWithoutX.values", FilePath: "p/T.java", Language: "java", Kind: core.KindField, + Name: "values", QualifiedName: "T.MapWithoutX.values", ParentSymbol: "MapWithoutX", + RawText: " public Map values;", + } + caller := core.SymbolRecord{ + ID: "p/T.java::T.test", FilePath: "p/T.java", Language: "java", Kind: core.KindMethod, + Name: "test", QualifiedName: "T.test", ParentSymbol: "T", Span: core.LineRange{Start: 10, End: 12}, + RawText: "public void test(MapWithoutX result) {\n result.values.size();\n}", + CallSites: []core.CallSite{{Callee: "values.size", Line: 11}}, + } + impl := core.SymbolRecord{ + ID: "p/M1.java::M1", FilePath: "p/M1.java", Language: "java", Kind: core.KindClass, + Name: "M1", QualifiedName: "M1", Signature: "class M1 implements Map", + } + size := core.SymbolRecord{ + ID: "p/M1.java::M1.size", FilePath: "p/M1.java", Language: "java", Kind: core.KindMethod, + Name: "size", QualifiedName: "M1.size", ParentSymbol: "M1", Signature: "public int size()", + } + edges := BuildEdges([]core.SymbolRecord{holder, field, caller, impl, size}) + if !javaHasCall(edges, caller.ID, size.ID) { + t.Fatal("result.values.size() must dispatch to the in-repo Map implementation") + } +} diff --git a/internal/graph/javalocaltypes.go b/internal/graph/javalocaltypes.go index ec26aa3..e7afe2d 100644 --- a/internal/graph/javalocaltypes.go +++ b/internal/graph/javalocaltypes.go @@ -38,20 +38,19 @@ func javaFunctionalInterfaceArity(paramType string) (int, bool) { // class body. Same shallow, harness-bounded approach as the other languages. var ( - // Type x = ... (typed local; also matches enhanced-for "for (Type x :") + // Type x = ... (typed local; also matches enhanced-for "for (Type x :" + // and a declaration assigned later, `Map map;` before a try). // The generic group tolerates one nesting level (JsonDeserializer>, // Map>) — regexes can't balance arbitrarily, one level // covers real declarations. - javaLocalDeclRe = regexp.MustCompile(`\b([A-Z]\w*)(?:<[^<>]*(?:<[^<>]*>[^<>]*)*>)?(?:\[\])?\s+(\w+)\s*[=:)]`) + javaLocalDeclRe = regexp.MustCompile(`\b([A-Z]\w*)(?:<[^<>]*(?:<[^<>]*>[^<>]*)*>)?(?:\[\])?\s+(\w+)\s*[=:);]`) // typed local including primitives and arrays, for overload matching: // anchored to statement starts so "return x =" / cast fragments can't // masquerade as declarations javaTypedLocalRe = regexp.MustCompile(`(?m)(?:^|[;{()]\s*)\s*(?:final\s+)?((?:boolean|byte|char|short|int|long|float|double|[A-Z][\w.]*)(?:<[^<>]*>)?(?:\[\])?)\s+(\w+)\s*[=;]`) - // field declaration line in a class body + // field declaration line in a class body; run only over classBody + // (masked, member bodies blanked) javaFieldRe = regexp.MustCompile(`(?m)^\s+(?:@[\w.]+(?:\([^)]*\))?\s+)*(?:(?:public|private|protected|static|final|transient|volatile)\s+)*([A-Z]\w*)(?:<[^<>]*(?:<[^<>]*>[^<>]*)*>)?(?:\[\])?\s+(\w+)\s*[;=]`) - // field declaration line, primitives included, raw type token kept — - // for overload matching (AT_SIGN is a char; javaFieldRe skips it) - javaFieldArgRe = regexp.MustCompile(`(?m)^\s+(?:@[\w.]+(?:\([^)]*\))?\s+)*(?:(?:public|private|protected|static|final|transient|volatile)\s+)*((?:boolean|byte|char|short|int|long|float|double|[A-Z]\w*)(?:<[^<>]*(?:<[^<>]*>[^<>]*)*>)?(?:\[\])?)\s+(\w+)\s*[;=]`) // Type prefix for an indexed field symbol. The symbol already supplies the // declarator name, so this deliberately accepts a comma after the first // declarator: `JsonSerializer key, value;` gives both indexed field @@ -61,6 +60,10 @@ var ( ) func javaIndexedFieldType(raw string) string { + if strings.ContainsAny(raw, "/\"'") { + // `@Named("a)b")` would end the annotation's arguments early. + raw = maskCode("java", raw) + } m := javaIndexedFieldTypeRe.FindStringSubmatch(" " + strings.TrimSpace(raw)) if m == nil { return "" @@ -68,6 +71,34 @@ func javaIndexedFieldType(raw string) string { return m[1] } +// javaClassFields records one Java class's own field name → bare type into +// out (names already present win). Indexed field symbols come first: they +// keep every declarator of `Foo a, b;`. The masked top-level class body +// (classBody) then fills names with no field symbol; a method-local or a +// commented-out declaration is never at that level. +func javaClassFields(idx *edgeIndex, cls *core.SymbolRecord, className string, out map[string]string) { + if idx != nil { + for _, f := range idx.byFile[cls.FilePath] { + if f.Kind != core.KindField || (f.ParentSymbol != className && f.ParentSymbol != cls.Name && f.ParentSymbol != cls.QualifiedName) { + continue + } + if _, exists := out[f.Name]; exists { + continue + } + if t := javaBareType(javaIndexedFieldType(f.RawText)); t != "" { + out[f.Name] = t + } + } + } + for _, m := range javaFieldRe.FindAllStringSubmatch(classBody(idx, cls), -1) { + if t := javaBareType(m[1]); t != "" { + if _, exists := out[m[2]]; !exists { + out[m[2]] = t + } + } + } +} + // javaArgTypes infers identifier → raw type token (primitives and arrays // preserved: "long[]", "int") for overload matching, from parameters and // typed locals. Distinct from javaLocalTypes, which normalizes to indexable @@ -91,9 +122,9 @@ func javaArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { out[name] = typ } } - if params := tsDeclParams(javaDeclSource(symbol)); params != "" { - for _, g := range splitTopLevel(params, ',') { - fields := strings.Fields(strings.TrimSpace(g)) + if params := tsDeclParams("java", javaDeclSource(symbol)); params != "" { + for _, g := range splitParams("java", params) { + fields := strings.Fields(javaCollapseGenericSpaces(strings.TrimSpace(g))) for len(fields) > 2 || (len(fields) == 2 && (fields[0] == "final" || strings.HasPrefix(fields[0], "@"))) { fields = fields[1:] } @@ -103,7 +134,7 @@ func javaArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { } } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) if i := strings.IndexByte(body, '{'); i >= 0 { body = body[i+1:] } @@ -140,7 +171,7 @@ func javaArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { } } } - next = append(next, tsBaseClasses(idx, className, dirOf(symbol.FilePath))...) + next = append(next, baseClassesInFileFor(idx, "java", className, symbol.FilePath)...) } classes = next } @@ -149,81 +180,73 @@ func javaArgTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { } // javaDeclSource returns the text to parse a Java declaration's head from, -// with leading annotations removed. `@SuppressWarnings("unchecked")` on the -// line before the method made the annotation's parens the "parameter +// with leading annotations and comments removed. `@SuppressWarnings("unchecked")` +// on the line before the method made the annotation's parens the "parameter // list" (and the Signature — the first line — was the annotation alone), // so every such method parsed as ("unchecked") and slipped through // overload narrowing as neutral. Prefers the raw text; falls back to the -// signature when the raw text is absent. +// signature when the raw text is absent. The annotations are skipped on +// masked text, so a comment inside an annotation's arguments (a quote or +// paren in it) cannot end them early. func javaDeclSource(s *core.SymbolRecord) string { src := s.RawText if src == "" { src = s.Signature } - for { - trimmed := strings.TrimLeft(src, " \t\r\n") - // Leading comments (a Javadoc "(may be {@code null})" has parens - // too) go the same way as annotations. - if strings.HasPrefix(trimmed, "/*") { - end := strings.Index(trimmed, "*/") - if end < 0 { - return trimmed - } - src = trimmed[end+2:] - continue + // Masking is a forward pass, so a prefix masks exactly as the whole + // text would; annotations rarely run past the first couple of KB. + const prefix = 2048 + if len(src) > prefix { + if off, ok := javaDeclStart(maskCode("java", src[:prefix])); ok { + return src[off:] } - if strings.HasPrefix(trimmed, "//") { - nl := strings.IndexByte(trimmed, '\n') - if nl < 0 { - return trimmed - } - src = trimmed[nl+1:] - continue + } + off, _ := javaDeclStart(maskCode("java", src)) + return src[off:] +} + +// javaDeclStart returns the offset in masked of the first byte after the +// leading whitespace and annotations; ok is false when an annotation's +// argument list does not close within masked (the offset is then that +// annotation's start). +func javaDeclStart(masked string) (int, bool) { + pos := 0 + for { + for pos < len(masked) && (masked[pos] == ' ' || masked[pos] == '\t' || masked[pos] == '\r' || masked[pos] == '\n') { + pos++ } - if !strings.HasPrefix(trimmed, "@") { - return trimmed + if pos >= len(masked) || masked[pos] != '@' { + return pos, true } - i := 1 - for i < len(trimmed) && (trimmed[i] == '.' || trimmed[i] == '_' || trimmed[i] == '$' || - (trimmed[i] >= 'a' && trimmed[i] <= 'z') || (trimmed[i] >= 'A' && trimmed[i] <= 'Z') || (trimmed[i] >= '0' && trimmed[i] <= '9')) { + i := pos + 1 + for i < len(masked) && (masked[i] == '.' || masked[i] == '_' || masked[i] == '$' || + (masked[i] >= 'a' && masked[i] <= 'z') || (masked[i] >= 'A' && masked[i] <= 'Z') || (masked[i] >= '0' && masked[i] <= '9')) { i++ } j := i - for j < len(trimmed) && (trimmed[j] == ' ' || trimmed[j] == '\t') { + for j < len(masked) && (masked[j] == ' ' || masked[j] == '\t') { j++ } - if j < len(trimmed) && trimmed[j] == '(' { + if j < len(masked) && masked[j] == '(' { depth := 0 - var quote byte k := j - for ; k < len(trimmed); k++ { - c := trimmed[k] - if quote != 0 { - if c == '\\' { - k++ - } else if c == quote { - quote = 0 - } - continue - } - switch c { - case '"', '\'': - quote = c + for ; k < len(masked); k++ { + switch masked[k] { case '(': depth++ case ')': depth-- } - if depth == 0 && c == ')' { + if depth == 0 { break } } - if k >= len(trimmed) { - return trimmed + if k >= len(masked) { + return pos, false } i = k + 1 } - src = trimmed[i:] + pos = i } } @@ -258,12 +281,12 @@ func javaCollapseGenericSpaces(s string) string { func javaParamTypes(s *core.SymbolRecord) []string { src := javaDeclSource(s) - params := tsDeclParams(src) + params := tsDeclParams("java", src) if params == "" { return nil } var out []string - for _, g := range splitTopLevel(params, ',') { + for _, g := range splitParams("java", params) { // strings.Fields splits on ANY whitespace: "Function // allocator" (the conventional space-after-comma generic style) // tore into four fields ("Function", "allocator") @@ -438,7 +461,7 @@ func narrowOverloadsByArgTypes(cands []*core.SymbolRecord, args []string, argTyp // javaVariadic reports whether the callable's last parameter is declared // with "..."; javaParamTypes flattens it to "[]" for comparison. func javaVariadic(s *core.SymbolRecord) bool { - return strings.Contains(tsDeclParams(javaDeclSource(s)), "...") + return strings.Contains(tsDeclParams("java", javaDeclSource(s)), "...") } // javaShapeMismatch reports bindings the wildcard rule must not rescue: @@ -631,7 +654,7 @@ func javaCallResultOwners(idx *edgeIndex, candidates []*core.SymbolRecord, name return nil } off := cs.Line - symbol.Span.Start - lines := strings.Split(symbol.RawText, "\n") + lines := maskedSymbolLines(idx, symbol) if off < 0 || off >= len(lines) { return nil } @@ -656,7 +679,7 @@ func javaCallResultOwners(idx *edgeIndex, candidates []*core.SymbolRecord, name for depth := 0; depth < 4 && len(level) > 0; depth++ { next := map[string]bool{} for owner := range level { - for _, base := range baseClassesFor(idx, "java", owner, dirOf(symbol.FilePath)) { + for _, base := range baseClassesInFileFor(idx, "java", owner, symbol.FilePath) { if !owners[base] { owners[base] = true next[base] = true @@ -870,39 +893,31 @@ func javaLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string continue } seen[className] = true - if cls := javaOwnerType(idx, className, symbol.FilePath); cls != nil && cls.RawText != "" { - for _, m := range javaFieldRe.FindAllStringSubmatch(cls.RawText, -1) { - if t := javaBareType(m[1]); t != "" { - if _, exists := out[m[2]]; !exists { - out[m[2]] = t - } - } - } - // Multi-declarator fields only let javaFieldRe see the first - // name. Indexed field symbols retain every declarator; recover - // their shared type from the declaration prefix. - for _, f := range idx.byFile[cls.FilePath] { - if f.Kind != core.KindField || (f.ParentSymbol != className && f.ParentSymbol != cls.Name && f.ParentSymbol != cls.QualifiedName) { - continue - } - if _, exists := out[f.Name]; exists { - continue - } - if t := javaBareType(javaIndexedFieldType(f.RawText)); t != "" { - out[f.Name] = t - } - } + if cls := javaOwnerType(idx, className, symbol.FilePath); cls != nil { + javaClassFields(idx, cls, className, out) } - next = append(next, tsBaseClasses(idx, className, dirOf(symbol.FilePath))...) + next = append(next, baseClassesInFileFor(idx, "java", className, symbol.FilePath)...) } classes = next } } + // A lambda or local class member sees the parameters and locals of the + // method it is written in (effectively-final captures), above fields. + // The lambda symbol is parented to the class, so the method is found by + // span. These used to arrive by accident, through a field map that + // regexed every method body of the class. + if outer := enclosingCallable(idx, symbol); outer != nil { + for name, typ := range javaLocalTypes(idx, outer) { + out[name] = typ + } + } + // Parameters: "Type name" pairs from the declaration's paren group. - if params := tsDeclParams(javaDeclSource(symbol)); params != "" { - for _, g := range splitTopLevel(params, ',') { - fields := strings.Fields(strings.TrimSpace(g)) + if params := tsDeclParams("java", javaDeclSource(symbol)); params != "" { + for _, g := range splitParams("java", params) { + // "Table table" is one type token, as in javaParamTypes. + fields := strings.Fields(javaCollapseGenericSpaces(strings.TrimSpace(g))) // Drop modifiers and annotations: "final @Nullable CharSequence seq" for len(fields) > 2 || (len(fields) == 2 && (fields[0] == "final" || strings.HasPrefix(fields[0], "@"))) { fields = fields[1:] @@ -918,7 +933,7 @@ func javaLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string // Typed locals in the body (highest precedence). if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) // Skip the declaration header so parameters aren't re-parsed with // the wrong regex. if i := strings.IndexByte(body, '{'); i >= 0 { @@ -933,3 +948,27 @@ func javaLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string delete(out, "this") return out } + +// enclosingCallable returns the narrowest method, constructor or function in +// the symbol's file whose span strictly contains it, or nil. +func enclosingCallable(idx *edgeIndex, symbol *core.SymbolRecord) *core.SymbolRecord { + var best *core.SymbolRecord + for _, cand := range idx.byFile[symbol.FilePath] { + if cand.ID == symbol.ID { + continue + } + switch cand.Kind { + case core.KindMethod, core.KindConstructor, core.KindFunction: + default: + continue + } + if cand.Span.Start > symbol.Span.Start || cand.Span.End < symbol.Span.End || + cand.Span.Start == symbol.Span.Start && cand.Span.End == symbol.Span.End { + continue + } + if best == nil || cand.Span.End-cand.Span.Start < best.Span.End-best.Span.Start { + best = cand + } + } + return best +} diff --git a/internal/graph/kotlinlocaltypes.go b/internal/graph/kotlinlocaltypes.go index 569b7f1..a2fbd8f 100644 --- a/internal/graph/kotlinlocaltypes.go +++ b/internal/graph/kotlinlocaltypes.go @@ -34,12 +34,12 @@ var kotlinPrimitives = map[string]bool{ func kotlinLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { out := map[string]string{} - for name, typ := range kotlinParamTypes(symbol.Signature) { + for name, typ := range kotlinParamTypes(codeSignature(symbol)) { out[name] = typ } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range kotlinValTypedRe.FindAllStringSubmatch(body, -1) { if typ := kotlinBareType(m[2]); typ != "" { out[m[1]] = typ @@ -120,9 +120,9 @@ func kotlinBaseClasses(idx *edgeIndex, className, preferDir string) []string { if decl == nil { return nil } - text := decl.Signature + text := codeSignature(decl) if text == "" { - text = firstLine(decl.RawText) + text = maskCode(decl.Language, firstLine(decl.RawText)) } for _, raw := range kotlinBaseNames(text) { if name, isCtor := kotlinBaseNameAndCtor(raw); isCtor && name != "" { @@ -143,9 +143,9 @@ func kotlinFieldType(idx *edgeIndex, class, name string) string { if cand.Kind != core.KindField || cand.Name != name || cand.ParentSymbol != class || cand.Language != "kotlin" { continue } - sig := cand.Signature + sig := codeSignature(cand) if sig == "" { - sig = firstLine(cand.RawText) + sig = maskCode(cand.Language, firstLine(cand.RawText)) } if m := kotlinValTypedRe.FindStringSubmatch(sig); m != nil { return kotlinShapeType(m[2]) @@ -190,7 +190,7 @@ func kotlinShapeType(t string) string { // constructor-initialized bindings. func kotlinArgShapes(symbol *core.SymbolRecord) map[string]string { out := map[string]string{} - if params, ok := kotlinParamGroups(symbol.Signature); ok { + if params, ok := kotlinParamGroups(codeSignature(symbol)); ok { for _, g := range params { g = strings.TrimPrefix(strings.TrimPrefix(strings.TrimSpace(g), "val "), "var ") if m := kotlinParamRe.FindStringSubmatch(g); m != nil { @@ -201,7 +201,7 @@ func kotlinArgShapes(symbol *core.SymbolRecord) map[string]string { } } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range kotlinValTypedRe.FindAllStringSubmatch(body, -1) { if shape := kotlinShapeType(m[2]); shape != "" { out[m[1]] = shape @@ -255,9 +255,9 @@ func kotlinParamGroups(signature string) ([]string, bool) { // kotlinDeclParamShapes returns a candidate's parameter shapes by position // and whether its last parameter is a vararg. func kotlinDeclParamShapes(s *core.SymbolRecord) (shapes []string, variadic bool, ok bool) { - src := s.Signature + src := codeSignature(s) if !strings.Contains(src, ")") { - src = firstLine(s.RawText) + src = maskCode(s.Language, firstLine(s.RawText)) } groups, ok := kotlinParamGroups(src) if !ok { @@ -430,7 +430,7 @@ func kotlinExtensionReceiver(signature string) string { func kotlinExtensionCandidates(cands []*core.SymbolRecord, typ string) []*core.SymbolRecord { var out []*core.SymbolRecord for _, cand := range cands { - recv := kotlinExtensionReceiver(cand.Signature) + recv := kotlinExtensionReceiver(codeSignature(cand)) if recv == "" { continue } @@ -458,7 +458,7 @@ func kotlinCallResultTypes(idx *edgeIndex, qualifier string) map[string]bool { } switch cand.Kind { case core.KindFunction, core.KindMethod: - if ret := kotlinReturnType(cand.Signature); ret != "" { + if ret := kotlinReturnType(codeSignature(cand)); ret != "" { out[ret] = true } case core.KindConstructor: diff --git a/internal/graph/localtypes.go b/internal/graph/localtypes.go index 0bd9b25..a87b90d 100644 --- a/internal/graph/localtypes.go +++ b/internal/graph/localtypes.go @@ -53,12 +53,10 @@ func goLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { // Receiver struct fields (lowest precedence: locals shadow fields). if symbol.Kind == core.KindMethod && symbol.ParentSymbol != "" { - if t := findTypeSymbol(idx, symbol); t != nil && t.RawText != "" { - body := t.RawText - if i := strings.IndexByte(body, '{'); i >= 0 { - body = body[i+1:] - } - for _, m := range goStructFieldRe.FindAllStringSubmatch(body, -1) { + if t := findTypeSymbol(idx, symbol); t != nil { + // Top-level struct fields only (classBody): a commented-out + // field or a nested anonymous struct's field is not one. + for _, m := range goStructFieldRe.FindAllStringSubmatch(classBody(idx, t), -1) { if typ := m[2]; !goTypeBlocklist[typ] && !goTypeBlocklist[m[1]] { out[m[1]] = typ } @@ -73,7 +71,7 @@ func goLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { // Body declarations (highest precedence). if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, re := range []*regexp.Regexp{goVarDeclRe, goCompositeLitRe} { for _, m := range re.FindAllStringSubmatch(body, -1) { if typ := m[2]; !goTypeBlocklist[typ] { @@ -302,12 +300,174 @@ func goParamList(signature string) (string, bool) { return "", false } +// classBody returns a class/struct declaration's body reduced to its own +// member declarations: comments and strings masked, and everything nested +// inside a member (method bodies, initializer blocks, anonymous classes, +// property accessors) blanked. Field regexes run over this, never over the +// raw class text, where a method-local `Foo helper = ...` or a commented-out +// declaration read as a field. Byte offsets and newlines are preserved. +// +// Brace languages keep only brace depth 1 (the braces opening and closing a +// member body stay, so `Bar P { get; }` and `void a() { }` keep their shape). +// Python keeps only the statements at the class body's own indentation. +// Memoized per class symbol: every method of the class asks for it. +func classBody(idx *edgeIndex, cls *core.SymbolRecord) string { + if cls == nil || cls.RawText == "" { + return "" + } + if idx != nil && cls.ID != "" { + if v, ok := idx.classBodies.Load(cls.ID); ok { + return v.(string) + } + } + var body string + if cls.Language == "python" { + body = pyClassBodyTopLevel(maskCode(cls.Language, cls.RawText)) + } else { + body = braceClassBodyTopLevel(maskCode(cls.Language, cls.RawText)) + } + if idx != nil && cls.ID != "" { + idx.classBodies.Store(cls.ID, body) + } + return body +} + +// maskedSymbolLines returns s.RawText masked (maskCode) and split into lines, +// memoized per symbol ID: the per-call-site line recoveries (receiver +// chains, argument lists, `new` namespaces) read one line of a body that +// may hold hundreds of call sites, and a comment or string on that line, or +// a block comment spanning it, must not read as the call. +func maskedSymbolLines(idx *edgeIndex, s *core.SymbolRecord) []string { + if s == nil || s.RawText == "" { + return nil + } + if idx != nil && s.ID != "" { + if v, ok := idx.maskedLines.Load(s.ID); ok { + return v.([]string) + } + } + lines := strings.Split(maskCode(s.Language, s.RawText), "\n") + if idx != nil && s.ID != "" { + idx.maskedLines.Store(s.ID, lines) + } + return lines +} + +// braceClassBodyTopLevel blanks masked text outside brace depth 1 of the +// first '{' that is not inside parens or brackets (a class annotation like +// `@JsonSubTypes({...})` comes before the body). +func braceClassBodyTopLevel(masked string) string { + b := []byte(masked) + open, paren := -1, 0 + for i, c := range b { + if c == '(' || c == '[' { + paren++ + } else if (c == ')' || c == ']') && paren > 0 { + paren-- + } else if c == '{' && paren == 0 { + open = i + break + } + } + if open < 0 { + return "" + } + depth := 0 + for i := range b { + c := b[i] + if i <= open { + if c != '\n' { + b[i] = ' ' + } + if i == open { + depth = 1 + } + continue + } + switch { + case c == '{': + depth++ + if depth == 2 { + continue + } + case c == '}': + depth-- + if depth == 1 { + continue + } + } + if depth != 1 && c != '\n' { + b[i] = ' ' + } + } + return string(b) +} + +// pyClassBodyTopLevel keeps the masked class text's body statements at the +// body's own indentation (the first indented line after the header), plus +// their continuation lines while a bracket is open; method bodies, nested +// class bodies, and the header are blanked. Docstrings are already spaces. +func pyClassBodyTopLevel(masked string) string { + lines := strings.Split(masked, "\n") + indent := -1 + open := 0 // bracket depth carried by a kept statement + for i, line := range lines { + blank := func() { lines[i] = strings.Repeat(" ", len(line)) } + if i == 0 { + blank() + continue + } + trimmed := strings.TrimLeft(line, " \t") + if trimmed == "" { + continue + } + lead := len(line) - len(trimmed) + if open == 0 { + if indent < 0 { + indent = lead + } + if lead != indent { + blank() + continue + } + } + for j := 0; j < len(trimmed); j++ { + switch trimmed[j] { + case '(', '[', '{': + open++ + case ')', ']', '}': + if open > 0 { + open-- + } + } + } + } + return strings.Join(lines, "\n") +} + +// splitParams splits a parameter or argument list (tsDeclParams) on +// top-level commas, judged on the list masked as lang: a `<`, `,` or paren +// inside a default-value string (`open: string = "<"`) or a comment neither +// opens a generic nor splits a group. The groups are slices of params. +func splitParams(lang, params string) []string { + if !strings.ContainsAny(params, "\"'`/#") { + return splitTopLevel(params, ',') + } + return splitTopLevelOf(params, maskCode(lang, params), ',') +} + // splitTopLevel splits on sep outside any (), [], {} nesting. func splitTopLevel(s string, sep byte) []string { + return splitTopLevelOf(s, s, sep) +} + +// splitTopLevelOf splits s at the top-level seps of scan, a same-length +// view of s (s itself, or s masked). +func splitTopLevelOf(s, scan string, sep byte) []string { var out []string depth, angle, last := 0, 0, 0 - for i := 0; i < len(s); i++ { - switch s[i] { + for i := 0; i < len(scan); i++ { + switch scan[i] { case '(', '[', '{': depth++ case ')', ']', '}': @@ -623,9 +783,9 @@ func typeOrInheritedMethodTargets(idx *edgeIndex, caller *core.SymbolRecord, typ func csharpExtensionTargets(cands []*core.SymbolRecord, receiverType string) []*core.SymbolRecord { var out []*core.SymbolRecord for _, cand := range cands { - params := tsDeclParams(cand.Signature) + params := tsDeclParams(cand.Language, cand.Signature) if params == "" { - params = tsDeclParams(cand.RawText) + params = tsDeclParams(cand.Language, cand.RawText) } first := strings.TrimSpace(strings.SplitN(params, ",", 2)[0]) fields := strings.Fields(first) diff --git a/internal/graph/localtypes_test.go b/internal/graph/localtypes_test.go index 54b6437..923e904 100644 --- a/internal/graph/localtypes_test.go +++ b/internal/graph/localtypes_test.go @@ -1113,3 +1113,17 @@ func TestPythonProxyTypedVariableDoesNotDropForwardedMember(t *testing.T) { t.Fatalf("proxy-typed default_storage.url lost its name-resolved target; edges=%+v", edges) } } + +func TestCFamilyParamsFromBodyIgnoreComments(t *testing.T) { + // doctest's `struct DOCTEST_INTERFACE AssertData` recovered as a + // function with no parameter list in its signature: parameters are + // read from the body, and the first `(` there was in a comment, so a + // comment line decided whether `str` was a parameter (and whether + // `content(str)` "passed" a function named str). + raw := "struct DOCTEST_INTERFACE AssertData\n{\n // ghost() { helper(1); }\n StringContains(const String& str) : content(str) { }\n};" + syms := []core.SymbolRecord{{ID: "d.h::AssertData@1", FilePath: "d.h", BlobSHA: "1", Language: "cpp", Kind: core.KindFunction, + Name: "AssertData", QualifiedName: "AssertData", Signature: "struct DOCTEST_INTERFACE AssertData", RawText: raw}} + if got := cFamilyLocalTypes(newEdgeIndex(syms), &syms[0])["str"]; got != "String" { + t.Fatalf("str = %q, want String", got) + } +} diff --git a/internal/graph/mainframe.go b/internal/graph/mainframe.go index b9de99e..5027dee 100644 --- a/internal/graph/mainframe.go +++ b/internal/graph/mainframe.go @@ -122,11 +122,66 @@ func hasMainframeSymbols(symbols []core.SymbolRecord) bool { var ( reFieldToken = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9-]+`) - reQuoted = regexp.MustCompile(`'[^']*'|"[^"]*"`) reAssignTo = regexp.MustCompile(`(?i)ASSIGN\s+TO\s+([A-Za-z0-9-]+)`) reFieldQualifier = regexp.MustCompile(`(?i)^\s+(?:OF|IN)\s+([A-Za-z0-9-]+)`) ) +// cobolStatementVerbs are the procedure-division words that begin a +// statement. A statement runs from one of them to the next, or to a +// separator period, across line breaks. +var cobolStatementVerbs = map[string]bool{ + "ACCEPT": true, "ADD": true, "ALTER": true, "CALL": true, "CANCEL": true, "CLOSE": true, + "COMPUTE": true, "CONTINUE": true, "DELETE": true, "DISPLAY": true, "DIVIDE": true, + "ELSE": true, "EVALUATE": true, "EXEC": true, "EXIT": true, "GENERATE": true, "GO": true, + "GOBACK": true, "IF": true, "INITIALIZE": true, "INITIATE": true, "INSPECT": true, + "MERGE": true, "MOVE": true, "MULTIPLY": true, "OPEN": true, "PERFORM": true, "READ": true, + "RELEASE": true, "RETURN": true, "REWRITE": true, "SEARCH": true, "SET": true, "SORT": true, + "START": true, "STOP": true, "STRING": true, "SUBTRACT": true, "TERMINATE": true, + "UNSTRING": true, "WHEN": true, "WRITE": true, +} + +// cobolStatements splits a masked paragraph or section body (comments and +// literals already blanked by maskCode) into statements. Line breaks are +// spaces in COBOL, so `MOVE A\n TO B.` is one statement whose TO clause +// makes B a write. A statement ends at the next statement verb or at a +// separator period (a period followed by a space or the end of the text). +// END-EXEC closes an EXEC block, whose embedded SQL words are not COBOL verbs. +func cobolStatements(masked string) []string { + text := strings.NewReplacer("\r", " ", "\n", " ").Replace(masked) + var out []string + start := 0 + flush := func(end int) { + if strings.TrimSpace(text[start:end]) != "" { + out = append(out, text[start:end]) + } + start = end + } + inExec := false + pos := 0 + for _, loc := range reFieldToken.FindAllStringIndex(text, -1) { + // Separator periods between the previous token and this one. + for i := pos; i < loc[0]; i++ { + if text[i] == '.' && !inExec && (i+1 == len(text) || text[i+1] == ' ') { + flush(i + 1) + } + } + pos = loc[1] + word := strings.ToUpper(text[loc[0]:loc[1]]) + if inExec { + if word == "END-EXEC" { + inExec = false + } + continue + } + if cobolStatementVerbs[word] { + flush(loc[0]) + inExec = word == "EXEC" + } + } + flush(len(text)) + return out +} + // mainframeAccessAt classifies one field occurrence by its position in a // COBOL statement. COBOL verbs frequently read and write different operands, // so treating an entire captured tail as a write target loses lineage (for @@ -334,8 +389,9 @@ func buildMainframeDataEdges(idx *edgeIndex, symbols []core.SymbolRecord) []core // // Statement verbs classify direction: MOVE/COMPUTE/ADD/SUBTRACT/SET/ // INITIALIZE/GIVING/INTO/VARYING targets are WRITES; every other - // referenced field in the statement is a READ. Quoted literals are - // stripped first so 'ACCT-NOT-FOUND' never matches a field. + // referenced field in the statement is a READ. Comments (`*>`) and + // literals are masked first so 'ACCT-NOT-FOUND' never matches a field, + // and statements are joined across line breaks (cobolStatements). // // References remain owned by their paragraph/section. Rolling same-file // fields up to the program destroys the caller identity used by impact, @@ -393,8 +449,7 @@ func buildMainframeDataEdges(idx *edgeIndex, symbols []core.SymbolRecord) []core continue } visible := fieldMap(s.FilePath) - for _, stmt := range strings.Split(s.RawText, "\n") { - stmt = reQuoted.ReplaceAllString(stmt, " ") + for _, stmt := range cobolStatements(maskCode("cobol", s.RawText)) { for _, loc := range reFieldToken.FindAllStringIndex(stmt, -1) { tok := stmt[loc[0]:loc[1]] u := strings.ToUpper(tok) diff --git a/internal/graph/mainframe_lineage_test.go b/internal/graph/mainframe_lineage_test.go index dd876c3..3a00dc1 100644 --- a/internal/graph/mainframe_lineage_test.go +++ b/internal/graph/mainframe_lineage_test.go @@ -71,6 +71,71 @@ func TestMainframeRenamePlanUsesHyphenBoundariesAndReferenceEdges(t *testing.T) } } +// A name inside a `*>` comment or a literal on an edited line is not +// renamed: only the code occurrence is replaced. +func TestMainframeRenamePlanLeavesCommentsAndLiterals(t *testing.T) { + symbols := []core.SymbolRecord{ + {ID: "field", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "WS-ID", QualifiedName: "WS-ID", Span: core.LineRange{Start: 2, End: 2}, Signature: "01 WS-ID PIC X.", RawText: "01 WS-ID PIC X."}, + {ID: "caller", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("paragraph"), Name: "RUN", QualifiedName: "JOB.RUN", Span: core.LineRange{Start: 10, End: 11}, + RawText: "MOVE WS-ID TO OUTPUT-ID. *> WAS WS-ID\nDISPLAY 'WS-ID' WS-ID."}, + {ID: "output", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "OUTPUT-ID", QualifiedName: "OUTPUT-ID"}, + } + g := New() + g.Replace(symbols, 1) + plan, err := g.RenamePlan("ws-id", "CUSTOMER-ID") + if err != nil { + t.Fatal(err) + } + want := map[int]string{ + 10: "MOVE CUSTOMER-ID TO OUTPUT-ID. *> WAS WS-ID", + 11: "DISPLAY 'WS-ID' CUSTOMER-ID.", + } + got := map[int]string{} + for _, edit := range append(plan.Edits, plan.Ambiguous...) { + if edit.SiteID == "caller" { + got[edit.Line] = edit.After + } + } + for line, after := range want { + if got[line] != after { + t.Errorf("line %d after = %q, want %q", line, got[line], after) + } + } +} + +// A statement continues across line breaks: `MOVE A` / `TO B.` writes B. +// An IF condition is read; the MOVE in its branch still writes its target. +func TestMainframeStatementsSpanLines(t *testing.T) { + symbols := []core.SymbolRecord{ + {ID: "p", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("paragraph"), Name: "RUN", QualifiedName: "JOB.RUN", Span: core.LineRange{Start: 10, End: 13}, RawText: strings.Join([]string{ + "RUN.", + "MOVE WS-A", + " TO WS-B.", + "IF WS-C > 0 MOVE WS-A TO WS-D.", + }, "\n")}, + {ID: "a", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "WS-A", QualifiedName: "WS-A"}, + {ID: "b", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "WS-B", QualifiedName: "WS-B"}, + {ID: "c", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "WS-C", QualifiedName: "WS-C"}, + {ID: "d", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("data-item"), Name: "WS-D", QualifiedName: "WS-D"}, + } + got := map[string]bool{} + for _, edge := range BuildEdges(symbols) { + if edge.Type == core.EdgeReads || edge.Type == core.EdgeWrites { + got[edge.From+" "+string(edge.Type)+" "+edge.To] = true + } + } + for _, edge := range []string{"p reads a", "p writes b", "p reads c", "p writes d"} { + if !got[edge] { + t.Errorf("missing %s; got %v", edge, got) + } + } + for _, edge := range []string{"p reads b", "p writes a", "p reads d", "p writes c"} { + if got[edge] { + t.Errorf("spurious %s", edge) + } + } +} + func TestMainframeImpactIncludesDatasetBindings(t *testing.T) { logical := core.SymbolRecord{ID: "lf", FilePath: "job.cbl", Language: "cobol", Kind: core.SymbolKind("logical-file"), Name: "INPUT", QualifiedName: "JOB.INPUT"} dataset := core.SymbolRecord{ID: "ds", FilePath: "job.jcl", Language: "jcl", Kind: core.SymbolKind("dataset"), Name: "DATA.SET", QualifiedName: "STEP.DATA.SET"} diff --git a/internal/graph/memberclassify.go b/internal/graph/memberclassify.go index 7695d46..9f7360c 100644 --- a/internal/graph/memberclassify.go +++ b/internal/graph/memberclassify.go @@ -579,7 +579,7 @@ func (c *memberClassifier) cDeclaredType(e *core.SymbolRecord, name string) stri if e == nil || name == "" { return "" } - text := stripCommentsAndStrings(e.Signature + "\n" + e.RawText) + text := maskCode(e.Language, e.Signature+"\n"+e.RawText) re := regexp.MustCompile(`(?:\bstruct\s+)?\b([A-Za-z_]\w*)\s*(?:\*+\s*(?:const\s+)?|&\s*|\s+)(?:\w+\s*(?:\[[^\]]*\])?\s*,\s*\**\s*)*` + regexp.QuoteMeta(name) + `\s*(?:[=;,)\[])`) for _, m := range re.FindAllStringSubmatch(text, -1) { switch m[1] { @@ -599,7 +599,7 @@ var goTupleAssignRe = regexp.MustCompile(`(?m)\b([A-Za-z_]\w*(?:\s*,\s*[A-Za-z_] // callee's result list — the shared local-type pass only types the first // variable (`c, router := CreateTestContext(w)` left router untyped). func (c *memberClassifier) goTupleResultType(e *core.SymbolRecord, name string) string { - for _, m := range goTupleAssignRe.FindAllStringSubmatch(stripCommentsAndStrings(e.RawText), -1) { + for _, m := range goTupleAssignRe.FindAllStringSubmatch(maskCode(e.Language, e.RawText), -1) { vars := strings.Split(m[1], ",") pos := -1 for i, v := range vars { @@ -785,7 +785,7 @@ func (c *memberClassifier) tsMemberType(owner, name string) string { // declared return type, unwrapping Promise) and `for (const m of )` // (the element type of a typed array field or local). func (c *memberClassifier) tsNameType(e *core.SymbolRecord, name string) string { - body := stripCommentsAndStrings(e.RawText) + body := maskCode(e.Language, e.RawText) q := regexp.QuoteMeta(name) callRe := regexp.MustCompile(`(?:const|let|var)\s+` + q + `\s*=\s*(?:await\s+)?[\w.!?]*?([A-Za-z_]\w*)\s*(?:<[^>()]*>)?\(`) if m := callRe.FindStringSubmatch(body); m != nil { @@ -892,7 +892,7 @@ func (c *memberClassifier) cFieldType(typeName, field string) string { if f.ParentSymbol != typeName || f.Kind != core.KindField { continue } - if m := cFieldTypeRe.FindStringSubmatch(f.Signature); m != nil { + if m := cFieldTypeRe.FindStringSubmatch(codeSignature(&f)); m != nil { t := lastSegment(strings.ReplaceAll(m[1], "::", ".")) if c.typeExists(t) { return t @@ -972,7 +972,7 @@ func (c *memberClassifier) returnTypeOf(f *core.SymbolRecord) string { return goReturnType(c.edgeIdx(), f) case "c", "cpp", "objc": if f.Kind == core.KindMacro { - body := f.RawText + body := maskCode(f.Language, f.RawText) for _, re := range []*regexp.Regexp{cContainerOfRe, cCastRe} { if m := re.FindStringSubmatch(body); m != nil && c.typeExists(m[1]) { return m[1] @@ -981,14 +981,14 @@ func (c *memberClassifier) returnTypeOf(f *core.SymbolRecord) string { return "" } re := regexp.MustCompile(`\b(?:struct\s+)?([A-Za-z_]\w*)\s*\*?\s*` + regexp.QuoteMeta(f.Name) + `\s*\(`) - if m := re.FindStringSubmatch(f.Signature); m != nil && c.typeExists(m[1]) { + if m := re.FindStringSubmatch(codeSignature(f)); m != nil && c.typeExists(m[1]) { return m[1] } case "python": if m := pyArrowRe.FindStringSubmatch(f.Signature); m != nil && c.typeExists(m[1]) { return m[1] } - body := stripCommentsAndStrings(f.RawText) + body := maskCode(f.Language, f.RawText) if m := pyReturnCtorRe.FindStringSubmatch(body); m != nil && c.typeExists(m[1]) { return m[1] } @@ -1078,7 +1078,7 @@ func (c *memberClassifier) pyModuleVarType(name string) string { if v.Language != "python" || v.ParentSymbol != "" || !dataMemberKinds[v.Kind] { continue } - m := pyModuleCtorRe.FindStringSubmatch(v.Signature + "\n" + v.RawText) + m := pyModuleCtorRe.FindStringSubmatch(maskCode(v.Language, v.Signature+"\n"+v.RawText)) if m == nil || !c.typeExists(m[1]) { continue } diff --git a/internal/graph/missingimpl.go b/internal/graph/missingimpl.go index a0fcf9b..899bf4c 100644 --- a/internal/graph/missingimpl.go +++ b/internal/graph/missingimpl.go @@ -2,6 +2,7 @@ package graph import ( "fmt" + "regexp" "strings" "github.com/provasign/grove/internal/core" @@ -310,10 +311,7 @@ func classExtendsNames(t *core.SymbolRecord) []string { } switch t.Language { case "typescript", "tsx", "javascript", "java": - text := t.Signature - if text == "" { - text = firstLine(t.RawText) - } + text := declHeader(t) return matchNameList(extendsRe, stripAngleBrackets(text)) case "csharp": // The base class (first non-interface entry) grants implementation @@ -321,10 +319,7 @@ func classExtendsNames(t *core.SymbolRecord) []string { // subtype with an unresolved/external base could never set // escapesIndex and would be wrongly bucketed Missing rather than // Unverifiable. - text := t.Signature - if text == "" { - text = firstLine(t.RawText) - } + text := declHeader(t) names := csharpBaseNames(text) if len(names) > 0 && !strings.HasPrefix(names[0], "I") { return names[:1] @@ -341,7 +336,7 @@ func classExtendsNames(t *core.SymbolRecord) []string { // method, or a concrete (non-abstract) method on a class seed. func contractProvidesBody(contract []core.SymbolRecord, seedKinds map[string]core.SymbolKind) bool { for _, c := range contract { - if hasModifier(&c, "default") || strings.Contains(c.Signature, "default ") { + if hasModifier(&c, "default") || strings.Contains(codeSignature(&c), "default ") { return true } if hasModifier(&c, "abstract") { @@ -359,8 +354,12 @@ func contractProvidesBody(contract []core.SymbolRecord, seedKinds map[string]cor (c.Language == "rust" && k == core.KindTrait) || (c.Language == "csharp" && k == core.KindInterface) if inheritsBody { - if strings.Contains(c.RawText, "{") || c.Language == "python" || - (c.Language == "csharp" && strings.Contains(c.RawText, "=>")) { + if c.Language == "python" { + return true + } + // A body, not a `{` or `=>` inside a comment or string. + if body := maskCode(c.Language, c.RawText); strings.Contains(body, "{") || + (c.Language == "csharp" && strings.Contains(body, "=>")) { return true } } @@ -387,7 +386,7 @@ var knownEmptyBases = map[string]bool{ // contract (abstract method, Python @abstractmethod, or a // raise-NotImplementedError stub, which fails the contract at runtime). func providesImplementation(m *core.SymbolRecord) bool { - if hasModifier(m, "abstract") || strings.Contains(m.Signature, "abstract ") { + if hasModifier(m, "abstract") || strings.Contains(codeSignature(m), "abstract ") { return false } if pythonAbstractMethod(m) { @@ -401,15 +400,58 @@ func pythonAbstractMethod(method *core.SymbolRecord) bool { return false } for _, annotation := range method.Annotations { - if annotation == "abstractmethod" || strings.HasSuffix(annotation, ".abstractmethod") { + // Annotations hold the decorator text: "abstractmethod", + // "abc.abstractmethod", and the deprecated abstractclassmethod / + // abstractstaticmethod / abstractproperty forms. + name, _, _ := strings.Cut(strings.TrimPrefix(annotation, "@"), "(") + if i := strings.LastIndexByte(name, '.'); i >= 0 { + name = name[i+1:] + } + switch strings.TrimSpace(name) { + case "abstractmethod", "abstractclassmethod", "abstractstaticmethod", "abstractproperty": return true } } - return strings.Contains(method.RawText, "abstractmethod") || strings.Contains(method.RawText, "NotImplementedError") + return pythonStubBody(method.RawText) +} + +// pythonNotImplementedRe matches `raise NotImplementedError` (bare, called, +// or module-qualified) at the start of a masked statement. +var pythonNotImplementedRe = regexp.MustCompile(`^raise\s+(?:[A-Za-z_]\w*\.)*NotImplementedError\b`) + +// pythonStubBody reports whether a Python def's first real statement — after +// the header and any docstring — is `raise NotImplementedError`. The body is +// masked, so a docstring or comment that mentions NotImplementedError does +// not make an implementation abstract. +func pythonStubBody(raw string) bool { + if !strings.Contains(raw, "NotImplementedError") { + return false + } + body := maskCode("python", raw) + // The header ends at the first ':' outside brackets (parameters may + // carry annotations and defaults with colons inside brackets). + depth := 0 + end := -1 + for i := 0; i < len(body) && end < 0; i++ { + switch body[i] { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + case ':': + if depth == 0 { + end = i + 1 + } + } + } + if end < 0 { + return false + } + return pythonNotImplementedRe.MatchString(strings.TrimLeft(body[end:], " \t\r\n;")) } func isAbstractType(t *core.SymbolRecord) bool { - return hasModifier(t, "abstract") || strings.Contains(t.Signature, "abstract class ") + return hasModifier(t, "abstract") || strings.Contains(codeSignature(t), "abstract class ") } func hasModifier(s *core.SymbolRecord, mod string) bool { diff --git a/internal/graph/objclocaltypes.go b/internal/graph/objclocaltypes.go index 5a856f5..d8fbc82 100644 --- a/internal/graph/objclocaltypes.go +++ b/internal/graph/objclocaltypes.go @@ -27,14 +27,14 @@ var ( func objcLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { out := map[string]string{} - for _, m := range objcParamRe.FindAllStringSubmatch(symbol.Signature, -1) { + for _, m := range objcParamRe.FindAllStringSubmatch(codeSignature(symbol), -1) { if typ := objcBareType(m[1]); typ != "" { out[m[2]] = typ } } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range objcLocalDeclRe.FindAllStringSubmatch(body, -1) { if typ := objcBareType(m[1]); typ != "" { out[m[2]] = typ @@ -78,7 +78,7 @@ func objcIvarTypes(idx *edgeIndex, class, file string) map[string]string { if field.Kind != core.KindField || field.ParentSymbol != class || field.Language != "objc" { continue } - m := objcMemberDeclRe.FindStringSubmatch(field.Signature) + m := objcMemberDeclRe.FindStringSubmatch(codeSignature(field)) if m == nil { continue } @@ -122,7 +122,7 @@ func objcCallResultClasses(idx *edgeIndex, name string, caller *core.SymbolRecor if cand.Language != "objc" || cand.Kind != core.KindMethod { continue } - m := objcReturnTypeRe.FindStringSubmatch(cand.Signature) + m := objcReturnTypeRe.FindStringSubmatch(codeSignature(cand)) if m == nil { continue } @@ -159,9 +159,9 @@ func objcBaseClasses(idx *edgeIndex, className string) []string { if decl.Language != "objc" || decl.Kind != core.KindClass { continue } - text := decl.Signature + text := codeSignature(decl) if text == "" { - text = firstLine(decl.RawText) + text = maskCode(decl.Language, firstLine(decl.RawText)) } if m := objcSuperclassRe.FindStringSubmatch(text); len(m) == 2 { return []string{m[1]} diff --git a/internal/graph/phplocaltypes.go b/internal/graph/phplocaltypes.go index 01217ef..5579fde 100644 --- a/internal/graph/phplocaltypes.go +++ b/internal/graph/phplocaltypes.go @@ -26,6 +26,11 @@ type phpTraitRules struct { traits []string } +// phpTraitRulesFor reads a class's trait composition from its raw text. +// Only `use` statements at class-body depth 1 of the masked text count: a +// comment ("We no longer use Loud;") or a closure's `use` inside a method +// is not trait composition. The adaptation block `{ ... }` is read from the +// masked text at the statement's offset. func phpTraitRulesFor(rawText string) phpTraitRules { rules := phpTraitRules{ used: map[string]bool{}, @@ -33,7 +38,18 @@ func phpTraitRulesFor(rawText string) phpTraitRules { excluded: map[string]map[string]bool{}, aliases: map[string]phpTraitAlias{}, } - for _, match := range phpTraitUseRe.FindAllStringSubmatch(rawText, -1) { + masked := maskCode("php", rawText) + top := braceClassBodyTopLevel(masked) + for _, loc := range phpTraitUseRe.FindAllStringSubmatchIndex(masked, -1) { + if loc[0] >= len(top) || !strings.HasPrefix(top[loc[0]:], "use") { + continue + } + match := make([]string, len(loc)/2) + for i := range match { + if loc[2*i] >= 0 { + match[i] = masked[loc[2*i]:loc[2*i+1]] + } + } if len(match) < 2 || strings.ContainsAny(match[1], "($") { continue // closure `use ($x) { ... }`, not trait composition } @@ -79,6 +95,18 @@ func phpTraitRulesFor(rawText string) phpTraitRules { return rules } +// phpClassTraitRules is phpTraitRulesFor memoized per class symbol (asked +// once per self call). +func phpClassTraitRules(idx *edgeIndex, class *core.SymbolRecord) phpTraitRules { + key := "php-traits\x00" + class.ID + if v, ok := idx.classBodies.Load(key); ok { + return v.(phpTraitRules) + } + rules := phpTraitRulesFor(class.RawText) + idx.classBodies.Store(key, rules) + return rules +} + func phpSimpleTypeName(raw string) string { raw = strings.Trim(strings.TrimSpace(raw), "\\") if i := strings.LastIndexByte(raw, '\\'); i >= 0 { @@ -101,7 +129,7 @@ func phpTraitCallTargets(idx *edgeIndex, caller *core.SymbolRecord, name string, if class == nil { return initial, false } - rules := phpTraitRulesFor(class.RawText) + rules := phpClassTraitRules(idx, class) lookupName := name wantedTrait := "" aliased := false @@ -175,7 +203,9 @@ func phpLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string default: continue } - for _, m := range phpPropertyRe.FindAllStringSubmatch(cls.RawText, -1) { + // Top-level members only (classBody): a commented-out + // property is not one. + for _, m := range phpPropertyRe.FindAllStringSubmatch(classBody(idx, cls), -1) { if t := phpBareType(m[1]); t != "" { if _, exists := out[m[2]]; !exists { out[m[2]] = t @@ -200,7 +230,7 @@ func phpLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string // ($stmt = BuilderHelpers::normalizeNode($stmt) → Node). A `new` or // declared type already recorded wins. if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range phpNewLocalRe.FindAllStringSubmatch(body, -1) { if t := phpBareType(m[2]); t != "" { out[m[1]] = t @@ -278,7 +308,7 @@ func phpReturnType(s *core.SymbolRecord) string { // the body, where `return $this` pins the concrete class. } } - body := stripCommentsAndStrings(s.RawText) + body := maskCode(s.Language, s.RawText) if m := phpReturnNewRe.FindStringSubmatch(body); m != nil { if t := phpBareType(m[1]); t != "" { return t @@ -394,7 +424,7 @@ func phpNarrowNewByNamespace(idx *edgeIndex, symbol *core.SymbolRecord, cs core. return ctors } off := cs.Line - symbol.Span.Start - lines := strings.Split(symbol.RawText, "\n") + lines := maskedSymbolLines(idx, symbol) if off < 0 || off >= len(lines) { return ctors } diff --git a/internal/graph/pylocaltypes.go b/internal/graph/pylocaltypes.go index 94e6539..656c1f5 100644 --- a/internal/graph/pylocaltypes.go +++ b/internal/graph/pylocaltypes.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) @@ -324,7 +325,7 @@ func pySetattrTargets(idx *edgeIndex, symbol *core.SymbolRecord, localTypes map[ if symbol.RawText == "" { return nil } - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) // Shadowing guard for the module-global fallback: a parameter or a plain // local rebinding with the same name as a module global refers to the // LOCAL value, not the global — resolving it through the global's type @@ -579,8 +580,11 @@ func pyLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { // show_server_banner()` is the imported cli module, not the // AppGroup in self.cli) — drop the attribute reading for it. if symbol.RawText != "" { + // Masked: a comment or docstring mentioning `self.cli` is not + // a use of the attribute. + code := maskCode(symbol.Language, symbol.RawText) for name := range out { - if !strings.Contains(symbol.RawText, "self."+name) && !strings.Contains(symbol.RawText, "cls."+name) { + if !strings.Contains(code, "self."+name) && !strings.Contains(code, "cls."+name) { delete(out, name) } } @@ -619,7 +623,7 @@ func pyLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { // Body declarations (highest precedence). if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range pyUnnestedMatches(pyAnnAssignRe, body) { if t := pyAnnotationType(idx, symbol, m[2]); t != "" { out[m[1]] = t @@ -693,7 +697,7 @@ func pyLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { if idx != nil && (len(idx.pyModuleGlobals) > 0 || len(idx.pyFileGlobals) > 0) { shadowed = pyParamNames(symbol.RawText) if symbol.RawText != "" { - for _, m := range pyLocalRebindRe.FindAllStringSubmatch(stripCommentsAndStrings(symbol.RawText), -1) { + for _, m := range pyLocalRebindRe.FindAllStringSubmatch(maskCode(symbol.Language, symbol.RawText), -1) { shadowed[m[1]] = true } } @@ -764,12 +768,13 @@ func pyClassAttrTypes(idx *edgeIndex, symbol *core.SymbolRecord, className strin } } if class != nil { - for _, m := range pyClassAnnRe.FindAllStringSubmatch(class.RawText, -1) { - record(m[1], pyAnnotationType(idx, class, m[2])) + attrs := pyClassBodyAttrs(idx, class) + for _, a := range attrs.anns { + record(a[0], pyAnnotationType(idx, class, a[1])) } - for _, m := range pyClassRefRe.FindAllStringSubmatch(class.RawText, -1) { - if typeSymbolExists(idx, m[2]) { - record(m[1], "class:"+m[2]) + for _, a := range attrs.refs { + if typeSymbolExists(idx, a[1]) { + record(a[0], "class:"+a[1]) } } } @@ -789,7 +794,7 @@ func pyClassAttrTypes(idx *edgeIndex, symbol *core.SymbolRecord, className strin } } if init != nil { - body := stripCommentsAndStrings(init.RawText) + body := maskCode(init.Language, init.RawText) for _, m := range pySelfAnnRe.FindAllStringSubmatch(body, -1) { record(m[1], pyAnnotationType(idx, init, m[2])) } @@ -801,6 +806,48 @@ func pyClassAttrTypes(idx *edgeIndex, symbol *core.SymbolRecord, className strin } } +// pyClassAttrs holds a class body's own attribute statements: annotations +// (`name: T`, `name: T = v`) and class references (`name = SomeClass`), as +// (name, text) pairs in source order. +type pyClassAttrs struct{ anns, refs [][2]string } + +// pyClassBodyAttrs reads the class-body attribute statements of cls, memoized +// per class. Only statements at the class body's own indentation count +// (classBody): a method's local `tmp: Bar` or a docstring's numpy +// "client : Bar" line is not an attribute. The annotation text is read with +// only comments masked, since a string annotation (`parent: "Node"`) is type +// syntax; a match counts only where the fully masked body has code, so a +// docstring is never read. +func pyClassBodyAttrs(idx *edgeIndex, cls *core.SymbolRecord) pyClassAttrs { + key := "py-attrs\x00" + cls.ID + if idx != nil && cls.ID != "" { + if v, ok := idx.classBodies.Load(key); ok { + return v.(pyClassAttrs) + } + } + var attrs pyClassAttrs + if top := classBody(idx, cls); top != "" { + text := textmask.MaskComments(cls.Language, cls.RawText) + inCode := func(loc []int) bool { + return loc[2] < len(top) && top[loc[2]] != ' ' && top[loc[2]] != '\t' + } + for _, loc := range pyClassAnnRe.FindAllStringSubmatchIndex(text, -1) { + if inCode(loc) { + attrs.anns = append(attrs.anns, [2]string{text[loc[2]:loc[3]], text[loc[4]:loc[5]]}) + } + } + for _, loc := range pyClassRefRe.FindAllStringSubmatchIndex(text, -1) { + if inCode(loc) { + attrs.refs = append(attrs.refs, [2]string{text[loc[2]:loc[3]], text[loc[4]:loc[5]]}) + } + } + } + if idx != nil && cls.ID != "" { + idx.classBodies.Store(key, attrs) + } + return attrs +} + // pyBaseClasses parses the base-class names from a class declaration // signature: "class Blueprint(Scaffold):" → [Scaffold]. Keyword arguments // (metaclass=...) and subscripted bases (Generic[T]) are skipped. @@ -838,14 +885,12 @@ func pyBaseRefs(idx *edgeIndex, className, preferDir string) []pyBaseRef { } } if chosen != nil { - sig := chosen.Signature - open := strings.IndexByte(sig, '(') - closeIdx := strings.LastIndexByte(sig, ')') - if open < 0 || closeIdx <= open { + list, ok := pythonClassBaseList(chosen) + if !ok || strings.TrimSpace(list) == "" { return nil } var bases []pyBaseRef - for _, b := range splitTopLevel(sig[open+1:closeIdx], ',') { + for _, b := range splitTopLevel(list, ',') { b = strings.TrimSpace(b) if b == "" || strings.Contains(b, "=") { continue @@ -964,7 +1009,7 @@ func inheritedTargets(idx *edgeIndex, symbol *core.SymbolRecord, calleeName stri if len(all) == 0 { return nil } - bases := baseClassesFor(idx, symbol.Language, symbol.ParentSymbol, dirOf(symbol.FilePath)) + bases := baseClassesInFileFor(idx, symbol.Language, symbol.ParentSymbol, symbol.FilePath) // TS/JS: a base class name is resolved through the subclass file's // imports. A monorepo declares `class Transport` in both engine.io and // engine.io-client; `this.onError()` in the client's Fetch transport @@ -1025,7 +1070,7 @@ func narrowBySuper(idx *edgeIndex, symbol *core.SymbolRecord, cands []*core.Symb } return nil } - bases := baseClassesFor(idx, symbol.Language, symbol.ParentSymbol, dirOf(symbol.FilePath)) + bases := baseClassesInFileFor(idx, symbol.Language, symbol.ParentSymbol, symbol.FilePath) for level := 0; level < 3 && len(bases) > 0; level++ { var matched []*core.SymbolRecord for _, base := range bases { @@ -1058,14 +1103,13 @@ func pyImportedDirectBaseTargets(idx *edgeIndex, symbol *core.SymbolRecord, cand if class == nil { return nil } - open := strings.IndexByte(class.Signature, '(') - closeIdx := strings.LastIndexByte(class.Signature, ')') - if open < 0 || closeIdx <= open { + list, ok := pythonClassBaseList(class) + if !ok || strings.TrimSpace(list) == "" { return nil } seen := map[string]bool{} var out []*core.SymbolRecord - for _, rawBase := range splitTopLevel(class.Signature[open+1:closeIdx], ',') { + for _, rawBase := range splitTopLevel(list, ',') { base := strings.TrimSpace(rawBase) if base == "" || strings.Contains(base, "=") { continue @@ -1228,7 +1272,7 @@ func pyWithTargets(idx *edgeIndex, symbol *core.SymbolRecord, localTypes map[str if symbol.RawText == "" { return nil } - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) preferDir := dirOf(symbol.FilePath) seen := map[string]bool{} var out []*core.SymbolRecord @@ -1287,7 +1331,7 @@ func pySubscriptTargets(idx *edgeIndex, symbol *core.SymbolRecord, localTypes ma } seen := map[string]bool{} var out []*core.SymbolRecord - for _, match := range pySubscriptRe.FindAllStringSubmatch(stripCommentsAndStrings(symbol.RawText), -1) { + for _, match := range pySubscriptRe.FindAllStringSubmatch(maskCode(symbol.Language, symbol.RawText), -1) { if match[2] != "" { continue } @@ -1384,8 +1428,14 @@ func subclassOverrides(idx *edgeIndex, language, className, calleeName, preferDi if c.Language == "cpp" && strings.Contains(c.QualifiedName, "::") { className = c.QualifiedName } - for _, base := range baseClassesFor(idx, c.Language, className, dirOf(c.FilePath)) { - byLang[base] = append(byLang[base], className) + sub := className + if fileLocalTypeName(className) { + // Kept apart from same-named anonymous classes of + // other files; see subclassMembers. + sub = className + "\x00" + c.FilePath + } + for _, base := range baseClassesInFileFor(idx, c.Language, className, c.FilePath) { + byLang[base] = append(byLang[base], sub) } } } @@ -1420,7 +1470,7 @@ func subclassOverrides(idx *edgeIndex, language, className, calleeName, preferDi } visited[sub] = true next = append(next, sub) - out = append(out, filterByParent(methods, sub)...) + out = append(out, subclassMembers(methods, sub)...) } } frontier = next @@ -1428,6 +1478,23 @@ func subclassOverrides(idx *edgeIndex, language, className, calleeName, preferDi return out } +// subclassMembers filters methods to those of the subclass key sub: a class +// name, or for a file-local (anonymous) class its name and file joined by +// NUL. +func subclassMembers(methods []*core.SymbolRecord, sub string) []*core.SymbolRecord { + name, file, local := strings.Cut(sub, "\x00") + if !local { + return filterByParent(methods, sub) + } + var out []*core.SymbolRecord + for _, m := range filterByParent(methods, name) { + if m.FilePath == file { + out = append(out, m) + } + } + return out +} + // pyFixtureValueRe finds the value a pytest fixture hands to a test: the // class constructed by its first `return X(` or `yield X(`. var pyFixtureValueRe = regexp.MustCompile(`(?m)^\s*(?:return|yield)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(`) @@ -1494,7 +1561,7 @@ func pyFixtureType(idx *edgeIndex, symbol *core.SymbolRecord, param string) stri } } } - body := stripCommentsAndStrings(best.RawText) + body := maskCode(best.Language, best.RawText) if m := pyFixtureValueRe.FindStringSubmatch(body); m != nil && typeSymbolExists(idx, m[1]) { return m[1] } @@ -1592,11 +1659,12 @@ func pyArityCompatible(cands []*core.SymbolRecord, argc int) []*core.SymbolRecor // pyCallHasSplat reports whether the call at cs unpacks arguments (*a, **kw) // -- then its real argument count is unknown -- or cannot be located (also // unknown). Only a located call with plain arguments has a known count. -func pyCallHasSplat(symbol *core.SymbolRecord, cs core.CallSite) bool { +func pyCallHasSplat(idx *edgeIndex, symbol *core.SymbolRecord, cs core.CallSite) bool { if symbol.RawText == "" || cs.Line < symbol.Span.Start { return true } - lines := strings.Split(symbol.RawText, "\n") + // Masked: a `leaf(` or `*` inside a string or comment is not the call. + lines := maskedSymbolLines(idx, symbol) off := cs.Line - symbol.Span.Start if off < 0 || off >= len(lines) { return true @@ -1694,16 +1762,16 @@ func pyAttrAnnotation(idx *edgeIndex, className, attr, preferDir string) string if cls == nil { return "" } - for _, m := range pyClassAnnRe.FindAllStringSubmatch(cls.RawText, -1) { - if m[1] == attr { - return m[2] + for _, a := range pyClassBodyAttrs(idx, cls).anns { + if a[0] == attr { + return a[1] } } for _, cand := range idx.byFile[cls.FilePath] { if cand.ParentSymbol != className || cand.Name != "__init__" { continue } - for _, m := range pySelfAnnRe.FindAllStringSubmatch(stripCommentsAndStrings(cand.RawText), -1) { + for _, m := range pySelfAnnRe.FindAllStringSubmatch(maskCode(cand.Language, cand.RawText), -1) { if m[1] == attr { return m[2] } diff --git a/internal/graph/renameplan.go b/internal/graph/renameplan.go index 57b021a..308c6bb 100644 --- a/internal/graph/renameplan.go +++ b/internal/graph/renameplan.go @@ -136,7 +136,7 @@ func (g *CodeGraph) RenamePlan(query, newName string) (*RenamePlanResult, error) } before := lines[idx] rawN := len(pattern.FindAllStringIndex(before, -1)) - strippedN := len(pattern.FindAllStringIndex(stripCommentsAndStrings(before), -1)) + strippedN := len(pattern.FindAllStringIndex(maskedLine(s.Language, s.RawText, idx), -1)) if strippedN == 0 { return // only comment/string-literal occurrences on this line } @@ -242,9 +242,9 @@ func (g *CodeGraph) RenamePlan(query, newName string) (*RenamePlanResult, error) if hash := strings.LastIndexByte(s.ID, '#'); hash > 0 { if t, ok := g.symbols[s.ID[:hash]]; ok && t.RawText != "" { before := len(res.Edits) + len(res.Ambiguous) - tLines := strings.Split(t.RawText, "\n") + tLines := strings.Split(maskCode(t.Language, t.RawText), "\n") for i := range tLines { - if pat.MatchString(stripCommentsAndStrings(tLines[i])) { + if pat.MatchString(tLines[i]) { editLine(t, t.Span.Start+i, true) } } @@ -256,9 +256,9 @@ func (g *CodeGraph) RenamePlan(query, newName string) (*RenamePlanResult, error) continue } declLine := -1 - lines := strings.Split(s.RawText, "\n") + lines := strings.Split(maskCode(s.Language, s.RawText), "\n") for i := range lines { - if pat.MatchString(stripCommentsAndStrings(lines[i])) { + if pat.MatchString(lines[i]) { declLine = s.Span.Start + i editLine(s, declLine, true) break @@ -362,7 +362,6 @@ func (g *CodeGraph) RenamePlan(query, newName string) (*RenamePlanResult, error) func renameMainframePlanLocked(ci *ChangeImpactResult, query, newName string) *RenamePlanResult { oldName := ci.Declarations[0].Name pattern := regexp.MustCompile(`(?i)(^|[^A-Za-z0-9-])(` + regexp.QuoteMeta(oldName) + `)([^A-Za-z0-9-]|$)`) - replacement := "${1}" + strings.ReplaceAll(newName, "$", "$$") + "${3}" result := &RenamePlanResult{ Query: query, NewName: newName, SitesTotal: len(ci.Sites()), ExternalSupers: ci.ExternalSupers, OverridesExternal: ci.OverridesExternal, @@ -377,6 +376,7 @@ func renameMainframePlanLocked(ci *ChangeImpactResult, query, newName string) *R if site.RawText == "" && site.Signature != "" { lines = []string{site.Signature} } + masked := strings.Split(maskCode(site.Language, strings.Join(lines, "\n")), "\n") edited := false callLines := map[int]bool{} for _, call := range site.CallSites { @@ -389,12 +389,23 @@ func renameMainframePlanLocked(ci *ChangeImpactResult, query, newName string) *R if !declarations[site.ID] && len(callLines) > 0 && !callLines[line] { continue } - clean := stripCommentsAndStrings(before) - matches := pattern.FindAllStringIndex(clean, -1) + clean := masked[idx] + matches := pattern.FindAllStringSubmatchIndex(clean, -1) if len(matches) == 0 { continue } - after := pattern.ReplaceAllString(before, replacement) + // Replace only the code occurrences: the masked line keeps + // offsets, so a name inside a `*>` comment or a literal on the + // same line stays as written. + var b strings.Builder + last := 0 + for _, m := range matches { + b.WriteString(before[last:m[4]]) + b.WriteString(newName) + last = m[5] + } + b.WriteString(before[last:]) + after := b.String() if after == before { continue } diff --git a/internal/graph/renamesite.go b/internal/graph/renamesite.go index 3e67504..6e052a4 100644 --- a/internal/graph/renamesite.go +++ b/internal/graph/renamesite.go @@ -186,7 +186,7 @@ func chainFromLine(caller *core.SymbolRecord, line int, methodName string) strin return "" } re := chainRe(methodName) - ms := re.FindAllStringSubmatch(stripCommentsAndStrings(lines[idx]), -1) + ms := re.FindAllStringSubmatch(maskedLine(caller.Language, caller.RawText, idx), -1) if len(ms) != 1 { return "" } diff --git a/internal/graph/rustlocaltypes.go b/internal/graph/rustlocaltypes.go index e56db10..81f12a8 100644 --- a/internal/graph/rustlocaltypes.go +++ b/internal/graph/rustlocaltypes.go @@ -53,7 +53,7 @@ func rustConstElementType(idx *edgeIndex, name string) string { if (cand.Kind != core.KindVariable && cand.Kind != core.KindConst) || cand.Language != "rust" { continue } - sig := cand.Signature + sig := codeSignature(cand) colon := strings.IndexByte(sig, ':') if colon < 0 { continue @@ -110,7 +110,7 @@ func rustEnumVariantType(idx *edgeIndex, enumName, variant, preferFile string) s if decl == nil { return "" } - body := stripCommentsAndStrings(decl.RawText) + body := maskCode(decl.Language, decl.RawText) re := regexp.MustCompile(`\b` + regexp.QuoteMeta(variant) + `\s*\(([^()]*)\)`) m := re.FindStringSubmatch(body) if m == nil || strings.ContainsRune(m[1], ',') { @@ -140,14 +140,14 @@ func rustLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string } } - params := rustParamTypes(symbol.Signature) + params := rustParamTypes(codeSignature(symbol)) for name, typ := range params { out[name] = typ } lets := map[string]string{} if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) // Function-result lets carry the lowest confidence of the let // family — explicit annotations and constructor conventions // overwrite them below. @@ -195,7 +195,7 @@ func rustLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string } } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) // `for flag in FLAGS.iter()` over a const/static slice: the element // type is the declared item type's (`&[&dyn Flag]` → Flag), and a // trait element dispatches through the trait's declarations. @@ -297,7 +297,7 @@ func rustCallResultTypes(idx *edgeIndex, qualifier string, symbol *core.SymbolRe if _, ok := scope[cand.FilePath]; !ok { continue } - r := rustReturnType(cand.Signature) + r := rustReturnType(codeSignature(cand)) if r == "Self" { r = cand.ParentSymbol } @@ -456,7 +456,7 @@ func rustFieldTypes(idx *edgeIndex, typeName, preferFile string) map[string]stri // Field signature: "pub mode: Mode," / "haystack: PathBuf," — or, // for a tuple struct's positional field (`struct Override(Gitignore)`, // reached as `self.0`), the bare type alone. - sig := cand.Signature + sig := codeSignature(cand) if i := strings.IndexByte(sig, ':'); i >= 0 { sig = sig[i+1:] } diff --git a/internal/graph/scope_sharing_test.go b/internal/graph/scope_sharing_test.go index b10cf6c..5fba7da 100644 --- a/internal/graph/scope_sharing_test.go +++ b/internal/graph/scope_sharing_test.go @@ -57,3 +57,37 @@ func TestImportedFiles_RustCrateScopeIsSharedPerCrate(t *testing.T) { t.Fatalf("files of one crate must share one scope map") } } + +// A library's own `mod tests` paths (`tests::TempDir`) must not pull an +// integration-test directory (the workspace's tests/, another crate's +// crates/x/tests/) into its scope. Those targets are crates nothing can +// name; before, every one registered as crate "tests" and the map +// iteration picked which one a library reached (ripgrep's ignore crate +// got tests/ on one run and crates/ignore/tests on the next). +func TestImportedFiles_RustTestTargetsAreNotNamedCrates(t *testing.T) { + mk := func(f, name, raw string) core.SymbolRecord { + return core.SymbolRecord{ + ID: f + "::" + name + "@sha", FilePath: f, BlobSHA: "sha", Language: "rust", + Kind: core.KindFunction, Name: name, QualifiedName: name, RawText: raw, + } + } + for run := 0; run < 20; run++ { + idx := newEdgeIndex([]core.SymbolRecord{ + mk("crates/ignore/src/lib.rs", "run", "fn run() {\n let d = tests::TempDir::new();\n}"), + mk("crates/ignore/src/walk.rs", "walk", "fn walk() {}"), + mk("crates/ignore/tests/matched.rs", "matched", "fn matched() {}"), + mk("crates/matcher/src/lib.rs", "find", "fn find() {}"), + mk("crates/matcher/tests/util.rs", "util", "fn util() {}"), + mk("tests/json.rs", "json", "fn json() {}"), + }) + scope := idx.importedFiles("crates/ignore/src/lib.rs") + for _, f := range []string{"tests/json.rs", "crates/ignore/tests/matched.rs", "crates/matcher/tests/util.rs"} { + if _, ok := scope[f]; ok { + t.Fatalf("run %d: ignore's scope reached test target %s: %v", run, f, sortedKeys(scope)) + } + } + if _, ok := scope["crates/ignore/src/walk.rs"]; !ok { + t.Fatalf("run %d: crate scope lost its own module: %v", run, sortedKeys(scope)) + } + } +} diff --git a/internal/graph/swiftlocaltypes.go b/internal/graph/swiftlocaltypes.go index d102be0..f7e3c7e 100644 --- a/internal/graph/swiftlocaltypes.go +++ b/internal/graph/swiftlocaltypes.go @@ -22,9 +22,9 @@ func swiftBaseClasses(idx *edgeIndex, className, preferDir string) []string { if decl == nil { return nil } - text := decl.Signature + text := codeSignature(decl) if text == "" { - text = firstLine(decl.RawText) + text = maskCode(decl.Language, firstLine(decl.RawText)) } names := swiftBaseNames(text) if len(names) == 0 || !namedSymbolIsClass(idx, names[0]) { @@ -65,12 +65,12 @@ func swiftLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]strin // Array receiver whose subscript/methods are the standard library's, // never JSONSubscriptType's — unwrapping it misattributed every // `path[0]` to the in-repo type's own subscripts. - for name, typ := range swiftParamShapes(symbol.Signature) { + for name, typ := range swiftParamShapes(codeSignature(symbol)) { out[name] = typ } if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range swiftLetTypedRe.FindAllStringSubmatch(body, -1) { if typ := swiftShapeType(m[2]); typ != "" { out[m[1]] = typ @@ -99,11 +99,11 @@ func swiftLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]strin if _, shadowed := out[cand.Name]; shadowed { continue } - if m := swiftLetTypedRe.FindStringSubmatch(cand.Signature); m != nil { + if m := swiftLetTypedRe.FindStringSubmatch(codeSignature(cand)); m != nil { if typ := swiftShapeType(m[2]); typ != "" { out[cand.Name] = typ } - } else if m := swiftLetCtorRe.FindStringSubmatch(cand.Signature); m != nil { + } else if m := swiftLetCtorRe.FindStringSubmatch(codeSignature(cand)); m != nil { out[cand.Name] = m[2] } } @@ -160,9 +160,9 @@ func swiftTypealiasTarget(idx *edgeIndex, name string) string { if cand.Language != "swift" || cand.Kind != core.KindType { continue } - m := swiftTypealiasRe.FindStringSubmatch(cand.Signature) + m := swiftTypealiasRe.FindStringSubmatch(codeSignature(cand)) if m == nil { - m = swiftTypealiasRe.FindStringSubmatch(firstLine(cand.RawText)) + m = swiftTypealiasRe.FindStringSubmatch(maskCode(cand.Language, firstLine(cand.RawText))) } if m != nil && m[1] == name { target := m[2] @@ -215,9 +215,9 @@ func swiftParamShapes(signature string) map[string]string { // swiftDeclParamShapes returns a candidate's parameter shapes by position. func swiftDeclParamShapes(s *core.SymbolRecord) []string { - src := s.Signature + src := codeSignature(s) if !strings.Contains(src, ")") { - src = firstLine(s.RawText) + src = maskCode(s.Language, firstLine(s.RawText)) } list, ok := swiftParamList(src) if !ok { @@ -358,9 +358,9 @@ type swiftParam struct { // narrowing cannot split) collapses to. ok is false when the signature has // no parseable parameter list, which callers treat as neutral. func swiftDeclParams(s *core.SymbolRecord) (params []swiftParam, ok bool) { - src := s.Signature + src := codeSignature(s) if !strings.Contains(src, ")") { - src = firstLine(s.RawText) + src = maskCode(s.Language, firstLine(s.RawText)) } list, found := swiftParamList(src) if !found { @@ -460,7 +460,7 @@ func swiftNarrowByLabels(cands []*core.SymbolRecord, args []string) (out []*core // re-source the same-name members it just ruled out. func swiftNarrowCall(cands []*core.SymbolRecord, cs core.CallSite, caller *core.SymbolRecord) (out []*core.SymbolRecord, decided bool) { if out, decided = swiftNarrowByLabels(cands, cs.Args); decided { - return swiftNarrowByArgShapes(out, cs.Args, swiftParamShapes(caller.Signature), caller), true + return swiftNarrowByArgShapes(out, cs.Args, swiftParamShapes(codeSignature(caller)), caller), true } return filterByArgc(cands, cs.Argc), false } diff --git a/internal/graph/tslocaltypes.go b/internal/graph/tslocaltypes.go index fe6137e..b4c90e3 100644 --- a/internal/graph/tslocaltypes.go +++ b/internal/graph/tslocaltypes.go @@ -372,8 +372,8 @@ func tsLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { } // Parameter annotations from the declaration's own parens. - if params := tsDeclParams(symbol.RawText); params != "" { - for _, g := range splitTopLevel(params, ',') { + if params := tsDeclParams(symbol.Language, symbol.RawText); params != "" { + for _, g := range splitParams(symbol.Language, params) { g = strings.TrimSpace(g) name, ann, ok := strings.Cut(g, ":") if !ok { @@ -395,7 +395,7 @@ func tsLocalTypes(idx *edgeIndex, symbol *core.SymbolRecord) map[string]string { // Body declarations (highest precedence). if symbol.RawText != "" { - body := stripCommentsAndStrings(symbol.RawText) + body := maskCode(symbol.Language, symbol.RawText) for _, m := range tsVarAnnRe.FindAllStringSubmatch(body, -1) { if t := tsAliasType(idx, symbol.FilePath, tsBareType(m[2])); t != "" { out[m[1]] = t @@ -428,7 +428,9 @@ func tsCollectionFieldElementTypes(idx *edgeIndex, className, preferFile string) (cand.Kind != core.KindClass && cand.Kind != core.KindInterface) { continue } - for _, match := range tsCollectionFieldRe.FindAllStringSubmatch(cand.RawText, -1) { + // Top-level members only (classBody): a comment or a method-local + // `items: Set` is not the field's declaration. + for _, match := range tsCollectionFieldRe.FindAllStringSubmatch(classBody(idx, cand), -1) { out[match[1]] = tsAliasType(idx, cand.FilePath, match[2]) } break @@ -536,7 +538,7 @@ func tsClassFieldTypes(idx *edgeIndex, className, preferFile string, out map[str record(member.Name, tsBareType(m[2])) } case core.KindConstructor: - body := stripCommentsAndStrings(member.RawText) + body := maskCode(member.Language, member.RawText) for _, m := range tsCtorPropRe.FindAllStringSubmatch(body, -1) { record(m[1], tsBareType(m[2])) } @@ -548,8 +550,8 @@ func tsClassFieldTypes(idx *edgeIndex, className, preferFile string, out map[str // this.transport = transport — a plain assignment from a typed // constructor parameter carries the parameter's annotation. ctorParams := map[string]string{} - if params := tsDeclParams(member.RawText); params != "" { - for _, g := range splitTopLevel(params, ',') { + if params := tsDeclParams(member.Language, member.RawText); params != "" { + for _, g := range splitParams(member.Language, params) { name, ann, ok := strings.Cut(strings.TrimSpace(g), ":") if !ok { continue @@ -587,40 +589,41 @@ func tsReceiverChainRe(method string) *regexp.Regexp { } // tsReceiverChainAt recovers the full receiver chain of a `.(` call -// from the caller's source. relLine is 0-based into rawText; the call may wrap, -// so a small window around it is searched. Returns "" if no chain is found. -func tsReceiverChainAt(rawText string, relLine int, calleeName string) string { - if rawText == "" { +// from the caller's masked source. relLine is 0-based into the caller's raw +// text. The call's own line wins; a call wrapped over lines is read from a +// small window around it, and a line outside the captured body (astkit line +// vs Span drift) from the whole body, both only when exactly one call +// matches there: a neighbouring line's or another statement's call to the +// same method must not supply the chain. Returns "" if no chain is found. +func tsReceiverChainAt(idx *edgeIndex, symbol *core.SymbolRecord, relLine int, calleeName string) string { + lines := maskedSymbolLines(idx, symbol) + if len(lines) == 0 { return "" } - lines := strings.Split(rawText, "\n") - if relLine < 0 || relLine >= len(lines) { - // Line outside the captured body (astkit line vs Span drift): fall - // back to scanning the whole body for the first matching call. - relLine = -1 - } re := tsReceiverChainRe(calleeName) - tryLines := func(s string) string { - if m := re.FindStringSubmatch(s); m != nil { - return strings.Join(strings.FieldsFunc(m[1], func(r rune) bool { - return r == ' ' || r == '\t' - }), "") + chainIn := func(s string, unique bool) string { + ms := re.FindAllStringSubmatch(s, 2) + if len(ms) == 0 || (unique && len(ms) > 1) { + return "" } - return "" + return strings.Join(strings.FieldsFunc(ms[0][1], func(r rune) bool { + return r == ' ' || r == '\t' + }), "") } - if relLine >= 0 { - lo, hi := relLine-1, relLine+2 - if lo < 0 { - lo = 0 - } - if hi > len(lines) { - hi = len(lines) - } - if chain := tryLines(strings.Join(lines[lo:hi], "")); chain != "" { - return chain - } + if relLine < 0 || relLine >= len(lines) { + return chainIn(strings.Join(lines, ""), true) + } + if chain := chainIn(lines[relLine], false); chain != "" { + return chain } - return tryLines(strings.Join(lines, "")) + lo, hi := relLine-1, relLine+2 + if lo < 0 { + lo = 0 + } + if hi > len(lines) { + hi = len(lines) + } + return chainIn(strings.Join(lines[lo:hi], ""), true) } // tsFamilyLang reports whether a language uses the TS/JS class-field model. @@ -676,7 +679,7 @@ func tsResolveReceiverChain(idx *edgeIndex, localTypes map[string]string, chain, } // javaClassFieldTypes records one Java class's field name → type from its -// source (javaFieldRe), scoped like tsClassFieldTypes: className resolves to +// fields (javaClassFields), scoped like tsClassFieldTypes: className resolves to // one file from preferFile's perspective, and that file is returned so a // chain walk can scope its next hop. func javaClassFieldTypes(idx *edgeIndex, className, preferFile string, out map[string]string) string { @@ -693,24 +696,7 @@ func javaClassFieldTypes(idx *edgeIndex, className, preferFile string, out map[s default: continue } - for _, m := range javaFieldRe.FindAllStringSubmatch(cls.RawText, -1) { - if t := javaBareType(m[1]); t != "" { - if _, exists := out[m[2]]; !exists { - out[m[2]] = t - } - } - } - for _, f := range idx.byFile[cls.FilePath] { - if f.Kind != core.KindField || (f.ParentSymbol != className && f.ParentSymbol != cls.Name && f.ParentSymbol != cls.QualifiedName) { - continue - } - if _, exists := out[f.Name]; exists { - continue - } - if t := javaBareType(javaIndexedFieldType(f.RawText)); t != "" { - out[f.Name] = t - } - } + javaClassFields(idx, cls, className, out) break } return classFile @@ -747,6 +733,18 @@ func narrowByChainType(idx *edgeIndex, sat *interfaceSatisfaction, localTypes ma } } } + if symbol.Language == "java" { + // A type declared outside the repo (`result.values.size()` on a + // `Map values` field) or a member inherited from a + // base: resolve the last hop exactly as a single-hop receiver of + // that type (subclass overrides, base walk, fan-out cap). It used to + // arrive by accident, through an outer-class field map that read + // nested classes' fields. + last := fullChain[strings.LastIndexByte(fullChain, '.')+1:] + if byType, dispatch, _ := narrowByLocalType(idx, sat, symbol, map[string]string{last: typ}, last, calleeName, cands, nil); len(byType) > 0 || len(dispatch) > 0 { + return byType, dispatch, true + } + } // The chain resolved to a type we know nothing more about (no candidate // on it, no satisfaction entry). Unlike narrowByLocalType's single-hop // contract, do NOT claim decided here: chain resolution is heuristic @@ -786,7 +784,7 @@ func tsTypeOwnMember(idx *edgeIndex, typ, calleeName, preferFile string) []*core // skipped, so a statement reaching the field regex has no '(' and is a field, // not a method signature. func tsClassBodyFieldTypes(rawText string, record func(name, typ string)) { - body := stripCommentsAndStrings(rawText) + body := maskCode("typescript", rawText) open := strings.IndexByte(body, '{') if open < 0 { return @@ -859,26 +857,48 @@ func tsClassBodyFieldTypes(rawText string, record func(name, typ string)) { flush() } -// tsDeclParams extracts the parameter list from a TS/JS declaration's raw -// text by scanning the first balanced paren group. -func tsDeclParams(rawText string) string { - start := strings.IndexByte(rawText, '(') - if start < 0 { - return "" +// tsDeclParams extracts the parameter list from a declaration's text (TS/JS, +// Java, C#, Python, ...): the first balanced paren group, found in the text +// masked as lang, so a `)` inside a default-value string +// (`close: string = ")"`) or a comment does not end the list. The returned +// slice is the original text (a string annotation such as Python's +// `other: "Node"` is type syntax); split it with splitParams. +func tsDeclParams(lang, text string) string { + // Masking is a forward pass, so a prefix masks exactly as it would in + // the whole text; most parameter lists close well inside the first + // couple of KB of a body. + const prefix = 2048 + if len(text) > prefix { + if start, end, ok := declParamGroup(maskCode(lang, text[:prefix])); ok { + return text[start:end] + } + } + if start, end, ok := declParamGroup(maskCode(lang, text)); ok { + return text[start:end] + } + return "" +} + +// declParamGroup returns the bounds of the inside of masked's first balanced +// paren group; ok is false when there is none or it does not close. +func declParamGroup(masked string) (start, end int, ok bool) { + open := strings.IndexByte(masked, '(') + if open < 0 { + return 0, 0, false } depth := 0 - for k := start; k < len(rawText); k++ { - switch rawText[k] { + for k := open; k < len(masked); k++ { + switch masked[k] { case '(', '[', '{': depth++ case ')', ']', '}': depth-- if depth == 0 { - return rawText[start+1 : k] + return open + 1, k, true } } } - return "" + return 0, 0, false } // tsChosenTypeDecl picks the declaration that tsBaseClasses reads @@ -920,6 +940,27 @@ func constructorBaseClasses(idx *edgeIndex, language, className, preferDir strin return baseClassesFor(idx, language, className, preferDir) } +// fileLocalTypeName reports whether className is a name the parser made up +// for an anonymous class (``). Its line and column are +// unique only inside its own file and move with every line above it, so +// looked up by name it binds whichever other file has a class at the same +// position (guava: AbstractIteratorTest's and ImmutableSortedMapTest's +// `` share a directory). +func fileLocalTypeName(className string) bool { + return strings.HasPrefix(className, "<") +} + +// baseClassesInFileFor is baseClassesFor for a class declared in file: a +// file-local (anonymous) class is read from its declaration in that file, +// any other name resolves as baseClassesFor does from file's directory. +func baseClassesInFileFor(idx *edgeIndex, language, className, file string) []string { + if fileLocalTypeName(className) { + bases, _ := tsBaseClassesInFile(idx, className, file) + return bases + } + return baseClassesFor(idx, language, className, dirOf(file)) +} + // tsBaseClassesInFile reads the inheritance clauses of the class-like // declaration named className in file (ok=false when file declares none). func tsBaseClassesInFile(idx *edgeIndex, className, file string) ([]string, bool) { diff --git a/internal/invariance/cases_a_test.go b/internal/invariance/cases_a_test.go new file mode 100644 index 0000000..6e3eb07 --- /dev/null +++ b/internal/invariance/cases_a_test.go @@ -0,0 +1,313 @@ +package invariance + +// Group A: astkit extractors (call sites, enum/protocol/base clauses, +// modifiers, Lombok accessors, COBOL verbs) observed through the grove index. + +func casesA() []twinCase { + return []twinCase{ + { + id: "A1-rust-macro-args", + clean: one("lib.rs", `pub fn phantom_one(x: i32) -> i32 { x } +pub fn bar(x: i32) -> i32 { x } +pub fn f(c: char) { + assert!(true, + ); + assert_eq!(c, '"', "see"); +} +`), + decoy: one("lib.rs", `pub fn phantom_one(x: i32) -> i32 { x } +pub fn bar(x: i32) -> i32 { x } +pub fn f(c: char) { + assert!(true, // note: phantom_one(2) + ); + assert_eq!(c, '"', "see bar(1)"); +} +`), + want: []string{"S f"}, + wantNot: []string{"E f calls phantom_one", "E f calls bar"}, + }, + { + id: "A2-c-macro-body-strings", + clean: one("a.c", `#include +int value(int x) { return x; } +int helper_fn(int x) { return x; } +int other_fn(int x) { return x; } +#define LOG(x) printf("v=%d", x) +#define LOG2(x) do { \ + printf("%d", (x)); \ + } while (0) +`), + decoy: one("a.c", `#include +int value(int x) { return x; } +int helper_fn(int x) { return x; } +int other_fn(int x) { return x; } +#define LOG(x) printf("value(%d) helper_fn(x)", x) +#define LOG2(x) do { \ + printf("%d", (x)); /* other_fn(x) */ \ + } while (0) // other_fn(x) +`), + want: []string{"S LOG", "S LOG2"}, + wantNot: []string{"E LOG calls value", "E LOG calls helper_fn", "E LOG2 calls other_fn"}, + }, + { + id: "A3-objc-enum-string-and-if0", + clean: one("m.m", `#import +static NSString *k = @"none"; +#define X 1 +typedef NS_ENUM(NSInteger, Mode) { + ModeA, +#if X + ModeB, +#endif + ModeC +}; +`), + decoy: one("m.m", `#import +static NSString *k = @"typedef NS_ENUM(NSInteger, Ghost) { GhostA };"; +#if 0 +typedef NS_ENUM(NSInteger, Dead) { DeadA }; +#endif +#define X 1 +typedef NS_ENUM(NSInteger, Mode) { + ModeA, +#if X + ModeB, +#endif + ModeC +}; +`), + want: []string{"S Mode", "S ModeA", "S ModeB", "S ModeC"}, + wantNot: []string{"S Ghost", "S GhostA", "S Dead", "S DeadA"}, + }, + { + id: "A4-objc-protocol-comment", + clean: one("m.h", `@interface Sup +@end +@protocol FakeProto +@end +@protocol RealProto +@end +@interface M : Sup +@end +`), + decoy: one("m.h", `@interface Sup +@end +@protocol FakeProto +@end +@protocol RealProto +@end +@interface M : Sup /* */ +@end +`), + want: []string{"E M implements RealProto", "E M extends Sup"}, + wantNot: []string{"E M implements FakeProto"}, + }, + { + id: "A4-kotlin-ctor-param-comment", + clean: one("Foo.kt", `open class Base +class Thing +class Foo( + val a: Int, +) : Base() +`), + decoy: one("Foo.kt", `open class Base +class Thing +class Foo( + val a: Int, // : Thing +) : Base() +`), + want: []string{"S Foo"}, + wantNot: []string{"E Foo uses-type Thing", "E Foo extends Thing", "E Foo implements Thing"}, + }, + { + id: "A4-python-base-and-param-comments", + clean: one("a.py", `class Base: + pass + + +class Fake: + pass + + +class A(Base): + pass + + +def g(a, + b): + return a +`), + decoy: one("a.py", `class Base: + pass + + +class Fake: + pass + + +class A(Base): # was Fake + pass + + +def g(a, # first, really (Fake) + b): + return a +`), + sig: true, + want: []string{"E A extends Base"}, + wantNot: []string{"E A uses-type Fake", "E A extends Fake", "E g uses-type Fake"}, + }, + { + id: "A4-python-multiline-def-return", + clean: one("a.py", `class Gadget: + pass + + +def build(size,) -> Gadget: + return Gadget() +`), + decoy: one("a.py", `class Gadget: + pass + + +def build( + size, +) -> Gadget: + return Gadget() +`), + sig: true, + want: []string{"E build uses-type Gadget"}, + }, + { + id: "A4-csharp-attribute-string", + rawAnnotations: true, + clean: one("A.cs", `using System.ComponentModel; +namespace P { +class Fake {} +public class A { + [Description("plain")] public void M() {} +} +} +`), + decoy: one("A.cs", `using System.ComponentModel; +namespace P { +class Fake {} +public class A { + [Description("see Fake and new Fake()")] public void M() {} +} +} +`), + want: []string{"S A.M"}, + wantNot: []string{"E A.M uses-type Fake", "E A uses-type Fake", "E A.M calls Fake"}, + }, + { + id: "A5-kotlin-fun-in-comment", + clean: one("p.kt", `// just for you +interface Plain { + fun x(): Int +} +`), + decoy: one("p.kt", `// just for fun +interface Plain { + fun x(): Int +} +`), + want: []string{"S Plain"}, + wantNot: []string{"S Plain mod=fun"}, + }, + { + id: "A6-java-public-in-name-and-string", + clean: one("src/p/A.java", `package p; +class A { + void publication() {} + String mode = "x"; +} +`), + decoy: one("src/p/A.java", `package p; +class A { + void publication() {} + String mode = "public"; +} +`), + want: []string{"S A.publication unexported", "S A.mode unexported"}, + wantNot: []string{"S A.publication exported", "S A.mode exported", "S A.mode mod=public"}, + }, + { + id: "A7-java-lombok-string-type", + clean: one("src/p/A.java", `package p; +import lombok.Getter; +class A { + @Getter String kind = "x"; +} +@lombok.Getter +class B { + String name; +} +`), + decoy: one("src/p/A.java", `package p; +import lombok.Getter; +class A { + @Getter String kind = "boolean x"; +} +@lombok.Getter +class B { + String name; +} +`), + want: []string{"S A.getKind", "S B.getName"}, + wantNot: []string{"S A.isKind"}, + }, + { + id: "A8-cobol-literals-and-inline-comment", + clean: one("PROG1.cbl", ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG1. + DATA DIVISION. + WORKING-STORAGE SECTION. + 01 X PIC 9. + PROCEDURE DIVISION. + MAIN-PARA. + DISPLAY 'PLEASE WAIT'. + MOVE 1 TO X + DISPLAY 'HELLO'. + STOP RUN. + BACKUP. + DISPLAY 'B'. + OLD-PARA. + DISPLAY 'O'. +`), + decoy: one("PROG1.cbl", ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG1. + DATA DIVISION. + WORKING-STORAGE SECTION. + 01 X PIC 9. + PROCEDURE DIVISION. + MAIN-PARA. + DISPLAY 'PLEASE PERFORM BACKUP FIRST'. + MOVE 1 TO X *> PERFORM OLD-PARA + DISPLAY 'CALL SUPPORT'. + STOP RUN. + BACKUP. + DISPLAY 'B'. + OLD-PARA. + DISPLAY 'O'. +`), + want: []string{"S BACKUP", "S OLD-PARA"}, + wantNot: []string{"E MAIN-PARA calls BACKUP", "E MAIN-PARA calls OLD-PARA", "E MAIN-PARA calls SUPPORT", "E * calls SUPPORT"}, + }, + { + id: "A9-c-header-class-comment", + clean: one("a.h", `/* + * Helpers for the C API. + */ +int real(void); +`), + decoy: one("a.h", `/* + class handling for the C API + */ +int real(void); +`), + want: []string{"S real lang=c"}, + wantNot: []string{"S real lang=cpp", "S handling"}, + }, + } +} diff --git a/internal/invariance/cases_c_test.go b/internal/invariance/cases_c_test.go new file mode 100644 index 0000000..2971e36 --- /dev/null +++ b/internal/invariance/cases_c_test.go @@ -0,0 +1,119 @@ +package invariance + +// Group C: decoy invariant round 2, C family (fmt, nlohmann/json, +// AFNetworking). Each clean file is one the grammar could not parse; the +// decoy adds a comment line after every `{` and a block comment at the +// end, as research's decoy injector does. Error recovery, the line +// scanner's window and the namespace choice moved with those comments. + +const cDecoyLine = " // decoy: class Fake extends Base { void ghost() { helper(1); new Fake(); } } Widget::draw(x); don't 'quote\n" + +const cDecoyTail = "/* decoy block: don't 'quote\nclass Fake2 { void ghost(); };\n*/\n" + +func casesC() []twinCase { + return []twinCase{ + { + id: "C1-objc-availability-macros", + clean: one("p.m", `#import + +typedef NS_ENUM(NSUInteger, AFSSLPinningMode) { + AFSSLPinningModeNone, + AFSSLPinningModeCertificate, +}; + +NS_ASSUME_NONNULL_BEGIN + +@interface AFSecurityPolicy : NSObject +@property (nonatomic, strong) NSURLSessionTaskMetrics *metrics AF_API_AVAILABLE(ios(10), macosx(10.12)); +- (instancetype)initWithMode:(AFSSLPinningMode)mode NS_DESIGNATED_INITIALIZER; +- (void)reset; +@end + +@implementation AFSecurityPolicy +- (void)didFinish:(NSURLSessionTaskMetrics *)metrics AF_API_AVAILABLE(ios(10), macosx(10.12)) { + [self reset]; +} +- (void)reset { +} +@end + +NS_ASSUME_NONNULL_END +`), + decoy: one("p.m", `#import + +typedef NS_ENUM(NSUInteger, AFSSLPinningMode) { +`+cDecoyLine+` AFSSLPinningModeNone, + AFSSLPinningModeCertificate, +}; + +NS_ASSUME_NONNULL_BEGIN + +@interface AFSecurityPolicy : NSObject +@property (nonatomic, strong) NSURLSessionTaskMetrics *metrics AF_API_AVAILABLE(ios(10), macosx(10.12)); +- (instancetype)initWithMode:(AFSSLPinningMode)mode NS_DESIGNATED_INITIALIZER; +- (void)reset; +@end + +@implementation AFSecurityPolicy +- (void)didFinish:(NSURLSessionTaskMetrics *)metrics AF_API_AVAILABLE(ios(10), macosx(10.12)) { +`+cDecoyLine+` [self reset]; +} +- (void)reset { +`+cDecoyLine+`} +@end + +NS_ASSUME_NONNULL_END +`+cDecoyTail), + want: []string{"S AFSecurityPolicy kind=class", "S AFSecurityPolicy.metrics kind=field", + "S AFSecurityPolicy.initWithMode: kind=method", "E AFSecurityPolicy.didFinish: calls AFSecurityPolicy.reset"}, + wantNot: []string{"S if", "S AF_API_AVAILABLE", "S initWithMode:NS_DESIGNATED_INITIALIZER:"}, + }, + { + id: "C2-cpp-export-macros", + clean: one("gtest.h", `namespace testing { +namespace internal { + +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned char, '\0'); +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(float, 0); + +class GTEST_API_ RE { + public: + static DOCTEST_CONSTEXPR size_type len = 24; + RE(const char* regex) { Init(regex); } + private: + void Init(const char* regex); +}; + +bool IsAlpha(char ch) { + return isalpha(ch) != 0; +} + +} // namespace internal +} // namespace testing +`), + decoy: one("gtest.h", `namespace testing { +`+cDecoyLine+`namespace internal { +`+cDecoyLine+` +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned char, '\0'); +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(float, 0); + +class GTEST_API_ RE { +`+cDecoyLine+` public: + static DOCTEST_CONSTEXPR size_type len = 24; + RE(const char* regex) { Init(regex); } + private: + void Init(const char* regex); +}; + +bool IsAlpha(char ch) { +`+cDecoyLine+` return isalpha(ch) != 0; +} + +} // namespace internal +} // namespace testing +`+cDecoyTail), + want: []string{"S testing::internal::RE kind=class", "S testing::internal::IsAlpha kind=function", "S testing::internal::RE::Init kind=method", "S testing::internal::RE::len kind=field"}, + wantNot: []string{"S GTEST_API_", "S GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_"}, + }, + } +} diff --git a/internal/invariance/cases_e_test.go b/internal/invariance/cases_e_test.go new file mode 100644 index 0000000..7b33cdd --- /dev/null +++ b/internal/invariance/cases_e_test.go @@ -0,0 +1,255 @@ +package invariance + +// Group E: grove internal/graph edge builders. Tool-output findings +// (E4 change-impact, E5 missing-implementations, E7/E8 rename-plan, +// E9 change-impact) are in pkg/grove/comment_string_tools_test.go. + +func casesE() []twinCase { + return []twinCase{ + { + id: "E1-python-docstring-local-def", + clean: one("a.py", `def helper(x): + return x + + +def caller(): + """Example.""" + return helper(1) +`), + decoy: one("a.py", `def helper(x): + return x + + +def caller(): + """Example: + + def helper(x): + return 2 + """ + return helper(1) +`), + want: []string{"E caller calls helper"}, + }, + { + id: "E1-js-comment-and-template-local-def", + clean: one("a.js", `function helper(x) { return x; } +function caller() { + return helper(1); +} +function caller2() { + const t = ""; + return helper(2); +} +`), + decoy: one("a.js", "function helper(x) { return x; }\nfunction caller() {\n /* function helper(x) { return 0; } */\n return helper(1);\n}\nfunction caller2() {\n const t = `\nfunction helper(y) {}\n`;\n return helper(2);\n}\n"), + want: []string{"E caller calls helper", "E caller2 calls helper"}, + }, + { + id: "E2-go-embedded-field-comment", + clean: map[string]string{"go.mod": goMod, "p.go": `package p + +type Reader struct{} + +type Holder struct { + x int +} +`}, + decoy: map[string]string{"go.mod": goMod, "p.go": `package p + +type Reader struct{} + +type Holder struct { + /* + Reader + */ + x int +} +`}, + want: []string{"S Holder"}, + wantNot: []string{"E Holder extends Reader", "E Holder uses-type Reader"}, + }, + { + id: "E3-rust-workspace-comment-path", + clean: map[string]string{ + "Cargo.toml": "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"2\"\n", + "alpha/Cargo.toml": "[package]\nname = \"alpha\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "beta/Cargo.toml": "[package]\nname = \"beta\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "beta/src/lib.rs": "pub fn normalize(x: i32) -> i32 { x }\n", + "alpha/src/util.rs": "pub fn normalize(x: i32) -> i32 { x + 1 }\n", + "alpha/src/b.rs": "use super::util::*;\n\npub fn other(x: i32) -> i32 { normalize(x) }\n", + "alpha/src/lib.rs": `pub mod b; +pub mod util; + +pub fn run() -> i32 { + 0 +} +`}, + decoy: map[string]string{ + "Cargo.toml": "[workspace]\nmembers = [\"alpha\", \"beta\"]\nresolver = \"2\"\n", + "alpha/Cargo.toml": "[package]\nname = \"alpha\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "beta/Cargo.toml": "[package]\nname = \"beta\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "beta/src/lib.rs": "pub fn normalize(x: i32) -> i32 { x }\n", + "alpha/src/util.rs": "pub fn normalize(x: i32) -> i32 { x + 1 }\n", + "alpha/src/b.rs": "use super::util::*;\n\npub fn other(x: i32) -> i32 { normalize(x) }\n", + "alpha/src/lib.rs": `pub mod b; +pub mod util; + +pub fn run() -> i32 { + // Same contract as beta::normalize + 0 +} +`}, + want: []string{"E other calls normalize@alpha/src/util.rs"}, + wantNot: []string{"E other calls normalize@beta/src/lib.rs"}, + }, + { + id: "E4-python-multiline-bases", + clean: one("a.py", `class Base: + def method(self): + return 0 + + +class Mixin: + pass + + +class Multi(Base,): + def method(self): + return 1 + + +class Commented(Base, Mixin): + pass +`), + decoy: one("a.py", `class Base: + def method(self): + return 0 + + +class Mixin: + pass + + +class Multi( + Base, +): + def method(self): + return 1 + + +class Commented(Base, # the (old) Mixin + Mixin): + pass +`), + want: []string{"E Multi extends Base", "E Commented extends Base", "E Commented extends Mixin"}, + }, + { + id: "E6-kotlin-supertype-comments", + clean: one("k.kt", `open class KBase +interface KIface +class KOther +class Widget +class Gadget +class KTrailing : KBase(), KIface {} +class KMultiComment : KBase(), KIface { +} +fun build(size: Int): Gadget? = null +`), + decoy: one("k.kt", `open class KBase +interface KIface +class KOther +class Widget +class Gadget +class KTrailing : KBase(), KIface /* see KOther */ {} +class KMultiComment : KBase(), // the base + KIface { +} +fun build( + size: Int, // not a Widget +): Gadget? = null +`), + want: []string{"E KTrailing implements KIface", "E KMultiComment implements KIface", + "E KTrailing extends KBase", "E KMultiComment extends KBase", "E build uses-type Gadget"}, + wantNot: []string{"E KTrailing uses-type KOther", "E KTrailing implements KOther", "E build uses-type Widget"}, + }, + { + id: "E10-cobol-move-comment-and-continuation", + clean: one("PROG2.cbl", ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG2. + DATA DIVISION. + WORKING-STORAGE SECTION. + 01 WS-TOTAL PIC 9(4). + 01 WS-OUT PIC 9(4). + 01 WS-OLD PIC 9(4). + PROCEDURE DIVISION. + PARA-ONE. + MOVE WS-TOTAL TO WS-OUT. + PARA-TWO. + MOVE WS-TOTAL TO WS-OUT. + STOP RUN. +`), + decoy: one("PROG2.cbl", ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG2. + DATA DIVISION. + WORKING-STORAGE SECTION. + 01 WS-TOTAL PIC 9(4). + 01 WS-OUT PIC 9(4). + 01 WS-OLD PIC 9(4). + PROCEDURE DIVISION. + PARA-ONE. + MOVE WS-TOTAL TO WS-OUT. *> WAS WS-OLD + PARA-TWO. + MOVE WS-TOTAL + TO WS-OUT. + STOP RUN. +`), + want: []string{"E PARA-ONE writes WS-OUT", "E PARA-ONE reads WS-TOTAL", + "E PARA-TWO writes WS-OUT", "E PARA-TWO reads WS-TOTAL"}, + wantNot: []string{"E PARA-ONE writes WS-OLD", "E PARA-ONE reads WS-OLD", "E PARA-TWO reads WS-OUT"}, + }, + { + // Suspected (not in the confirmed 53): javaQualifiedCallRe reads + // a trailing comment on a call line. + id: "E-suspect-java-qualified-call-comment", + clean: one("src/p/U.java", `package p; +class Other { static void build() {} } +class Real { static void build() {} } +class U { + void m() { + Real.build(); + } +} +`), + decoy: one("src/p/U.java", `package p; +class Other { static void build() {} } +class Real { static void build() {} } +class U { + void m() { + Real.build(); // Other.build() + } +} +`), + want: []string{"E U.m calls Real.build"}, + wantNot: []string{"E U.m calls Other.build", "E U.m uses-type Other"}, + }, + { + // Suspected: graphCallableSymbol takes a JS class field whose + // string initializer contains "=>" for a callable. + id: "E-suspect-js-field-arrow-in-string", + clean: one("c.js", `export class C { + label = "a to b"; + run() { return this.label; } +} +export function use(c) { return c.label; } +`), + decoy: one("c.js", `export class C { + label = "a => b"; + run() { return this.label; } +} +export function use(c) { return c.label; } +`), + want: []string{"S C.run"}, + wantNot: []string{"S C.label kind=method", "S C.label kind=function", "E use calls C.label", "E C.run calls C.label"}, + }, + } +} diff --git a/internal/invariance/cases_j_test.go b/internal/invariance/cases_j_test.go new file mode 100644 index 0000000..2b8b15c --- /dev/null +++ b/internal/invariance/cases_j_test.go @@ -0,0 +1,184 @@ +package invariance + +// Group J: Java differences the decoy invariant found on real repositories +// (round 2: guava, jackson-databind). + +func casesJ() []twinCase { + return []twinCase{ + { + // jackson-databind BuilderCreatorSubtype4742Test: javaDeclSource + // skipped leading annotations with its own quote tracking, so an + // apostrophe in a comment inside @JsonSubTypes({...}) ran the + // annotation to the end of the text, the abstract method lost + // its parameter list, and the override edge went with it. + id: "J1-java-comment-inside-annotation-args", + clean: one("src/p/B.java", `package p; +abstract class Builder { + @Sub({ + @Type(name = "bird") + }) + public abstract Builder properties(String p); +} +class Impl extends Builder { + @Override + public Impl properties(String p) { return this; } +} +`), + decoy: one("src/p/B.java", `package p; +abstract class Builder { + @Sub({ // don't 'quote "unterminated ( + @Type(name = "bird") /* ) */ + }) + public abstract Builder properties(String p); +} +class Impl extends Builder { + @Override + public Impl properties(String p) { return this; } +} +`), + want: []string{"E Impl.properties overrides Builder"}, + rawAnnotations: true, + }, + { + // guava: anonymous classes are named by position, so two files of + // one package each had a ``. The hierarchy and + // method-set maps looked them up by name, gave one file's class + // the other's base, and a comment line above either one moved + // the collision (10k dispatch edges changed under the decoy). + // The decoy keeps every line in place (names carry line numbers) + // and only adds comments. + id: "J2-java-anonymous-classes-same-position", + clean: map[string]string{ + "src/p/Task.java": "package p;\ninterface Task { void run(); }\n", + "src/p/A.java": `package p; +class A { + Task make() { + return new Task() { + public void run() {} + }; + } +} +`, + "src/p/B.java": `package p; +class B { + Thread make() { + return new Thread() { + public void run() {} + }; + } +} +`, + "src/p/C.java": `package p; +class C { + void go(Task task) { + task.run(); + } +} +`, + }, + decoy: map[string]string{ + "src/p/Task.java": "package p;\ninterface Task { void run(); } // class Fake implements Task\n", + "src/p/A.java": `package p; +class A { // new Thread() { + Task make() { // new Task() { + return new Task() { // public void run() {} + public void run() {} + }; + } +} +`, + "src/p/B.java": `package p; +class B { /* extends Task */ + Thread make() { + return new Thread() { // implements Task + public void run() {} + }; + } +} +`, + "src/p/C.java": `package p; +class C { + void go(Task task) { // Thread task; + task.run(); + } +} +`, + }, + // Dispatch edges are astkit-mode only (javac binds Task.run), so + // the facts are the hierarchy; graph/java_anonymous_test.go + // covers the by-name lookups. + want: []string{"E run@src/p/A.java overrides Task", "E C.go calls Task.run"}, + wantNot: []string{"E run@src/p/B.java overrides Task"}, + }, + { + // A local declared without an initializer and assigned later + // (`Map map;` before a try). Its type used to arrive by + // accident, from a class "field" regex run over every method + // body; masking the class body to depth 1 lost it. + id: "J3-java-local-declared-then-assigned", + clean: one("src/p/U.java", `package p; +interface Store { int size(); } +class Mem implements Store { public int size() { return 0; } } +class U { + Store make() { return new Mem(); } + void m() { + Store s; + s = make(); + s.size(); + } +} +`), + decoy: one("src/p/U.java", `package p; +interface Store { int size(); } +class Mem implements Store { public int size() { return 0; } } +class U { + Store make() { return new Mem(); } // Mem s; + void m() { + Store s; /* Mem s; */ + s = make(); + s.size(); // Mem.size() + } +} +`), + want: []string{"E U.m calls Store.size"}, + }, + { + // newtonsoft: astkit cut C# attribute sections out of signatures + // (strings in them read as types), so typeof(T) and Enum.Member + // arguments lost their uses-type edges with them. They are read + // from the masked attribute text now; a string or a named + // argument's name is not a type. + id: "J4-csharp-attribute-type-references", + clean: one("A.cs", `namespace P { + public class AConverter { } + public enum Mode { Fast, Slow } + public class Name { } + public class ConverterAttribute : System.Attribute { } + public class Holder { + [Converter(typeof(AConverter), Mode.Fast)] + public int Value { get; set; } + [Converter(Name = "x")] + public int Other { get; set; } + } +} +`), + decoy: one("A.cs", `namespace P { + public class AConverter { } + public enum Mode { Fast, Slow } + public class Name { } + public class ConverterAttribute : System.Attribute { } + public class Holder { + [Converter(typeof(AConverter), /* typeof(Name) */ Mode.Fast)] // Name.Thing + public int Value { get; set; } + [Converter(Name = "x Name.Thing typeof(Mode)")] + public int Other { get; set; } + } +} +`), + rawAnnotations: true, + want: []string{"E Holder.Value uses-type AConverter", "E Holder.Value uses-type Mode", + "E Holder.Value uses-type ConverterAttribute", "E Holder.Other uses-type ConverterAttribute"}, + wantNot: []string{"E Holder.Value uses-type Name", "E Holder.Other uses-type Name", "E Holder.Other uses-type Mode"}, + }, + } +} diff --git a/internal/invariance/cases_l_test.go b/internal/invariance/cases_l_test.go new file mode 100644 index 0000000..cd3fb62 --- /dev/null +++ b/internal/invariance/cases_l_test.go @@ -0,0 +1,539 @@ +package invariance + +// Group L: grove internal/graph local-type inference reads comments and +// strings as declarations/assignments. + +const goMod = "module example.com/p\n\ngo 1.21\n" + +func one(path, body string) map[string]string { return map[string]string{path: body} } + +// with returns a copy of base with files replaced/added. +func with(base map[string]string, kv ...string) map[string]string { + out := map[string]string{} + for k, v := range base { + out[k] = v + } + for i := 0; i+1 < len(kv); i += 2 { + out[kv[i]] = kv[i+1] + } + return out +} + +func casesL() []twinCase { + return []twinCase{ + { + id: "L1-python", + clean: one("app.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +def work(): + print("it was removed") + client = Foo() + return client.go() +`), + decoy: one("app.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +def work(): + print('client = Bar() was removed') + client = Foo() + return client.go() +`), + want: []string{"E work calls Foo.go"}, + wantNot: []string{"E work calls Bar.go"}, + }, + { + id: "L1-ts", + clean: one("app.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export function work() { + const note = "it was removed"; + const client = new Foo(); + return client.go(); +} +`), + decoy: one("app.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export function work() { + const note = 'client = new Bar() was removed'; + const client = new Foo(); + return client.go(); +} +`), + want: []string{"E work calls Foo.go"}, + wantNot: []string{"E work calls Bar.go"}, + }, + { + id: "L1-php", + clean: one("app.php", `go(); + } +} +`), + decoy: one("app.php", `go(); + } +} +`), + want: []string{"E Svc.work calls Foo.go"}, + wantNot: []string{"E Svc.work calls Bar.go"}, + }, + { + id: "L2-rust-rawstring", + clean: map[string]string{ + "Cargo.toml": "[package]\nname = \"p\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "src/lib.rs": `pub struct Foo; +impl Foo { + pub fn new() -> Foo { Foo } + pub fn go(&self) -> i32 { 1 } +} +pub struct Bar; +impl Bar { + pub fn new() -> Bar { Bar } + pub fn go(&self) -> i32 { 2 } +} +pub fn work() -> i32 { + let client = Foo::new(); + let _fixture = ""; + client.go() +} +`}, + decoy: map[string]string{ + "Cargo.toml": "[package]\nname = \"p\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "src/lib.rs": `pub struct Foo; +impl Foo { + pub fn new() -> Foo { Foo } + pub fn go(&self) -> i32 { 1 } +} +pub struct Bar; +impl Bar { + pub fn new() -> Bar { Bar } + pub fn go(&self) -> i32 { 2 } +} +pub fn work() -> i32 { + let client = Foo::new(); + let _fixture = r#" +let client = Bar::new(); +"#; + client.go() +} +`}, + want: []string{"E work calls Foo.go"}, + wantNot: []string{"E work calls Bar.go"}, + }, + { + id: "L2-python-docstring", + clean: one("svc.py", `class Real: + def go(self): + return 0 + + +class Foo: + def go(self): + return 1 + + +class Redis: + def go(self): + return 2 + + +class Svc: + def __init__(self): + """Set up the service.""" + self.x = Real() + self.cache = Real() + + def run(self): + return self.x.go() + + def run2(self): + return self.cache.go() +`), + decoy: one("svc.py", `class Real: + def go(self): + return 0 + + +class Foo: + def go(self): + return 1 + + +class Redis: + def go(self): + return 2 + + +class Svc: + def __init__(self): + """He said "hi. + self.x = Foo() + """ + '''Don't do this: self.cache = Redis()''' + self.x = Real() + self.cache = Real() + + def run(self): + return self.x.go() + + def run2(self): + return self.cache.go() +`), + want: []string{"E Svc.run calls Real.go", "E Svc.run2 calls Real.go"}, + wantNot: []string{"E Svc.run calls Foo.go", "E Svc.run2 calls Redis.go"}, + }, + { + // Shape 1 (a local in another method leaks into b) is checked by + // want/wantNot; shape 2 (a commented-out declaration) by the twin. + id: "L3-java", + clean: one("src/p/Svc.java", `package p; +class Foo { void run() {} } +class Bar { void run() {} } +class Svc { + void a() { Foo helper = new Foo(); helper.run(); } + private Bar helper; + void b() { helper.run(); } +} +`), + decoy: one("src/p/Svc.java", `package p; +class Foo { void run() {} } +class Bar { void run() {} } +class Svc { + void a() { Foo helper = new Foo(); helper.run(); } + /* + Foo helper = null; + */ + private Bar helper; + void b() { helper.run(); } +} +`), + want: []string{"E Svc.b calls Bar.run", "E Svc.a calls Foo.run"}, + wantNot: []string{"E Svc.b calls Foo.run"}, + }, + { + id: "L4-csharp", + clean: one("Svc.cs", `namespace P { +class Foo { public void Go() {} } +class Bar { public void Go() {} } +class Svc { + void A() { Foo helper = new Foo(); helper.Go(); } + private Bar helper; + void B() { helper.Go(); } +} +} +`), + decoy: one("Svc.cs", `namespace P { +class Foo { public void Go() {} } +class Bar { public void Go() {} } +class Svc { + void A() { Foo helper = new Foo(); helper.Go(); } + /* + Foo helper = null; + */ + private Bar helper; + void B() { helper.Go(); } +} +} +`), + want: []string{"E Svc.B calls Bar.Go", "E Svc.A calls Foo.Go"}, + wantNot: []string{"E Svc.B calls Foo.Go"}, + }, + { + id: "L5-cpp-header", + clean: one("svc.hpp", `struct Foo { int go() { return 1; } }; +struct Bar { int go() { return 2; } }; +struct Svc { + int a() { Foo helper; return helper.go(); } + int b() { return helper.go(); } +private: + Bar helper; +}; +`), + decoy: one("svc.hpp", `struct Foo { int go() { return 1; } }; +struct Bar { int go() { return 2; } }; +struct Svc { + int a() { Foo helper; return helper.go(); } + int b() { return helper.go(); } +private: + /* + Foo helper; + */ + Bar helper; +}; +`), + want: []string{"E Svc.b calls Bar.go", "E Svc.a calls Foo.go"}, + wantNot: []string{"E Svc.b calls Foo.go"}, + }, + { + id: "L6-python-numpy-docstring", + clean: one("svc.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +class Svc: + """Service.""" + + def __init__(self): + self.client = Foo() + + def run(self): + return self.client.go() +`), + decoy: one("svc.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +class Svc: + """Service. + + Attributes + ---------- + client : Bar + The old client. + """ + + def __init__(self): + self.client = Foo() + + def run(self): + return self.client.go() +`), + want: []string{"E Svc.run calls Foo.go"}, + wantNot: []string{"E Svc.run calls Bar.go"}, + }, + { + // A method-local annotated variable must not type self.tmp. + id: "L6-python-method-local", + clean: one("svc.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +class Svc: + def __init__(self): + self.tmp = Foo() + + def helper(self): + tmp: Bar = Bar() + return tmp + + def run(self): + return self.tmp.go() +`), + decoy: one("svc.py", `class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +class Svc: + def __init__(self): + # self.tmp: Bar + self.tmp = Foo() + + def helper(self): + tmp: Bar = Bar() + return tmp + + def run(self): + return self.tmp.go() +`), + want: []string{"E Svc.run calls Foo.go"}, + wantNot: []string{"E Svc.run calls Bar.go"}, + }, + { + id: "L7-ts", + clean: one("svc.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export class Svc { + items: Set = new Set(); + run() { + this.items.forEach((x) => x.go()); + } +} +`), + decoy: one("svc.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export class Svc { + items: Set = new Set(); + run() { + // items: Set was the old shape + this.items.forEach((x) => x.go()); + } +} +`), + want: []string{"E Svc.run calls Foo.go"}, + wantNot: []string{"E Svc.run calls Bar.go"}, + }, + { + id: "L8-ts", + clean: one("svc.ts", `export class Cache { fetch(k: string) { return k; } } +export class Store { fetch(k: string) { return k; } } +export class Inner { cache: Cache = new Cache(); store: Store = new Store(); } +export class Svc { + inner: Inner = new Inner(); + get(k: string) { + return this.inner.store.fetch("k"); + } +} +`), + decoy: one("svc.ts", `export class Cache { fetch(k: string) { return k; } } +export class Store { fetch(k: string) { return k; } } +export class Inner { cache: Cache = new Cache(); store: Store = new Store(); } +export class Svc { + inner: Inner = new Inner(); + get(k: string) { + // prefer this.inner.cache.fetch(k) once warmed + return this.inner.store.fetch("k"); + } +} +`), + want: []string{"E Svc.get calls Store.fetch"}, + wantNot: []string{"E Svc.get calls Cache.fetch"}, + }, + { + id: "L9-php-extends-comment", + clean: one("dog.php", `speak(); } +} +`), + decoy: one("dog.php", `speak(); } +} +`), + want: []string{"E Dog.run calls Base.speak", "E Dog extends Base"}, + wantNot: []string{"E Dog implements Loud", "E Dog.run calls Loud.speak"}, + }, + { + id: "L9-php-property-comment", + clean: one("svc.php", `client->go(); } +} +`), + decoy: one("svc.php", `client->go(); } +} +`), + want: []string{"E Svc.run calls Foo.go"}, + wantNot: []string{"E Svc.run calls Bar.go"}, + }, + { + id: "L10-ts-default-string", + clean: one("p.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export function angle(open: string = "x", client: Foo) { return client.go(); } +export function paren(close: string = "x", client: Foo) { return client.go(); } +`), + decoy: one("p.ts", `export class Foo { go() { return 1; } } +export class Bar { go() { return 2; } } +export function angle(open: string = "<", client: Foo) { return client.go(); } +export function paren(close: string = ")", client: Foo) { return client.go(); } +`), + want: []string{"E angle calls Foo.go", "E paren calls Foo.go"}, + wantNot: []string{"E angle calls Bar.go", "E paren calls Bar.go"}, + }, + { + id: "L11-go-struct-comment", + clean: map[string]string{"go.mod": goMod, "p.go": `package p + +type Foo struct{} + +func (Foo) Go() int { return 1 } + +type Bar struct{} + +func (Bar) Go() int { return 2 } + +type Svc struct { + client *Foo +} + +func (s *Svc) Run() int { return s.client.Go() } +`}, + decoy: map[string]string{"go.mod": goMod, "p.go": `package p + +type Foo struct{} + +func (Foo) Go() int { return 1 } + +type Bar struct{} + +func (Bar) Go() int { return 2 } + +type Svc struct { + client *Foo + /* + client Bar + */ +} + +func (s *Svc) Run() int { return s.client.Go() } +`}, + want: []string{"E Svc.Run calls Foo.Go"}, + wantNot: []string{"E Svc.Run calls Bar.Go"}, + }, + } +} diff --git a/internal/invariance/cases_n_test.go b/internal/invariance/cases_n_test.go new file mode 100644 index 0000000..85593dd --- /dev/null +++ b/internal/invariance/cases_n_test.go @@ -0,0 +1,306 @@ +package invariance + +// Group N: grove internal/native semantic passes read comments and strings +// as type references / clauses. The bugs live in the native layer, so the +// "native" mode is the one that proves them; "astkit" mode asserts the same +// facts hold without native (they must: comments are never code). + +const phpComposer = `{"autoload":{"psr-4":{"App\\":"src/"}}}` + +func casesN() []twinCase { + return []twinCase{ + { + id: "N1-go-comment-types", + clean: map[string]string{"go.mod": goMod, "p.go": `package p + +type Widget struct{} +type Gadget struct{} + +func Use() int { + return 1 +} +`}, + decoy: map[string]string{"go.mod": goMod, "p.go": `package p + +type Widget struct{} +type Gadget struct{} + +func Use() int { // returns a Widget count; Gadget is unused here + return 1 +} +`}, + want: []string{"S Use"}, + wantNot: []string{"E Use uses-type Widget", "E Use uses-type Gadget"}, + }, + { + id: "N1-java-comment-types", + clean: one("src/p/A.java", `package p; +class Fake {} +class Other {} +public class A { + int m() { + return 1; + } +} +`), + decoy: one("src/p/A.java", `package p; +class Fake {} +class Other {} +public class A { + int m() { + // keep Fake around; new Other() is created lazily + return 1; + } +} +`), + want: []string{"S A.m"}, + wantNot: []string{"E A.m uses-type Fake", "E A.m uses-type Other", "E A.m calls Other"}, + }, + { + id: "N1-csharp-comment-types", + clean: one("A.cs", `namespace P { +class Fake {} +public class A { + int M() { + return 1; + } +} +} +`), + decoy: one("A.cs", `namespace P { +class Fake {} +public class A { + int M() { + // TODO: new Fake() later + return 1; + } +} +} +`), + want: []string{"S A.M"}, + wantNot: []string{"E A.M uses-type Fake", "E A.M calls Fake"}, + }, + { + id: "N1-php-comment-types", + clean: map[string]string{"composer.json": phpComposer, + "src/Loggable.php": " i32 { + let msg = "error: missing"; + msg.len() as i32 +} +`}, + decoy: map[string]string{ + "Cargo.toml": "[package]\nname = \"p\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + "src/lib.rs": `pub struct Config; +pub trait Shape {} +pub struct Wrapper; + +pub fn load() -> i32 { + // e.g. impl Shape for Wrapper {} + let msg = "error: Config missing"; + msg.len() as i32 +} +`}, + want: []string{"S load"}, + wantNot: []string{"E load uses-type Config", "E Wrapper implements Shape"}, + }, + { + id: "N8-c-comment-include-and-type", + clean: map[string]string{ + "dead.h": "int dead(void);\n", + "widget.h": "typedef struct Widget { int x; } Widget;\n", + "a.c": `#include "widget.h" +int use(int arg) { + return arg; +} +`}, + decoy: map[string]string{ + "dead.h": "int dead(void);\n", + "widget.h": "typedef struct Widget { int x; } Widget;\n", + "a.c": `#include "widget.h" +/* +#include "dead.h" +*/ +int use(int arg) { + // Widget(arg) is legacy + return arg; +} +`}, + want: []string{"S use"}, + wantNative: []string{"E file:a.c imports file:widget.h"}, + wantNot: []string{"E file:a.c imports file:dead.h", "E use uses-type Widget", "E use calls Widget"}, + }, + } +} diff --git a/internal/invariance/cases_p_test.go b/internal/invariance/cases_p_test.go new file mode 100644 index 0000000..04086c2 --- /dev/null +++ b/internal/invariance/cases_p_test.go @@ -0,0 +1,468 @@ +package invariance + +// Group P: grove internal/parser. P6-P15 put a syntax error in BOTH forms so +// the file takes the regex recovery path (extractSymbols merges regex-found +// names the AST missed); the decoy hides code-looking text in comments or +// strings that recovery must not read. + +func casesP() []twinCase { + return []twinCase{ + // --- P1 precedingCommentBlock docstring bleed --- + { + id: "P1-java-trailing-comment", + clean: one("A.java", `/** The A class. */ +class A { + int x; + void b() {} +} +`), + decoy: one("A.java", `/** The A class. */ +class A { + int x; /* trailing */ + void b() {} +} +`), + doc: true, + want: []string{"S A.b"}, + wantNot: []string{"S A.b doc~int_x", "S A.b doc~trailing", "S A.b doc~class_A"}, + }, + { + id: "P1-js-glob-string", + clean: one("a.js", `const glob = "src"; +/* helper */ +function f() {} +`), + decoy: one("a.js", `const glob = "src/**"; +/* helper */ +function f() {} +`), + doc: true, + want: []string{"S f"}, + wantNot: []string{"S f doc~const_glob"}, + }, + { + id: "P1-go-trailing-comment", + clean: one("a.go", `/** Package p. */ +package p + +var x = 1 + +func F() {} +`), + decoy: one("a.go", `/** Package p. */ +package p + +var x = 1 /* trailing */ +func F() {} +`), + doc: true, + want: []string{"S F"}, + wantNot: []string{"S F doc~var_x", "S F doc~trailing", "S F doc~package"}, + }, + // --- P2-P5 clean files --- + { + id: "P2-cpp-raw-string", + clean: one("a.cpp", `const char* kSrc = ""; +int real(int y) { return y; } +`), + decoy: one("a.cpp", `const char* kSrc = R"( +int phantom(int x) { + return x; +} +)"; +int real(int y) { return y; } +`), + want: []string{"S real"}, + wantNot: []string{"S phantom"}, + }, + { + id: "P3-c-multiline-define", + clean: one("a.c", `#define DECL(n) (n) +int real(void) { return 0; } +`), + decoy: one("a.c", `#define DECL(n) \ + static int helper(void) { \ + return n; \ + } +int real(void) { return 0; } +`), + want: []string{"S real"}, + wantNot: []string{"S helper"}, + }, + { + id: "P4-cpp-digit-separator", + clean: one("a.cpp", `namespace foo { struct Widget { int go() { return 1; } }; } +namespace other { struct Widget { int go() { return 2; } }; } +constexpr int kMax = 1000; +using namespace foo; +int work() { Widget w; return w.go(); } +`), + decoy: one("a.cpp", `namespace foo { struct Widget { int go() { return 1; } }; } +namespace other { struct Widget { int go() { return 2; } }; } +constexpr int kMax = 1'000; +using namespace foo; +int work() { Widget w; return w.go(); } +`), + want: []string{"E work calls foo.Widget.go"}, + wantNot: []string{"E work calls other.Widget.go"}, + }, + { + id: "P4-cpp-error-apostrophe", + clean: one("a.cpp", `namespace foo { struct Widget { int go() { return 1; } }; } +namespace other { struct Widget { int go() { return 2; } }; } +#if 0 +#error do not +#endif +using namespace foo; +int work() { Widget w; return w.go(); } +`), + decoy: one("a.cpp", `namespace foo { struct Widget { int go() { return 1; } }; } +namespace other { struct Widget { int go() { return 2; } }; } +#if 0 +#error don't +#endif +using namespace foo; +int work() { Widget w; return w.go(); } +`), + want: []string{"E work calls foo.Widget.go"}, + wantNot: []string{"E work calls other.Widget.go"}, + }, + { + id: "P5-js-export-default-string", + clean: one("a.js", `function helper() { return "x"; } +export function other() {} +`), + decoy: one("a.js", `function helper() { return "export default"; } +export function other() {} +`), + want: []string{"S helper unexported", "S other exported"}, + wantNot: []string{"S helper mod=default-export", "S helper mod=default", "S helper exported"}, + }, + // --- P6-P14 regex recovery (syntax error in both forms) --- + { + id: "P6-go-recovery", + clean: one("a.go", `package p + +var s = "" + +func Real() int { return 1 } + +func Broken( { +`), + decoy: one("a.go", `package p + +/* +func Phantom() {} +type Ghost struct{} +*/ +var s = `+"`"+` +func Phantom2() {} +type Ghost2 struct{} +`+"`"+` + +func Real() int { return 1 } + +func Broken( { +`), + want: []string{"S Real"}, + wantNot: []string{"S Phantom", "S Ghost", "S Phantom2", "S Ghost2"}, + }, + { + id: "P7-java-text-block-recovery", + clean: one("A.java", `class A { + String s = ""; + void real() {} + void broken( { +} +`), + decoy: one("A.java", `class A { + String s = """ + class Phantom { void ghost() {} } + """; + void real() {} + void broken( { +} +`), + want: []string{"S A.real"}, + wantNot: []string{"S Phantom", "S ghost"}, + }, + { + id: "P8-csharp-raw-verbatim-recovery", + clean: one("A.cs", `class A { + string s = ""; + string v = ""; + void Real() {} + void Broken( { +} +`), + decoy: one("A.cs", `class A { + string s = """ + class Phantom { } + """; + string v = @" +class Phantom2 { } +"; + void Real() {} + void Broken( { +} +`), + want: []string{"S A.Real"}, + wantNot: []string{"S Phantom", "S Phantom2"}, + }, + { + id: "P9-ts-template-recovery", + clean: one("a.ts", `const tpl = ""; +export function real() { return 1; } +export function broken( { +`), + decoy: one("a.ts", "const tpl = `\nclass Phantom {\nfunction ghost(a) {\n`;\nexport function real() { return 1; }\nexport function broken( {\n"), + want: []string{"S real"}, + wantNot: []string{"S Phantom", "S ghost"}, + }, + { + id: "P9-tsx-jsx-text-recovery", + clean: one("v.tsx", `export function View() { + return
+ styles are applied +
; +} +export function broken( { +`), + decoy: one("v.tsx", `export function View() { + return
+ class names are applied +
; +} +export function broken( { +`), + want: []string{"S View"}, + wantNot: []string{"S names"}, + }, + { + id: "P10-php-heredoc-html-recovery", + clean: one("a.php", ` +

+ styles here +

+`), + decoy: one("a.php", ` +

+ class names here +

+`), + want: []string{"S real"}, + wantNot: []string{"S Phantom", "S ghost", "S names"}, + }, + { + id: "P11-rust-raw-string-recovery", + clean: one("lib.rs", `const S: &str = ""; +pub fn real() -> i32 { 1 } +pub fn broken( { +`), + decoy: one("lib.rs", `const S: &str = r#" +fn ghost() {} +struct Phantom; +"#; +pub fn real() -> i32 { 1 } +pub fn broken( { +`), + want: []string{"S real"}, + wantNot: []string{"S ghost", "S Phantom"}, + }, + { + id: "P12-python-continued-string-recovery", + clean: one("a.py", `x = "abc" + + +def real(): + return 1 + + +def broken(: + pass +`), + decoy: one("a.py", `x = "abc \ +class Phantom:" + + +def real(): + return 1 + + +def broken(: + pass +`), + want: []string{"S real"}, + wantNot: []string{"S Phantom"}, + }, + { + id: "P13-c-if0", + clean: one("a.c", `int real(void) { return 0; } +int broken( { +`), + decoy: one("a.c", `#if 0 +int ghost(int a) { +#endif +int real(void) { return 0; } +int broken( { +`), + want: []string{"S real"}, + wantNot: []string{"S ghost"}, + }, + { + id: "P13-cpp-if0-namespace", + clean: one("a.cpp", `int real(void) { return 0; } +int broken( { +`), + decoy: one("a.cpp", `#if 0 +namespace legacy { +int ghost(int a) { +#endif +int real(void) { return 0; } +int broken( { +`), + want: []string{"S real"}, + wantNot: []string{"S legacy", "S ghost", "S legacy.real"}, + }, + { + id: "P13-csharp-if-false", + clean: one("A.cs", `public class Real { public void Go() {} } +public class Broken { void X( { } +`), + decoy: one("A.cs", `#if false +public class Legacy { +#endif +public class Real { public void Go() {} } +public class Broken { void X( { } +`), + want: []string{"S Real", "S Real.Go"}, + wantNot: []string{"S Legacy", "S Legacy.Real"}, + }, + { + id: "P14-cpp-comment-qualified-method", + clean: one("a.cpp", `int broken( { +void helper(int x) { + (void)x; +} +`), + decoy: one("a.cpp", `int broken( { +void helper(int x) { // see Widget::draw(x) + (void)x; +} +`), + want: []string{"S helper"}, + wantNot: []string{"S Widget.draw", "S draw"}, + }, + // --- P15 recovered spans: the symbol follows the syntax error (so + // recovery, not the AST, supplies it); decoys keep line numbers and + // spans are compared --- + { + id: "P15-java-span-block-comment-brace", + clean: one("A.java", `class A { + void broken( { + void real() { + int a = 0; /* x */ + a++; + } + void after() {} +} +`), + decoy: one("A.java", `class A { + void broken( { + void real() { + int a = 0; /* } */ + a++; + } + void after() {} +} +`), + spans: true, + want: []string{"S real end=6"}, + }, + { + id: "P15-ts-span-template-brace", + clean: one("a.ts", "export function broken( {\nexport function real() {\n const s = `\nx\n`;\n return s;\n}\nexport function after() {}\n"), + decoy: one("a.ts", "export function broken( {\nexport function real() {\n const s = `\n}\n`;\n return s;\n}\nexport function after() {}\n"), + spans: true, + want: []string{"S real end=7"}, + }, + { + id: "P15-python-span-docstring-col0", + clean: one("a.py", `x = foo( + + +def real(): + """Doc. + end + """ + return 1 + + +def after(): + return 2 +`), + decoy: one("a.py", `x = foo( + + +def real(): + """Doc. +end +""" + return 1 + + +def after(): + return 2 +`), + spans: true, + want: []string{"S real start=4", "S real doc~end"}, + }, + { + // TS overload widening: a call statement above a function was + // taken as an overload signature and widened the span upward. + // Not a comment/string twin: the clean form has a blank line + // where the decoy has the call (same line numbers), and makes + // the same call after the function, so both forms carry the + // real -> foo call. + id: "P16-ts-overload-widening", + clean: one("a.ts", ` +function foo(x: number) { + return x; +} +foo(1); +`), + decoy: one("a.ts", `foo(1); +function foo(x: number) { + return x; +} +`), + want: []string{"S foo start=2"}, + }, + { + id: "P17-c-header-namespace-comment", + clean: one("a.h", `/* + * Shared declarations. + */ +int real(void); +`), + decoy: one("a.h", `/* +namespace handling is not supported here + */ +int real(void); +`), + want: []string{"S real lang=c"}, + wantNot: []string{"S real lang=cpp"}, + }, + } +} diff --git a/internal/invariance/twin_test.go b/internal/invariance/twin_test.go new file mode 100644 index 0000000..1d393e7 --- /dev/null +++ b/internal/invariance/twin_test.go @@ -0,0 +1,531 @@ +// Package invariance holds metamorphic regression tests for grove's code +// graph: a comment, string literal, docstring, or formatting change must not +// change the graph. +// +// Each twin case is a tiny repository in two forms. CLEAN carries no +// comment/string trap; DECOY differs only by comments, strings, docstrings, +// or layout that a correct indexer ignores. Both are indexed end-to-end +// through the public engine (pkg/grove: parser -> store -> edges -> native +// merge), normalized (symbol IDs carry blob SHAs and lines, so endpoints are +// rewritten to file::qualifiedName#kind), and compared. Every case also +// lists want/wantNot facts so equality alone cannot pass a case where both +// forms are equally wrong. +// +// Cases are named by the comment/string-as-code audit IDs (2026-10-10). +// They were written against branch fix/comment-string-masking before the +// fixes landed: most fail there by design. +// +// Native analyzers: every case runs twice, mode "astkit" (native disabled +// via grove.Config.NativeAnalyzers=false) and mode "native" (enabled). The +// native mode is skipped when no native analyzer completed a pass for the +// fixture (toolchain missing on this machine, e.g. no dotnet for C#), which +// is read from IndexResult.Native diagnostics. +// +// Run: go test ./internal/invariance/ -run TestCommentStringInvariance +// One case: -run 'TestCommentStringInvariance/astkit/L1-python' +package invariance + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "testing" + + "github.com/provasign/grove/pkg/grove" +) + +// twinCase is one audit finding as a metamorphic pair. +type twinCase struct { + id string // audit ID + short tag, used as the subtest name + clean map[string]string // repo-relative path -> content + decoy map[string]string // same paths; differs only in comments/strings/layout + // want / wantNot are facts checked on BOTH forms (see holds): + // "E " an edge; src/dst match a symbol's + // qualified name, Parent.Name, or bare name (no dot), optionally + // suffixed "@"; "file:" for file nodes; "*" for any + // "S [kind=K] [mod=M] [exported] [unexported] [lang=L] + // [doc~TEXT] [start=N] [end=N]" a symbol with all those properties + // (TEXT uses "_" for spaces) + want, wantNot []string + // wantNative are facts only a native analyzer produces (e.g. uses-type + // from a local variable declaration); checked in native mode only. + wantNative []string + doc bool // compare docstrings + sig bool // compare signatures (whitespace/comments collapsed) + spans bool // compare spans (decoy must keep line numbers) + // rawAnnotations: the decoy edits an attribute/decorator argument, whose + // raw text annotations carry verbatim, so annotations are not compared. + rawAnnotations bool +} + +func allTwinCases() []twinCase { + var all []twinCase + all = append(all, casesP()...) + all = append(all, casesL()...) + all = append(all, casesN()...) + all = append(all, casesA()...) + all = append(all, casesE()...) + all = append(all, casesC()...) + all = append(all, casesJ()...) + return all +} + +func TestCommentStringInvariance(t *testing.T) { + cases := allTwinCases() + seen := map[string]bool{} + for _, tc := range cases { + if seen[tc.id] { + t.Fatalf("duplicate case id %s", tc.id) + } + seen[tc.id] = true + } + for _, mode := range []string{"astkit", "native"} { + mode := mode + t.Run(mode, func(t *testing.T) { + if mode == "native" && testing.Short() && runtime.GOOS == "windows" { + // Native analyzers start compilers per fixture, which Windows + // runners do slowly; Linux and macOS cover this mode in -short. + t.Skip("native mode skipped in -short on Windows") + } + for _, tc := range cases { + tc := tc + t.Run(tc.id, func(t *testing.T) { + // Each case indexes its own temp repos twice; serially + // the table exceeded CI's 2-minute budget on Windows. + // Native cases stay serial: concurrent javac/go runs + // starved other packages' compiler-timeout tests. + if mode == "astkit" { + t.Parallel() + } + runTwin(t, tc, mode == "native") + }) + } + }) + } +} + +func runTwin(t *testing.T, tc twinCase, nativeOn bool) { + clean, cleanDiag := indexFixture(t, tc.clean, nativeOn) + decoy, decoyDiag := indexFixture(t, tc.decoy, nativeOn) + if nativeOn && !nativeRan(cleanDiag) { + t.Skipf("no native analyzer completed a pass for this fixture here: %v", cleanDiag) + } + + // 1. The right answer, on both forms. + for _, form := range []struct { + name string + g *view + diag []string + }{{"clean", clean, cleanDiag}, {"decoy", decoy, decoyDiag}} { + wants := tc.want + if nativeOn { + wants = append(append([]string(nil), wants...), tc.wantNative...) + } + for _, f := range wants { + if !form.g.holds(t, f) { + t.Errorf("[%s] want %q: not found\n%s", form.name, f, form.g.dump(f)) + } + } + for _, f := range tc.wantNot { + if form.g.holds(t, f) { + t.Errorf("[%s] wantNot %q: present\n%s", form.name, f, form.g.dump(f)) + } + } + } + + // 2. Metamorphic equality. + cs, ds := clean.symbolKeys(tc), decoy.symbolKeys(tc) + if extra, missing := setDiff(ds, cs); len(extra)+len(missing) > 0 { + t.Errorf("symbols differ (decoy vs clean)\n only in decoy: %s\n only in clean: %s", + strings.Join(extra, "\n "), strings.Join(missing, "\n ")) + } + ce, de := clean.edgeKeys(), decoy.edgeKeys() + if extra, missing := setDiff(de, ce); len(extra)+len(missing) > 0 { + t.Errorf("edges differ (decoy vs clean)\n only in decoy: %s\n only in clean: %s", + strings.Join(decoy.annotate(extra), "\n "), + strings.Join(clean.annotate(missing), "\n ")) + } + if t.Failed() && nativeOn { + t.Logf("native diagnostics (decoy): %v", decoyDiag) + } +} + +func nativeRan(diags []string) bool { + for _, d := range diags { + if strings.Contains(d, ": full pass over ") { + return true + } + } + return false +} + +// indexFixture writes files to a fresh root and indexes it through the +// public engine. +func indexFixture(t *testing.T, files map[string]string, nativeOn bool) (*view, []string) { + t.Helper() + ctx := context.Background() + root := t.TempDir() + if nativeOn { + files = withNativeConfig(t, root, files) + } + for rel, body := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + on := nativeOn + eng, err := grove.Open(ctx, grove.Config{RepoRoot: root, NativeAnalyzers: &on, OneShot: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { eng.Close() }) + res, err := eng.Index(ctx, "") + if err != nil { + t.Fatal(err) + } + syms, edges, err := eng.SnapshotGraph(ctx) + if err != nil { + t.Fatal(err) + } + return newView(syms, edges), res.Native +} + +// withNativeConfig adds the project files each native analyzer requires +// before it runs (identical for both forms of a twin, so the comparison is +// unaffected). TypeScript is resolved from GROVE_TEST_TYPESCRIPT (a +// node_modules/typescript dir), as in internal/index's compiler tests. +func withNativeConfig(t *testing.T, root string, files map[string]string) map[string]string { + t.Helper() + exts := map[string]bool{} + var cfiles []string + for rel := range files { + ext := strings.ToLower(filepath.Ext(rel)) + exts[ext] = true + if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".m" { + cfiles = append(cfiles, rel) + } + } + sort.Strings(cfiles) + add := func(name, body string) { + if _, ok := files[name]; !ok { + files = with(files, name, body) + } + } + if exts[".go"] { + add("go.mod", goMod) + } + if exts[".py"] { + add("pyproject.toml", "[project]\nname = \"p\"\nversion = \"0.1.0\"\n") + } + if exts[".php"] { + add("composer.json", `{"autoload":{"psr-4":{"App\\":"src/"}}}`) + } + if exts[".cs"] { + add("p.csproj", `net8.0`) + } + if exts[".rs"] { + add("Cargo.toml", "[package]\nname = \"p\"\nversion = \"0.1.0\"\nedition = \"2021\"\n") + } + if exts[".c"] || exts[".h"] || exts[".cpp"] || exts[".hpp"] || exts[".cc"] { + var cmds []string + for _, f := range cfiles { + cmds = append(cmds, fmt.Sprintf(`{"directory":%q,"file":%q,"arguments":["cc","-c",%q]}`, root, f, f)) + } + add("compile_commands.json", "["+strings.Join(cmds, ",")+"]") + } + if exts[".ts"] || exts[".tsx"] || exts[".js"] || exts[".jsx"] { + add("tsconfig.json", `{"compilerOptions":{"allowJs":true,"checkJs":false,"jsx":"react","target":"es2020","strict":false},"include":["**/*"]}`) + if ts := os.Getenv("GROVE_TEST_TYPESCRIPT"); ts != "" { + if err := os.MkdirAll(filepath.Join(root, "node_modules"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(ts, filepath.Join(root, "node_modules", "typescript")); err != nil { + t.Fatal(err) + } + } + } + return files +} + +// view is a normalized graph. +type view struct { + syms []grove.Symbol + byID map[string]*grove.Symbol + edges []grove.Edge +} + +func newView(syms []grove.Symbol, edges []grove.Edge) *view { + v := &view{syms: syms, byID: map[string]*grove.Symbol{}, edges: edges} + for i := range v.syms { + v.byID[v.syms[i].ID] = &v.syms[i] + } + return v +} + +// node renders an edge endpoint without blob SHAs or line numbers. +func (v *view) node(id string) string { + if s := v.byID[id]; s != nil { + return fmt.Sprintf("%s::%s#%s", s.FilePath, s.QualifiedName, s.Kind) + } + if i := strings.LastIndexByte(id, '@'); i >= 0 && strings.Contains(id, "::") { + return id[:i] + "@?" + } + return id +} + +var ( + reBlockComment = regexp.MustCompile(`(?s)/\*.*?\*/`) + reLineComment = regexp.MustCompile(`(?m)(//|#).*$`) + reSpace = regexp.MustCompile(`\s+`) +) + +func collapseSig(s string) string { + s = reBlockComment.ReplaceAllString(s, " ") + s = reLineComment.ReplaceAllString(s, " ") + return reSpace.ReplaceAllString(s, "") +} + +func (v *view) symbolKeys(tc twinCase) []string { + var out []string + for _, s := range v.syms { + mods := append([]string(nil), s.Modifiers...) + sort.Strings(mods) + var anns []string + if !tc.rawAnnotations { + anns = append(anns, s.Annotations...) + sort.Strings(anns) + } + k := fmt.Sprintf("%s|%s|%s|%s|parent=%s|exported=%v|mods=%s|annotations=%s", + s.FilePath, s.Language, s.Kind, s.QualifiedName, s.ParentSymbol, s.Exports, strings.Join(mods, ","), strings.Join(anns, ",")) + if tc.doc { + k += "|doc=" + strconv.Quote(strings.TrimSpace(s.Docstring)) + } + if tc.sig { + k += "|sig=" + strconv.Quote(collapseSig(s.Signature)) + } + if tc.spans { + k += fmt.Sprintf("|span=%d-%d", s.Span.Start, s.Span.End) + } + out = append(out, k) + } + return out +} + +func (v *view) edgeKeys() []string { + var out []string + for _, e := range v.edges { + out = append(out, fmt.Sprintf("%s -%s-> %s", v.node(e.From), e.Type, v.node(e.To))) + } + return out +} + +// annotate appends confidence/source/reason (ignored by the comparison) to +// edge keys for failure output. +func (v *view) annotate(keys []string) []string { + info := map[string][]string{} + for _, e := range v.edges { + k := fmt.Sprintf("%s -%s-> %s", v.node(e.From), e.Type, v.node(e.To)) + info[k] = append(info[k], fmt.Sprintf("conf=%.2f src=%s reason=%s", e.Confidence, e.Source, e.Reason)) + } + out := make([]string, len(keys)) + for i, k := range keys { + out[i] = k + " [" + strings.Join(info[k], "; ") + "]" + } + return out +} + +func setDiff(a, b []string) (onlyA, onlyB []string) { + ma, mb := map[string]bool{}, map[string]bool{} + for _, x := range a { + ma[x] = true + } + for _, x := range b { + mb[x] = true + } + for x := range ma { + if !mb[x] { + onlyA = append(onlyA, x) + } + } + for x := range mb { + if !ma[x] { + onlyB = append(onlyB, x) + } + } + sort.Strings(onlyA) + sort.Strings(onlyB) + return +} + +// symMatches reports whether a symbol answers to pattern p ("name", +// "Parent.name", a qualified name, or any of those + "@" + file path). +func symMatches(s *grove.Symbol, p string) bool { + if p == "*" { + return true + } + if name, file, ok := strings.Cut(p, "@"); ok { + if s.FilePath != file { + return false + } + p = name + } + if s.QualifiedName == p || s.Name == p && !strings.ContainsAny(p, ".:") { + return true + } + if s.ParentSymbol != "" { + parent := s.ParentSymbol + if i := strings.LastIndexAny(parent, ".:"); i >= 0 { + parent = parent[i+1:] + } + if parent+"."+s.Name == p || s.ParentSymbol+"."+s.Name == p { + return true + } + } + // C++/Rust/PHP qualified forms written with "." in the pattern. + return strings.NewReplacer("::", ".", "\\", ".").Replace(s.QualifiedName) == p +} + +func (v *view) nodeMatches(id, p string) bool { + if p == "*" { + return true + } + if strings.HasPrefix(p, "file:") { + return id == p + } + if s := v.byID[id]; s != nil { + return symMatches(s, p) + } + return false +} + +func (v *view) holds(t *testing.T, fact string) bool { + t.Helper() + f := strings.Fields(fact) + switch { + case len(f) == 4 && f[0] == "E": + for _, e := range v.edges { + if string(e.Type) == f[2] && v.nodeMatches(e.From, f[1]) && v.nodeMatches(e.To, f[3]) { + return true + } + } + return false + case len(f) >= 2 && f[0] == "S": + for i := range v.syms { + if v.symHolds(t, &v.syms[i], f[1], f[2:]) { + return true + } + } + return false + } + t.Fatalf("bad fact %q", fact) + return false +} + +func (v *view) symHolds(t *testing.T, s *grove.Symbol, name string, props []string) bool { + if !symMatches(s, name) { + return false + } + for _, p := range props { + key, val, _ := strings.Cut(p, "=") + if strings.Contains(p, "~") { + key, val, _ = strings.Cut(p, "~") + } + switch key { + case "kind": + if string(s.Kind) != val { + return false + } + case "mod": + has := false + for _, m := range s.Modifiers { + has = has || m == val + } + if !has { + return false + } + case "exported": + if !s.Exports { + return false + } + case "unexported": + if s.Exports { + return false + } + case "lang": + if s.Language != val { + return false + } + case "doc": + if !strings.Contains(s.Docstring, strings.ReplaceAll(val, "_", " ")) { + return false + } + case "start", "end": + n, err := strconv.Atoi(val) + if err != nil { + t.Fatalf("bad %s", p) + } + if key == "start" && s.Span.Start != n || key == "end" && s.Span.End != n { + return false + } + case "file": + if s.FilePath != val { + return false + } + default: + t.Fatalf("bad symbol property %q", p) + } + } + return true +} + +// dump prints the graph facts near a failed check: symbols and edges whose +// rendering mentions any name in the fact. +func (v *view) dump(fact string) string { + var names []string + for _, w := range strings.Fields(fact)[1:] { + if w == "*" || strings.ContainsAny(w, "=~") { + continue + } + if i := strings.LastIndexAny(w, ".:"); i >= 0 && !strings.HasPrefix(w, "file:") { + names = append(names, w[i+1:]) + } else { + names = append(names, strings.TrimPrefix(w, "file:")) + } + } + mention := func(s string) bool { + for _, n := range names { + if n != "" && strings.Contains(s, n) { + return true + } + } + return false + } + var b strings.Builder + for _, s := range v.syms { + line := fmt.Sprintf(" sym %s %s %s parent=%q exported=%v mods=%v annotations=%v span=%d-%d lang=%s doc=%q", + s.FilePath, s.Kind, s.QualifiedName, s.ParentSymbol, s.Exports, s.Modifiers, s.Annotations, s.Span.Start, s.Span.End, s.Language, s.Docstring) + if mention(s.QualifiedName) { + b.WriteString(line + "\n") + } + } + var es []string + for _, e := range v.edges { + k := fmt.Sprintf(" edge %s -%s-> %s [conf=%.2f src=%s reason=%s]", v.node(e.From), e.Type, v.node(e.To), e.Confidence, e.Source, e.Reason) + if e.Type != "contains" && e.Type != "defines" && mention(k) { + es = append(es, k) + } + } + sort.Strings(es) + b.WriteString(strings.Join(es, "\n")) + return b.String() +} diff --git a/internal/native/cfamily.go b/internal/native/cfamily.go index 24884d8..c95f853 100644 --- a/internal/native/cfamily.go +++ b/internal/native/cfamily.go @@ -7,7 +7,9 @@ import ( "regexp" "strings" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" + "github.com/provasign/grove/internal/parser" ) type cFamilyAnalyzer struct{} @@ -49,7 +51,7 @@ func (cFamilyAnalyzer) Analyze(ctx context.Context, req Request) Result { if err != nil { continue } - for _, inc := range cIncludes(string(content)) { + for _, inc := range cIncludes(cIncludeScanText(file, string(content))) { if target, ok := resolveCInclude(req.Root, file, inc, includeDirs, fileScope); ok { edges = append(edges, nativeImportEdge(file, target, 0.95)) includeTargets[file] = append(includeTargets[file], target) @@ -118,6 +120,18 @@ func cIncludeDirs(root string, commands []compileCommand) []string { var cIncludePattern = regexp.MustCompile(`(?m)^\s*#\s*include\s+["<]([^">]+)[">]`) +// cIncludeScanText is file content prepared for cIncludes: comments and +// `#if 0` branches masked, string literals kept (the `"x.h"` path is one). +func cIncludeScanText(file, content string) string { + lang := parser.DetectLanguage(file) + if lang != "c" && lang != "cpp" { + lang = "cpp" + } + return textmask.MaskComments(lang, textmask.MaskInactivePreprocessor(lang, content)) +} + +// cIncludes returns the include paths of content, which must already have +// comments and inactive preprocessor branches masked (cIncludeScanText). func cIncludes(content string) []string { matches := cIncludePattern.FindAllStringSubmatch(content, -1) out := make([]string, 0, len(matches)) @@ -212,6 +226,7 @@ func cFamilySemanticEdges(symbols []core.SymbolRecord, includeTargets map[string if (caller.Language != "c" && caller.Language != "cpp") || caller.RawText == "" || !callableKind(caller.Kind) { continue } + masked := maskCode(caller.Language, caller.RawText) // Call edges intentionally NOT emitted here (same lesson as the Java, // Rust, C#, and PHP native passes): text matching edged every // same-named callable it saw. The graph layer's call-site resolution @@ -223,12 +238,12 @@ func cFamilySemanticEdges(symbols []core.SymbolRecord, includeTargets map[string if target.ID == caller.ID { continue } - if typeKind(target.Kind) && cFamilyContainsType(caller, target, file != caller.FilePath) { + if typeKind(target.Kind) && cFamilyContainsType(caller, masked, target, file != caller.FilePath) { add(symbolEdge(caller, target, core.EdgeUsesType, 0.91)) } } } - for _, typeName := range cFamilyConstructedTypes(caller.RawText) { + for _, typeName := range cFamilyConstructedTypes(masked) { if target, ok := cFamilyBestType(idx, typeName, caller); ok && target.ID != caller.ID { add(symbolEdge(caller, target, core.EdgeUsesType, 0.93)) } @@ -249,21 +264,9 @@ func cFamilyScopeFiles(fromFile string, includeTargets map[string][]string) []st return out } -func cFamilyContainsCallable(rawText string, target core.SymbolRecord, crossFile bool) bool { - if !crossFile { - return containsCall(rawText, target.Name) - } - if target.ParentSymbol != "" { - pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(target.ParentSymbol) + `::` + regexp.QuoteMeta(target.Name) + `\s*\(`) - if pattern.MatchString(stripQuotedText(rawText)) { - return true - } - } - return containsCall(rawText, target.Name) -} - -func cFamilyContainsType(caller, target core.SymbolRecord, crossFile bool) bool { - clean := stripCStyleComments(stripQuotedText(caller.RawText)) +// cFamilyContainsType reports whether the caller body names target. clean +// is caller.RawText through maskCode. +func cFamilyContainsType(caller core.SymbolRecord, clean string, target core.SymbolRecord, crossFile bool) bool { if strings.Contains(target.QualifiedName, "::") { pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(target.QualifiedName) + `\b`) if pattern.MatchString(clean) { @@ -303,57 +306,12 @@ func cFamilyContainsType(caller, target core.SymbolRecord, crossFile bool) bool return containsTypeToken(clean, target.Name) } -func stripCStyleComments(text string) string { - var out strings.Builder - out.Grow(len(text)) - for i := 0; i < len(text); { - if i+1 < len(text) && text[i] == '/' && text[i+1] == '/' { - for i < len(text) && text[i] != '\n' { - i++ - } - continue - } - if i+1 < len(text) && text[i] == '/' && text[i+1] == '*' { - i += 2 - for i+1 < len(text) && !(text[i] == '*' && text[i+1] == '/') { - if text[i] == '\n' { - out.WriteByte('\n') - } - i++ - } - if i+1 < len(text) { - i += 2 - } - continue - } - out.WriteByte(text[i]) - i++ - } - return out.String() -} - -type cFamilyQualifiedCall struct { - Qualifier string - Method string -} - -var cFamilyQualifiedCallPattern = regexp.MustCompile(`\b([A-Z_][A-Za-z0-9_]*)::([A-Za-z_][A-Za-z0-9_]*)\s*\(`) - -func cFamilyQualifiedCalls(rawText string) []cFamilyQualifiedCall { - matches := cFamilyQualifiedCallPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) - out := make([]cFamilyQualifiedCall, 0, len(matches)) - for _, match := range matches { - if len(match) == 3 { - out = append(out, cFamilyQualifiedCall{Qualifier: match[1], Method: match[2]}) - } - } - return out -} - var cFamilyConstructorPattern = regexp.MustCompile(`\b(?:new\s+)?([A-Z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*)\s*\(`) -func cFamilyConstructedTypes(rawText string) []string { - matches := cFamilyConstructorPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) +// cFamilyConstructedTypes returns the constructor-call type names of masked +// (see maskCode). +func cFamilyConstructedTypes(masked string) []string { + matches := cFamilyConstructorPattern.FindAllStringSubmatch(masked, -1) seen := map[string]bool{} var out []string for _, match := range matches { diff --git a/internal/native/csharp.go b/internal/native/csharp.go index f5c5687..67b9d46 100644 --- a/internal/native/csharp.go +++ b/internal/native/csharp.go @@ -162,7 +162,7 @@ func csharpSemanticEdges(symbols []core.SymbolRecord) []core.Edge { continue } if typeKind(symbol.Kind) { - for _, ref := range csharpInheritanceRefs(symbol.Signature+"\n"+firstLine(symbol.RawText), symbol.Kind) { + for _, ref := range csharpInheritanceRefs(maskedDeclHeader("csharp", symbol), symbol.Kind) { if target, ok := csharpBestType(idx, ref.Name, symbol.FilePath); ok && target.ID != symbol.ID { edgeType := ref.EdgeType if symbol.Kind != core.KindInterface { @@ -185,24 +185,21 @@ func csharpSemanticEdges(symbols []core.SymbolRecord) []core.Edge { // libraries (Newtonsoft: P 0.20). The graph layer's call-site // resolution owns calls; this pass keeps only the type-usage // evidence text matching is still reliable for. - for _, className := range csharpConstructedTypes(symbol.RawText) { + masked := maskCode("csharp", symbol.RawText) + for _, className := range csharpConstructedTypes(masked) { if target, ok := csharpBestType(idx, className, symbol.FilePath); ok && target.ID != symbol.ID { add(symbolEdge(symbol, target, core.EdgeUsesType, 0.95)) } } names := make([]string, 0, 8) - for t := range typeTokensIn(symbol.RawText) { + for t := range typeTokensIn(masked) { if _, ok := idx.typesByName[t]; ok { names = append(names, t) } } sort.Strings(names) - stripped := "" - if len(slowNames) > 0 { - stripped = stripQuotedText(symbol.RawText) - } for _, name := range slowNames { - if containsTypeTokenStripped(stripped, name) { + if containsTypeToken(masked, name) { names = append(names, name) } } @@ -220,12 +217,11 @@ type csharpInheritanceRef struct { EdgeType core.EdgeType } -// The base-list capture excludes newlines ([ \t], not \s): the caller feeds -// `Signature + "\n" + firstLine(RawText)`, and a \s here swallowed the join -// and ran into the duplicated second line, producing a garbage name that -// resolved to nothing — so C# extends/implements resolution emitted zero -// edges on real code. `{` is also excluded so a same-line body brace can't -// leak in. +// csharpDeclHeadPattern runs on maskedDeclHeader output (one line, comments +// and strings masked) with leading attribute sections skipped, so `class` +// inside an attribute argument (`[Description("class Y : Fake")]`) is never +// taken for the declaration. The base list still stops at a newline, `{` or +// `;` for callers passing raw multi-line text. var csharpDeclHeadPattern = regexp.MustCompile(`\b(?:class|struct|interface|record(?:\s+(?:class|struct))?)\s+[A-Za-z_][A-Za-z0-9_]*`) func csharpInheritanceRefs(text string, kind core.SymbolKind) []csharpInheritanceRef { @@ -245,7 +241,21 @@ func csharpInheritanceRefs(text string, kind core.SymbolKind) []csharpInheritanc return refs } +// csharpSkipAttributes drops the `[...]` attribute sections that open a +// declaration header. +func csharpSkipAttributes(text string) string { + for { + text = strings.TrimLeft(text, " \t\r\n") + n := balancedSuffixEnd(text, '[', ']') + if n == 0 { + return text + } + text = text[n:] + } +} + func csharpBaseList(text string) string { + text = csharpSkipAttributes(text) loc := csharpDeclHeadPattern.FindStringIndex(text) if loc == nil { return "" @@ -286,8 +296,10 @@ func csharpBaseList(text string) string { var csharpNewPattern = regexp.MustCompile(`\bnew\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s*<[^;(){}]*>)?\s*\(`) -func csharpConstructedTypes(rawText string) []string { - matches := csharpNewPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) +// csharpConstructedTypes returns the `new T(` class names of masked (see +// maskCode). +func csharpConstructedTypes(masked string) []string { + matches := csharpNewPattern.FindAllStringSubmatch(masked, -1) seen := map[string]bool{} var out []string for _, match := range matches { diff --git a/internal/native/go.go b/internal/native/go.go index 33f00d2..da46028 100644 --- a/internal/native/go.go +++ b/internal/native/go.go @@ -891,12 +891,12 @@ func goImportScope(caller core.SymbolRecord, idx *goNativeIndex, pkgDirsByImport return scope } -// goIdentRe extracts identifier tokens from a stripped body in one pass. +// goIdentRe extracts identifier tokens from a masked body in one pass. var goIdentRe = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_]*`) var goQualifiedIdentRe = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b`) -// goTypeUseEdges emits lexical type-use edges. Each caller body is stripped -// and tokenized exactly once and tokens are resolved through the type index; +// goTypeUseEdges emits lexical type-use edges. Each caller body is masked +// (comments and literals blanked) and tokenized exactly once and tokens are resolved through the type index; // the previous implementation compiled a fresh regex and re-stripped the // body for every (caller, type-name) pair, unbounded by the analyzer // completion (linear cost; the deadline governs subprocesses, not pure-Go passes). @@ -915,7 +915,7 @@ func goTypeUseEdges(ctx context.Context, callers, all []core.SymbolRecord, pkgDi continue } scope := goImportScope(caller, idx, pkgDirsByImport) - stripped := stripQuotedText(caller.RawText) + stripped := maskCode("go", caller.RawText) emit := func(target core.SymbolRecord) { if target.ID == caller.ID { return @@ -960,14 +960,15 @@ func goTypeUseEdges(ctx context.Context, callers, all []core.SymbolRecord, pkgDi } // goContainsType reports whether name appears as a bare or package-qualified -// type token in rawText. Retained for direct callers and tests; the hot path -// in goTypeUseEdges tokenizes instead. +// type token in rawText's code. Retained for direct callers and tests; the +// hot path in goTypeUseEdges tokenizes instead. func goContainsType(rawText, name string) bool { - if containsTypeToken(rawText, name) { + masked := maskCode("go", rawText) + if containsTypeToken(masked, name) { return true } pattern := regexp.MustCompile(`\b[A-Za-z_][A-Za-z0-9_]*\.` + regexp.QuoteMeta(name) + `\b`) - return pattern.MatchString(stripQuotedText(rawText)) + return pattern.MatchString(masked) } // goListTopologyKey hashes everything go list output depends on: module diff --git a/internal/native/java.go b/internal/native/java.go index d886970..3759bb0 100644 --- a/internal/native/java.go +++ b/internal/native/java.go @@ -124,7 +124,9 @@ func javaSemanticEdgesIn(symbols []core.SymbolRecord, inScope func(string) bool) continue } if typeKind(symbol.Kind) { - for _, ref := range javaInheritanceRefs(symbol.Signature + "\n" + firstLine(symbol.RawText)) { + // The masked declaration header only: the raw first line also + // carried a trailing comment (`class A { // extends Base`). + for _, ref := range javaInheritanceRefs(maskedDeclHeader("java", symbol)) { if target, ok := javaBestType(idx, ref.Name, symbol.FilePath); ok && target.ID != symbol.ID { add(symbolEdge(symbol, target, ref.EdgeType, 0.97)) } @@ -139,24 +141,21 @@ func javaSemanticEdgesIn(symbols []core.SymbolRecord, inScope func(string) bool) // arity-tagged call sites for Java, so the graph layer's narrowed // resolution is authoritative. This pass keeps what text matching // is still good for: inheritance and type-usage evidence. - for _, className := range javaConstructedTypes(symbol.RawText) { + masked := maskCode("java", symbol.RawText) + for _, className := range javaConstructedTypes(masked) { if target, ok := javaBestType(idx, className, symbol.FilePath); ok && target.ID != symbol.ID { add(symbolEdge(symbol, target, core.EdgeUsesType, 0.96)) } } names := make([]string, 0, 8) - for t := range typeTokensIn(symbol.RawText) { + for t := range typeTokensIn(masked) { if _, ok := idx.typesByName[t]; ok { names = append(names, t) } } sort.Strings(names) - stripped := "" - if len(slowNames) > 0 { - stripped = stripQuotedText(symbol.RawText) - } for _, name := range slowNames { - if containsTypeTokenStripped(stripped, name) { + if containsTypeToken(masked, name) { names = append(names, name) } } @@ -204,28 +203,12 @@ func javaDeclarationTail(text string) string { return tail } -type javaQualifiedCall struct { - Qualifier string - Method string -} - -var javaQualifiedCallPattern = regexp.MustCompile(`\b([A-Z][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\s*\(`) - -func javaQualifiedCalls(rawText string) []javaQualifiedCall { - matches := javaQualifiedCallPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) - out := make([]javaQualifiedCall, 0, len(matches)) - for _, match := range matches { - if len(match) == 3 { - out = append(out, javaQualifiedCall{Qualifier: match[1], Method: match[2]}) - } - } - return out -} - var javaNewPattern = regexp.MustCompile(`\bnew\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s*<[^;(){}]*>)?\s*\(`) -func javaConstructedTypes(rawText string) []string { - matches := javaNewPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) +// javaConstructedTypes returns the `new T(` class names of masked (see +// maskCode). +func javaConstructedTypes(masked string) []string { + matches := javaNewPattern.FindAllStringSubmatch(masked, -1) seen := map[string]bool{} var out []string for _, match := range matches { @@ -310,10 +293,3 @@ func lastDottedName(name string) string { } return name } - -func firstLine(text string) string { - if i := strings.IndexByte(text, '\n'); i >= 0 { - return text[:i] - } - return text -} diff --git a/internal/native/javac.go b/internal/native/javac.go index 2b515ee..8e89645 100644 --- a/internal/native/javac.go +++ b/internal/native/javac.go @@ -429,9 +429,7 @@ func javaDependentDirs(req Request, files []string) map[string]bool { data, err := os.ReadFile(filepath.Join(req.Root, f)) pkg := "" if err == nil { - if m := javaPackageDecl.FindSubmatch(data); m != nil { - pkg = string(m[1]) - } + pkg = javaPackageOf(data) } classes := map[string]bool{strings.TrimSuffix(filepath.Base(f), ".java"): true} for _, list := range [][]core.SymbolRecord{req.PrevChanged, req.Symbols} { @@ -465,6 +463,8 @@ func javaDependentDirs(req Request, files []string) map[string]bool { break } } + // Raw text on purpose: a fully qualified name in a comment only + // widens the re-attributed set (more javac work), never an edge. if hit || (r.pkg != "" && s.ParentSymbol == "" && strings.Contains(s.RawText, fq)) { out[packageDir(s.FilePath)] = true break @@ -476,6 +476,16 @@ func javaDependentDirs(req Request, files []string) map[string]bool { var javaPackageDecl = regexp.MustCompile(`(?m)^\s*package\s+([A-Za-z_$][\w$.]*)\s*;`) +// javaPackageOf returns the file's package name ("" for the default +// package), read from masked source so a commented-out or Javadoc +// `package x;` line is not taken for the declaration. +func javaPackageOf(data []byte) string { + if m := javaPackageDecl.FindStringSubmatch(maskCode("java", string(data))); m != nil { + return m[1] + } + return "" +} + func javacResolve(ctx context.Context, req Request) ([]core.Edge, []string) { edges, diags, _ := javacResolveScoped(ctx, req) return edges, diags @@ -752,11 +762,11 @@ func javaSourceRoot(root, file string) string { if err != nil { return dir } - m := javaPackageDecl.FindSubmatch(data) - if m == nil { + pkg := javaPackageOf(data) + if pkg == "" { return dir } - pkgPath := strings.ReplaceAll(string(m[1]), ".", "/") + pkgPath := strings.ReplaceAll(pkg, ".", "/") switch { case dir == pkgPath: return "." diff --git a/internal/native/native_test.go b/internal/native/native_test.go index 4868fc3..e0b6eb0 100644 --- a/internal/native/native_test.go +++ b/internal/native/native_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) @@ -1840,3 +1841,80 @@ func TestTSPayloadSkipsStrayStdout(t *testing.T) { t.Fatalf("output without sentinel must pass through, got %q", got) } } + +// Comment/string-as-code audit (2026-10-10), native text passes: every scan +// runs on textmask output, and declaration clauses come from the masked +// header only. +func TestNativeTextPassesIgnoreCommentsAndStrings(t *testing.T) { + names := func(refs []phpInheritanceRef) []string { + var out []string + for _, r := range refs { + out = append(out, r.Name) + } + return out + } + php := "class A {\n public function run() {\n $f = function () use ($x) { return 1; };\n // we no longer use Loggable;\n $s = \"use Other;\";\n }\n use RealTrait;\n}" + if got := names(phpInheritanceRefs(maskCode("php", php))); !reflect.DeepEqual(got, []string{"RealTrait"}) { + t.Errorf("PHP trait uses: got %v, want [RealTrait]", got) + } + if got := phpReferencedClasses(textmask.MaskFile("php", "

new \\App\\Html()

\n(x: &'a mut Input) -> &'a Output {\n let c = 'x';\n let m = \"err: Config\"; // y: Shape\n}")); !reflect.DeepEqual(got, []string{"Input", "Output"}) { + t.Errorf("Rust signature types: got %v, want [Input Output]", got) + } + if got := rustImplRefs(maskCode("rust", "fn f() {\n // impl Shape for Wrapper {}\n let s = \"impl Shape for Other {}\";\n}")); len(got) != 0 { + t.Errorf("Rust impl refs from comment/string: %v", got) + } + if got := javaPackageOf([]byte("/*\npackage wrong;\n*/\n// package gone;\npackage right.one;\n")); got != "right.one" { + t.Errorf("Java package: got %q", got) + } + java := core.SymbolRecord{RawText: "@Ann({\"x\"})\npublic class A>\n extends Base\n implements I, J { // extends Fake implements K\n void m() { /* extends Q */ }\n}"} + var java2 []string + for _, r := range javaInheritanceRefs(maskedDeclHeader("java", java)) { + java2 = append(java2, string(r.EdgeType)+" "+r.Name) + } + if want := []string{"extends Base", "implements I", "implements J"}; !reflect.DeepEqual(java2, want) { + t.Errorf("Java inheritance: got %v, want %v", java2, want) + } + for _, raw := range []string{ + "class A extends B {" + strings.Repeat("\n int x;", 1000) + "\n}", + "@Ann(\"" + strings.Repeat("x", 5000) + "\") class A extends B {}", + } { + if got := maskedDeclHeader("java", core.SymbolRecord{RawText: raw}); !strings.HasSuffix(got, "class A extends B") { + t.Errorf("long Java header: got %q", clipTail(got)) + } + } + cs := core.SymbolRecord{RawText: "[Description(\"e.g. class Y : Fake, IOther\")]\n[Obsolete]\npublic class A : // not Fake\n Base,\n IReal where T : new() {\n void M() {}\n}"} + var csNames []string + for _, r := range csharpInheritanceRefs(maskedDeclHeader("csharp", cs), core.KindClass) { + csNames = append(csNames, r.Name) + } + if want := []string{"Base", "IReal"}; !reflect.DeepEqual(csNames, want) { + t.Errorf("C# bases: got %v, want %v", csNames, want) + } + cSrc := "#include \"a.h\"\n/*\n#include \"dead.h\"\n*/\n#if 0\n#include \"off.h\"\n#else\n#include \n#endif\n// #include \"c.h\"\nconst char *s = \"#include \\\"q.h\\\"\";\n" + if got := cIncludes(cIncludeScanText("a.c", cSrc)); !reflect.DeepEqual(got, []string{"a.h", "on.h"}) { + t.Errorf("C includes: got %v, want [a.h on.h]", got) + } + if got := cFamilyConstructedTypes(maskCode("cpp", "void f() {\n // Widget(arg) is legacy\n auto s = \"Gadget(1)\";\n Real r = Real(2);\n}")); !reflect.DeepEqual(got, []string{"Real"}) { + t.Errorf("C++ constructed types: got %v, want [Real]", got) + } + if goContainsType("func f() int { // auth.User\n\ts := \"User\"\n\treturn len(s)\n}", "User") { + t.Error("Go type token from comment/string") + } + // An apostrophe in a comment no longer blanks the code after it. + if got := csharpConstructedTypes(maskCode("csharp", "void M() {\n // don't use Real yet\n var r = new Real(); var c = 'x';\n}")); !reflect.DeepEqual(got, []string{"Real"}) { + t.Errorf("C# constructed types after apostrophe: got %v", got) + } +} + +func clipTail(s string) string { + if len(s) > 80 { + return "..." + s[len(s)-80:] + } + return s +} diff --git a/internal/native/php.go b/internal/native/php.go index b5f0f6a..c0d1865 100644 --- a/internal/native/php.go +++ b/internal/native/php.go @@ -4,10 +4,11 @@ import ( "context" "path/filepath" "regexp" + "sort" "strings" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" - "sort" ) type phpAnalyzer struct{} @@ -53,7 +54,7 @@ func (phpAnalyzer) Analyze(ctx context.Context, req Request) Result { if err != nil { continue } - for _, class := range phpReferencedClasses(string(content)) { + for _, class := range phpReferencedClasses(textmask.MaskFile("php", string(content))) { if target, ok := resolvePHPClass(class, psr4, fileScope); ok && target != file { edges = append(edges, nativeImportEdge(file, target, 0.94)) } @@ -93,7 +94,10 @@ func addPSR4(out map[string][]string, in map[string]any) { var phpUsePattern = regexp.MustCompile(`(?m)^[ \t]*use[ \t]+([^;\n]+);`) var phpNewPattern = regexp.MustCompile(`\bnew\s+\\?([A-Za-z_\\][A-Za-z0-9_\\]*)\s*\(`) -func phpReferencedClasses(content string) []string { +// phpReferencedClasses returns the classes a file imports (`use`) or +// constructs (`new`). masked is the file through textmask.MaskFile, so text +// in comments, strings, heredocs and inline HTML is never read as a class. +func phpReferencedClasses(masked string) []string { seen := map[string]bool{} var out []string add := func(name string) { @@ -103,10 +107,10 @@ func phpReferencedClasses(content string) []string { out = append(out, name) } } - for _, ref := range phpUseRefs(content) { + for _, ref := range phpUseRefs(masked) { add(ref.Name) } - for _, match := range phpNewPattern.FindAllStringSubmatch(content, -1) { + for _, match := range phpNewPattern.FindAllStringSubmatch(masked, -1) { if len(match) == 2 { add(match[1]) } @@ -192,8 +196,12 @@ func phpSemanticEdges(symbols []core.SymbolRecord, psr4 map[string][]string, fil if symbol.Language != "php" { continue } + if !typeKind(symbol.Kind) && !callableKind(symbol.Kind) { + continue + } + masked := maskCode("php", symbol.RawText) if typeKind(symbol.Kind) { - for _, ref := range phpInheritanceRefs(symbol.RawText) { + for _, ref := range phpInheritanceRefs(masked) { name := phpResolveAlias(ref.Name, aliases[symbol.FilePath]) if target, ok := phpBestType(idx, name, symbol.FilePath, psr4, fileScope); ok && target.ID != symbol.ID { add(symbolEdge(symbol, target, ref.EdgeType, 0.95)) @@ -208,25 +216,21 @@ func phpSemanticEdges(symbols []core.SymbolRecord, psr4 map[string][]string, fil // method it saw. The graph layer's call-site resolution owns calls; // this pass keeps only the type-usage evidence text matching is still // reliable for. - for _, className := range phpConstructedTypes(symbol.RawText) { + for _, className := range phpConstructedTypes(masked) { className = phpResolveAlias(className, aliases[symbol.FilePath]) if target, ok := phpBestType(idx, className, symbol.FilePath, psr4, fileScope); ok && target.ID != symbol.ID { add(symbolEdge(symbol, target, core.EdgeUsesType, 0.94)) } } names := make([]string, 0, 8) - for t := range typeTokensIn(symbol.RawText) { + for t := range typeTokensIn(masked) { if _, ok := idx.typesByName[t]; ok { names = append(names, t) } } sort.Strings(names) - stripped := "" - if len(slowNames) > 0 { - stripped = stripQuotedText(symbol.RawText) - } for _, name := range slowNames { - if containsTypeTokenStripped(stripped, name) { + if containsTypeToken(masked, name) { names = append(names, name) } } @@ -321,16 +325,20 @@ type phpInheritanceRef struct { var phpTraitUsePattern = regexp.MustCompile(`\buse[ \t]+([^;{\n]+)(?:;|\{)`) -func phpInheritanceRefs(rawText string) []phpInheritanceRef { +// phpInheritanceRefs reads extends/implements from the top-level headers and +// trait `use` clauses from the class bodies (depth 1) of masked (see +// maskCode). Method bodies are out of scope: a closure's `use ($x)` or any +// `use` text there is not a trait use. +func phpInheritanceRefs(masked string) []phpInheritanceRef { var refs []phpInheritanceRef - declaration := phpTopLevelHeaders(rawText) + declaration := phpTopLevelHeaders(masked) for _, name := range inheritanceClause(declaration, "extends", "implements") { refs = append(refs, phpInheritanceRef{Name: strings.Trim(name, "\\"), EdgeType: core.EdgeExtends}) } for _, name := range inheritanceClause(declaration, "implements") { refs = append(refs, phpInheritanceRef{Name: strings.Trim(name, "\\"), EdgeType: core.EdgeImplements}) } - for _, match := range phpTraitUsePattern.FindAllStringSubmatch(rawText, -1) { + for _, match := range phpTraitUsePattern.FindAllStringSubmatch(phpClassBodyText(masked), -1) { if len(match) != 2 { continue } @@ -369,26 +377,41 @@ func phpTopLevelHeaders(rawText string) string { return out.String() } -type phpStaticCall struct { - Class string - Method string -} - -var phpStaticCallPattern = regexp.MustCompile(`\b\\?([A-Za-z_\\][A-Za-z0-9_\\]*)::([A-Za-z_][A-Za-z0-9_]*)\s*\(`) - -func phpStaticCalls(rawText string) []phpStaticCall { - matches := phpStaticCallPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) - out := make([]phpStaticCall, 0, len(matches)) - for _, match := range matches { - if len(match) == 3 && match[2] != "class" { - out = append(out, phpStaticCall{Class: strings.Trim(match[1], "\\"), Method: match[2]}) +// phpClassBodyText returns the depth-1 text of masked: class-body members +// with every nested block reduced to `{}`, statements separated by newlines. +func phpClassBodyText(masked string) string { + var out strings.Builder + depth := 0 + for i := 0; i < len(masked); i++ { + switch c := masked[i]; c { + case '{': + depth++ + if depth == 2 { + out.WriteByte('{') + } + case '}': + if depth == 2 { + out.WriteByte('}') + } + if depth == 1 { + out.WriteByte('\n') + } + if depth > 0 { + depth-- + } + default: + if depth == 1 { + out.WriteByte(c) + } } } - return out + return out.String() } -func phpConstructedTypes(rawText string) []string { - matches := phpNewPattern.FindAllStringSubmatch(stripQuotedText(rawText), -1) +// phpConstructedTypes returns the `new T(` class names of masked (see +// maskCode). +func phpConstructedTypes(masked string) []string { + matches := phpNewPattern.FindAllStringSubmatch(masked, -1) seen := map[string]bool{} var out []string for _, match := range matches { diff --git a/internal/native/rust.go b/internal/native/rust.go index 3747209..96a8435 100644 --- a/internal/native/rust.go +++ b/internal/native/rust.go @@ -112,7 +112,7 @@ func rustModuleEdges(root string, files []string) []core.Edge { if err != nil { continue } - for _, mod := range rustModuleNames(string(content)) { + for _, mod := range rustModuleNames(maskCode("rust", string(content))) { for _, target := range rustModuleCandidates(file, mod) { if fileScope[target] { edges = append(edges, nativeImportEdge(file, target, 0.93)) @@ -124,6 +124,8 @@ func rustModuleEdges(root string, files []string) []core.Edge { return edges } +// rustModuleNames returns the `mod x;` declarations of masked file content +// (see maskCode), so a commented-out declaration is not a module edge. func rustModuleNames(content string) []string { lines := strings.Split(content, "\n") var mods []string @@ -181,9 +183,13 @@ func rustSemanticEdges(symbols []core.SymbolRecord, files []string) []core.Edge seen[key] = true edges = append(edges, edge) } - for _, symbol := range symbols { + // Mask each body once; the impl and type-use passes below read only + // masked text. + masked := make([]string, len(symbols)) + for i, symbol := range symbols { if symbol.Language == "rust" && symbol.RawText != "" { - for _, ref := range rustImplRefs(symbol.RawText) { + masked[i] = maskCode("rust", symbol.RawText) + for _, ref := range rustImplRefs(masked[i]) { concrete, okConcrete := rustBestType(typesByName, ref.TypeName, symbol.FilePath) trait, okTrait := rustBestType(typesByName, ref.TraitName, symbol.FilePath) if okConcrete && okTrait && concrete.ID != trait.ID { @@ -192,10 +198,11 @@ func rustSemanticEdges(symbols []core.SymbolRecord, files []string) []core.Edge } } } - for _, caller := range symbols { + for i, caller := range symbols { if caller.Language != "rust" || caller.RawText == "" || !callableKind(caller.Kind) { continue } + body := masked[i] // Call edges intentionally NOT emitted here (same lesson as the Java // native pass): the text-matching approach edged every same-named // function/method it saw, a multi-x edge explosion on a crate with @@ -206,17 +213,17 @@ func rustSemanticEdges(symbols []core.SymbolRecord, files []string) []core.Edge // (This analyzer only runs when `cargo` is on PATH — CI runners and // dev machines with rustup — so the explosion was invisible wherever // cargo was absent.) - for _, file := range append([]string{caller.FilePath}, rustModuleScopeFiles(caller.FilePath, caller.RawText, moduleFiles)...) { + for _, file := range append([]string{caller.FilePath}, rustModuleScopeFiles(caller.FilePath, body, moduleFiles)...) { for _, target := range byFile[file] { if target.ID == caller.ID { continue } - if typeKind(target.Kind) && rustContainsType(caller.RawText, target, file != caller.FilePath) { + if typeKind(target.Kind) && rustContainsType(body, target, file != caller.FilePath) { add(symbolEdge(caller, target, core.EdgeUsesType, 0.93)) } } } - for _, typeName := range rustSignatureTypes(caller.Signature + "\n" + caller.RawText) { + for _, typeName := range rustSignatureTypes(maskCode("rust", caller.Signature) + "\n" + body) { if target, ok := rustBestType(typesByName, typeName, caller.FilePath); ok && target.ID != caller.ID { add(symbolEdge(caller, target, core.EdgeUsesType, 0.94)) } @@ -230,6 +237,8 @@ type rustImplRef struct { TypeName string } +// rustImplRefs returns the `impl Trait for Type` pairs of masked (see +// maskCode). func rustImplRefs(rawText string) []rustImplRef { var out []rustImplRef for search := 0; search < len(rawText); { @@ -361,6 +370,8 @@ func rustImplBaseName(text string) string { var rustSignatureTypePattern = regexp.MustCompile(`(?:->|:)\s*&?(?:'[A-Za-z_][A-Za-z0-9_]*\s+)?(?:mut\s+)?([A-Z][A-Za-z0-9_:]*)`) +// rustSignatureTypes returns the `: Type` / `-> Type` names of masked text +// (see maskCode). func rustSignatureTypes(text string) []string { matches := rustSignatureTypePattern.FindAllStringSubmatch(text, -1) seen := map[string]bool{} @@ -422,6 +433,7 @@ func rustModuleFileIndex(files []string) map[string]string { return out } +// rustModuleScopeFiles: rawText is masked (see maskCode). func rustModuleScopeFiles(fromFile, rawText string, moduleFiles map[string]string) []string { seen := map[string]bool{} var out []string @@ -449,13 +461,15 @@ func rustPathPrefixes(rawText string) []string { return out } -func rustContainsType(rawText string, target core.SymbolRecord, qualified bool) bool { +// rustContainsType reports whether masked (see maskCode) names target, as a +// bare token in its own file or module-qualified across files. +func rustContainsType(masked string, target core.SymbolRecord, qualified bool) bool { if !qualified { - return containsTypeToken(rawText, target.Name) + return containsTypeToken(masked, target.Name) } module := rustModuleName(target.FilePath) pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(module) + `::` + regexp.QuoteMeta(target.Name) + `\b`) - return pattern.MatchString(stripQuotedText(rawText)) + return pattern.MatchString(masked) } func rustModuleName(file string) string { diff --git a/internal/native/semantic.go b/internal/native/semantic.go index 6f39d07..2e80ab7 100644 --- a/internal/native/semantic.go +++ b/internal/native/semantic.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) @@ -141,9 +142,9 @@ func lexicalSemanticEdges(symbols []core.SymbolRecord, languages map[string]bool if !languages[caller.Language] || caller.RawText == "" || !callableKind(caller.Kind) { continue } - // Strip and tokenize once per caller instead of compiling a regex - // and re-stripping the body for every (caller, target) pair. - stripped := stripQuotedText(caller.RawText) + // Mask and tokenize once per caller instead of compiling a regex + // and re-masking the body for every (caller, target) pair. + stripped := maskCode(caller.Language, caller.RawText) callNames := map[string]bool{} for _, m := range lexCallRe.FindAllStringSubmatch(stripped, -1) { callNames[m[1]] = true @@ -180,12 +181,11 @@ func typeKind(kind core.SymbolKind) bool { } } -// Pattern caches: the per-language analyzers call containsCall / -// containsTypeToken inside (symbol × candidate) loops; compiling a fresh -// regex per probe dominated their cost. +// Pattern cache: the per-language analyzers call containsTypeToken inside +// (symbol × candidate) loops; compiling a fresh regex per probe dominated +// their cost. var ( patternCacheMu sync.Mutex - callPatternCache = map[string]*regexp.Regexp{} tokenPatternCache = map[string]*regexp.Regexp{} ) @@ -200,19 +200,10 @@ func cachedPattern(cache map[string]*regexp.Regexp, name, prefix, suffix string) return p } -func containsCall(text, name string) bool { - if name == "" { - return false - } - pattern := cachedPattern(callPatternCache, name, `\b`, `\s*\(`) - return pattern.MatchString(stripQuotedText(text)) -} - // identTokenRe extracts identifier-shaped tokens for typeTokensIn. var identTokenRe = regexp.MustCompile(`[A-Za-z_$][A-Za-z0-9_$]*`) -// typeTokensIn returns the distinct identifier tokens of text with quoted -// text stripped. Type-use passes scan the body ONCE and look tokens up, +// Type-use passes tokenize the masked body ONCE (typeTokensIn) and look tokens up, // replacing the per-(symbol, type-name) regex scans that made the java/ // csharp/php analyzers quadratic (285s on a 1.2k-file repo — never noticed // while the 5s timeout killed them before completion). @@ -239,62 +230,82 @@ func slowTypeNames(typesByName map[string][]core.SymbolRecord) []string { return out } -func typeTokensIn(text string) map[string]bool { +// typeTokensIn returns the distinct identifier tokens of masked (see +// maskCode). +func typeTokensIn(masked string) map[string]bool { tokens := map[string]bool{} - for _, t := range identTokenRe.FindAllString(stripQuotedText(text), -1) { + for _, t := range identTokenRe.FindAllString(masked, -1) { tokens[t] = true } return tokens } -// containsTypeTokenStripped is containsTypeToken on text already passed -// through stripQuotedText, for loops probing one body with many names. -func containsTypeTokenStripped(stripped, name string) bool { +// containsTypeToken reports whether name occurs as a whole token in masked +// (see maskCode). +func containsTypeToken(masked, name string) bool { if name == "" { return false } - return cachedPattern(tokenPatternCache, name, `\b`, `\b`).MatchString(stripped) + return cachedPattern(tokenPatternCache, name, `\b`, `\b`).MatchString(masked) } -func containsTypeToken(text, name string) bool { - if name == "" { - return false +// maskCode blanks the comments and string/char literals of a symbol snippet +// (lexed as code from byte 0) so this package's text passes never read prose or +// literal text as a type, call or clause. Offsets and newlines are kept. +// Every regex or token scan over RawText, Signature or file content in this +// package must run on masked text: native edges carry 0.9+ confidence, so a +// comment-derived one is the worst kind of wrong edge. +func maskCode(lang, src string) string { + if !textmask.Supported(lang) { + lang = "c" // unknown languages get the generic C-style lexer } - pattern := cachedPattern(tokenPatternCache, name, `\b`, `\b`) - return pattern.MatchString(stripQuotedText(text)) + return textmask.Mask(lang, src) } -func stripQuotedText(text string) string { - var out strings.Builder - out.Grow(len(text)) - inString := false - var quote rune - escaped := false - for _, r := range text { - if inString { - if escaped { - escaped = false - out.WriteRune(' ') - continue - } - if r == '\\' { - escaped = true - out.WriteRune(' ') - continue +// maskedDeclHeader returns a type declaration's header — annotations or +// attributes, modifiers, name, type parameters and base clauses — masked and +// with whitespace runs collapsed to one space: the masked RawText up to the +// first `{` or `;` outside parentheses and brackets (annotation arguments +// like `@A({1})` and attribute arguments may hold braces). Without RawText it +// falls back to the masked Signature. The header never includes member +// bodies, so a comment after the brace (`class A { // extends B`) is out of +// reach twice over. +func maskedDeclHeader(lang string, symbol core.SymbolRecord) string { + text := symbol.RawText + if text == "" { + text = symbol.Signature + } + // A class RawText can be a whole file; the header is almost always in + // the first few KB. The mask is a forward lexer, so masking a prefix + // gives the prefix of the full mask. + if len(text) > declHeaderProbe { + if header, ok := declHeaderCut(maskCode(lang, text[:declHeaderProbe])); ok { + return header + } + } + header, _ := declHeaderCut(maskCode(lang, text)) + return header +} + +const declHeaderProbe = 4096 + +// declHeaderCut cuts masked at its first `{` or `;` outside () and [] and +// collapses whitespace; ok is false when there is no such cut. +func declHeaderCut(masked string) (string, bool) { + depth := 0 + for i := 0; i < len(masked); i++ { + switch masked[i] { + case '(', '[': + depth++ + case ')', ']': + if depth > 0 { + depth-- } - if r == quote { - inString = false + case '{', ';': + if depth == 0 { + return strings.Join(strings.Fields(masked[:i]), " "), true } - out.WriteRune(' ') - continue - } - if r == '"' || r == '\'' || r == '`' { - inString = true - quote = r - out.WriteRune(' ') - continue } - out.WriteRune(r) } - return out.String() + return strings.Join(strings.Fields(masked), " "), false } diff --git a/internal/parser/cfamily_decoy_test.go b/internal/parser/cfamily_decoy_test.go new file mode 100644 index 0000000..f8d19b8 --- /dev/null +++ b/internal/parser/cfamily_decoy_test.go @@ -0,0 +1,296 @@ +package parser + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/provasign/astkit" + "github.com/provasign/grove/internal/core" +) + +// Decoy invariant, C family (2026-10-10 round 2): a comment line added +// after a `{` must not change the symbols of a file, even one tree-sitter +// cannot parse. fmt, nlohmann/json and AFNetworking each had files whose +// error recovery, line-scanner window or namespace choice moved with the +// comments. + +// decoyLines inserts a code-shaped comment line after every line of src +// that ends in `{`, as research's decoy injector does, and returns the +// decoy and, per decoy line (1-based), the clean line it came from (0 for +// an inserted line). +func decoyLines(src string) (string, []int) { + const decoy = "// decoy: class Fake extends Base { void ghost() { helper(1); new Fake(); } } Widget::draw(x); don't" + var out []string + from := []int{0} + for i, line := range strings.Split(src, "\n") { + out = append(out, line) + from = append(from, i+1) + if strings.HasSuffix(strings.TrimSpace(line), "{") { + out = append(out, " "+decoy) + from = append(from, 0) + } + } + // src ends in a newline: the block starts on the last (empty) line. + return strings.Join(out, "\n") + "/* decoy block: don't 'quote\nclass Fake2 { void ghost(); };\n*/\n", from +} + +// symbolKeys lists symbols as kind/qualified name/start line, decoy lines +// mapped back to clean ones. +func symbolKeys(syms []core.SymbolRecord, from []int) []string { + var out []string + for _, s := range syms { + start := s.Span.Start + if from != nil && start < len(from) { + start = from[start] + } + out = append(out, fmt.Sprintf("%s %s @%d", s.Kind, s.QualifiedName, start)) + } + sort.Strings(out) + return out +} + +func assertDecoyTwin(t *testing.T, path, clean string) []core.SymbolRecord { + t.Helper() + decoy, from := decoyLines(clean) + cleanSyms := extractFor(t, path, clean) + decoySyms := extractFor(t, path, decoy) + a, b := symbolKeys(cleanSyms, nil), symbolKeys(decoySyms, from) + if strings.Join(a, "\n") != strings.Join(b, "\n") { + t.Fatalf("decoy changed the symbols\nclean:\n %s\ndecoy:\n %s", strings.Join(a, "\n "), strings.Join(b, "\n ")) + } + if ci, di := extractImports(DetectLanguageContent(path, []byte(clean)), clean), extractImports(DetectLanguageContent(path, []byte(decoy)), decoy); fmt.Sprint(ci) != fmt.Sprint(di) { + t.Fatalf("decoy changed the imports: %v vs %v", ci, di) + } + return decoySyms +} + +func TestCFamilyBrokenFileDecoyTwin(t *testing.T) { + // `int broken( {` keeps the file broken; the rest must come out the + // same with or without comment lines, with spans on the decoy's own + // lines and bodies that still hold its comments. + clean := `#include +#include "widget.h" + +namespace ns { +int broken( { +} + +class Widget { + public: + void draw() { + helper(1); + } +}; + +int helper(int x) { + return x; +} +} // namespace ns +` + syms := assertDecoyTwin(t, "w.cpp", clean) + // Tree-sitter's recovery prices every byte and line it skips, so the + // broken file must reach the parser as the same bytes either way. + decoy, _ := decoyLines(clean) + eng, _ := bridge() + inputs := map[string]string{} + for name, src := range map[string]string{"clean": clean, "decoy": decoy} { + b := parseBrokenCFamily(context.Background(), eng, astkit.LangCPP, "cpp", []byte(src), []byte(src), []byte(src), true) + if b == nil { + t.Fatalf("%s: no parse", name) + } + b.tree.Close() + inputs[name] = string(b.src) + } + if inputs["clean"] != inputs["decoy"] { + t.Fatalf("parse input differs:\nclean %q\ndecoy %q", inputs["clean"], inputs["decoy"]) + } + for _, s := range syms { + if s.QualifiedName != "ns::helper" { + continue + } + if !strings.HasPrefix(s.RawText, "int helper(int x) {\n // decoy:") || !strings.HasSuffix(s.RawText, "}") { + t.Fatalf("helper body lost its comment or its edges: %q", s.RawText) + } + return + } + t.Fatalf("ns::helper missing:%s", describe(syms)) +} + +func TestObjCAvailabilityMacrosDecoyTwin(t *testing.T) { + // AFNetworking: trailing AF_API_AVAILABLE(...) attributes, an + // NS_ENUM, NS_ASSUME_NONNULL_BEGIN and NS_DESIGNATED_INITIALIZER + // broke the parse; recovery then flipped with comment lines. + clean := `#import + +typedef NS_ENUM(NSUInteger, AFSSLPinningMode) { + AFSSLPinningModeNone, + AFSSLPinningModeCertificate, +}; + +NS_ASSUME_NONNULL_BEGIN + +@interface AFSecurityPolicy : NSObject +@property (readonly, nonatomic, assign) AFSSLPinningMode SSLPinningMode; +@property (nonatomic, strong) NSURLSessionTaskMetrics *metrics AF_API_AVAILABLE(ios(10), macosx(10.12)); +- (instancetype)initWithMode:(AFSSLPinningMode)mode NS_DESIGNATED_INITIALIZER; +@end + +@implementation AFSecurityPolicy +- (void)didFinish:(NSURLSessionTaskMetrics *)metrics AF_API_AVAILABLE(ios(10), macosx(10.12)) { + [self reset]; +} +- (void)reset { + if (@available(macOS 10.11, *)) { + [self description]; + } +} +@end + +NS_ASSUME_NONNULL_END +` + syms := assertDecoyTwin(t, "p.m", clean) + for _, want := range []string{"AFSecurityPolicy", "AFSecurityPolicy.SSLPinningMode", "AFSecurityPolicy.metrics", + "AFSecurityPolicy.initWithMode:", "AFSecurityPolicy.didFinish:", "AFSecurityPolicy.reset", "AFSSLPinningMode"} { + found := false + for _, s := range syms { + found = found || s.QualifiedName == want + } + if !found { + t.Errorf("%s missing:%s", want, describe(syms)) + } + } +} + +func TestObjCInterfaceInsideConditional(t *testing.T) { + // AFNetworking's UIKit headers wrap the whole @interface in + // `#if TARGET_OS_IOS`; a clean parse put it under preproc_if, where + // the Objective-C walker did not look. + src := `#import +#if TARGET_OS_IOS +@interface AFNetworkActivityIndicatorManager : NSObject +@property (nonatomic, assign, getter = isEnabled) BOOL enabled; +- (void)incrementActivityCount; +@end +#endif +` + syms := extractFor(t, "a.h", src) + for _, want := range []string{"AFNetworkActivityIndicatorManager", "AFNetworkActivityIndicatorManager.enabled", "AFNetworkActivityIndicatorManager.incrementActivityCount"} { + found := false + for _, s := range syms { + found = found || s.QualifiedName == want + } + if !found { + t.Errorf("%s missing:%s", want, describe(syms)) + } + } +} + +func TestCPPExportMacrosDecoyTwin(t *testing.T) { + // gtest/doctest: export macros between `class` and the name, after + // `static`, before an operator, macro statements at namespace scope + // and a member-declaring macro in a class body. Each threw the + // grammar; a run of them folded gmock.h into one ERROR, and which + // recovery won depended on the comment lines. + clean := `namespace testing { +namespace internal { + +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(unsigned char, '\0'); +GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(float, 0); + +class GTEST_API_ RE { + public: + static DOCTEST_CONSTEXPR size_type len = 24; + RE(const char* regex) { Init(regex); } + friend DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const RE& in); + private: + void Init(const char* regex); +}; + +struct DOCTEST_INTERFACE IContextScope +{ + DOCTEST_DECLARE_INTERFACE(IContextScope) + virtual void stringify(std::ostream*) const = 0; +}; + +bool IsAlpha(char ch) { + return isalpha(ch) != 0; +} + +} // namespace internal +} // namespace testing +` + syms := assertDecoyTwin(t, "gtest.h", clean) + for _, want := range []string{"testing::internal::RE", "testing::internal::RE::Init", "testing::internal::IContextScope", + "testing::internal::IContextScope::stringify", "testing::internal::IsAlpha"} { + found := false + for _, s := range syms { + found = found || s.QualifiedName == want + } + if !found { + t.Errorf("%s missing:%s", want, describe(syms)) + } + } + for _, s := range syms { + if strings.HasPrefix(s.Name, "GMOCK_") || s.Name == "GTEST_API_" { + t.Errorf("macro use read as a symbol: %s %s", s.Kind, s.QualifiedName) + } + } +} + +func TestCommentCutsMapBack(t *testing.T) { + src := []byte("int a /* x */ = 1; // tail\n // only a comment\n#define M(x) \\\n // inside\nint b;\n/* one\n two */ int c;\n") + c := cFamilyCommentCuts("c", src) + cut := string(c.apply(src)) + want := "int a = 1; \n#define M(x) \\\n \nint b;\n int c;\n" + if cut != want { + t.Fatalf("cut = %q, want %q", cut, want) + } + // Line 2 of the cut copy is the #define (source line 3); the + // comment-only line after its backslash stays, blank. + for cutLine, srcLine := range map[int]int{1: 1, 2: 3, 3: 4, 4: 5, 5: 7} { + if got := c.row(cutLine); got != srcLine { + t.Errorf("row(%d) = %d, want %d", cutLine, got, srcLine) + } + } + // `=` is column 8 of the cut line, column 14 of the source line. + if got := c.column(0, 8); got != 14 { + t.Errorf("column(0, 8) = %d, want 14", got) + } + if got := c.column(4, 5); got != 10 { + t.Errorf("column(4, 5) = %d, want 10 (`int c` after the block comment)", got) + } +} + +func TestBraceBodyWindowCountsCodeLines(t *testing.T) { + // The line scanner gives up 500 lines into a body. Counting blank + // (masked comment) lines toward that let comments cut a class short. + lines := []string{"class Big {"} + for i := 0; i < 400; i++ { + lines = append(lines, " int f"+fmt.Sprint(i)+";", "") + } + lines = append(lines, "};") + end, _ := extractBraceBody(lines, 0) + if end != len(lines) { + t.Fatalf("body ends at %d, want %d", end, len(lines)) + } +} + +func TestCppNamespaceOverlappingSpans(t *testing.T) { + // Recovered namespace spans can overlap without nesting + // (`namespace testing {` 10-100, `namespace internal {` 11-101); + // the innermost opening wins, whatever the line counts. + for _, internalEnd := range []int{101, 140} { + syms := []core.SymbolRecord{ + {Kind: core.KindNamespace, Name: "testing", QualifiedName: "testing", Span: core.LineRange{Start: 10, End: 100}}, + {Kind: core.KindNamespace, Name: "internal", QualifiedName: "internal", Span: core.LineRange{Start: 11, End: internalEnd}}, + {Kind: core.KindFunction, Name: "IsAlpha", QualifiedName: "IsAlpha", Span: core.LineRange{Start: 50, End: 52}}, + } + syms = enrichCppNamespaces(syms, "") + if got := syms[2].QualifiedName; got != "internal::IsAlpha" { + t.Errorf("internal ends %d: IsAlpha = %q, want internal::IsAlpha", internalEnd, got) + } + } +} diff --git a/internal/parser/comment_string_recovery_test.go b/internal/parser/comment_string_recovery_test.go new file mode 100644 index 0000000..8c978e1 --- /dev/null +++ b/internal/parser/comment_string_recovery_test.go @@ -0,0 +1,212 @@ +package parser + +import ( + "fmt" + "strings" + "testing" + + "github.com/provasign/grove/internal/core" +) + +// Comment/string-as-code audit (2026-10-10), parser layer. Files with a +// syntax error take the regex recovery path (extractSymbols merges names +// the AST missed); text inside comments, strings, heredocs, inline HTML, +// JSX text, and inactive #if branches must not become symbols, must not +// stretch or cut recovered spans, and must not bleed into docstrings. +// Index-level twins of the same findings: internal/invariance. + +func extractFor(t *testing.T, path, src string) []core.SymbolRecord { + t.Helper() + syms, err := NewEngine().ExtractContent(path, []byte(src)) + if err != nil { + t.Fatalf("extract %s: %v", path, err) + } + return syms +} + +func describe(syms []core.SymbolRecord) string { + var b strings.Builder + for _, s := range syms { + fmt.Fprintf(&b, "\n %s %s parent=%q span=%d-%d lang=%s ann=%v", s.Kind, s.QualifiedName, s.ParentSymbol, s.Span.Start, s.Span.End, s.Language, s.Annotations) + } + return b.String() +} + +func hasSymbolNamed(syms []core.SymbolRecord, name string) bool { + parent, leaf, qualified := "", name, strings.Contains(name, ".") + if qualified { + i := strings.LastIndex(name, ".") + parent, leaf = name[:i], name[i+1:] + } + for _, s := range syms { + if s.Name != leaf && s.QualifiedName != name { + continue + } + if !qualified || s.QualifiedName == name || strings.HasSuffix(s.ParentSymbol, parent) || + strings.ReplaceAll(s.QualifiedName, "::", ".") == name { + return true + } + } + return false +} + +func TestCommentStringRecovery_NoPhantomSymbols(t *testing.T) { + cases := []struct { + id, path, src string + absent []string + present []string + }{ + {"P6-go", "a.go", "package p\n\n/*\nfunc Phantom() {}\ntype Ghost struct{}\n*/\nvar s = `\nfunc Phantom2() {}\ntype Ghost2 struct{}\n`\n\nfunc Real() int { return 1 }\n\nfunc Broken( {\n", + []string{"Phantom", "Ghost", "Phantom2", "Ghost2"}, []string{"Real"}}, + {"P7-java-text-block", "A.java", "class A {\n String s = \"\"\"\n class Phantom { void ghost() {} }\n \"\"\";\n void real() {}\n void broken( {\n}\n", + []string{"Phantom", "ghost"}, []string{"real"}}, + {"P8-csharp-raw-verbatim", "A.cs", "class A {\n string s = \"\"\"\n class Phantom { }\n \"\"\";\n string v = @\"\nclass Phantom2 { }\n\";\n void Real() {}\n void Broken( {\n}\n", + []string{"Phantom", "Phantom2"}, []string{"Real"}}, + {"P9-ts-template", "a.ts", "const tpl = `\nclass Phantom {\nfunction ghost(a) {\n`;\nexport function real() { return 1; }\nexport function broken( {\n", + []string{"Phantom", "ghost"}, []string{"real"}}, + {"P9-tsx-jsx-text", "v.tsx", "export function View() {\n return
\n class names are applied\n
;\n}\nexport function broken( {\n", + []string{"names"}, []string{"View"}}, + {"P10-php-heredoc-html", "a.php", "\n

\n class names here\n

\n", + []string{"Phantom", "ghost", "names"}, []string{"real"}}, + {"P11-rust-raw-string", "lib.rs", "const S: &str = r#\"\nfn ghost() {}\nstruct Phantom;\n\"#;\npub fn real() -> i32 { 1 }\npub fn broken( {\n", + []string{"ghost", "Phantom"}, []string{"real"}}, + {"P12-python-continued-string", "a.py", "x = \"abc \\\nclass Phantom:\"\n\n\ndef real():\n return 1\n\n\ndef broken(:\n pass\n", + []string{"Phantom"}, []string{"real"}}, + {"P13-c-if0", "a.c", "#if 0\nint ghost(int a) {\n#endif\nint real(void) { return 0; }\nint broken( {\n", + []string{"ghost"}, []string{"real"}}, + {"P13-cpp-if0-namespace", "a.cpp", "#if 0\nnamespace legacy {\nint ghost(int a) {\n#endif\nint real(void) { return 0; }\nint broken( {\n", + []string{"legacy", "ghost", "legacy.real"}, []string{"real"}}, + {"P13-csharp-if-false", "A.cs", "#if false\npublic class Legacy {\n#endif\npublic class Real { public void Go() {} }\npublic class Broken { void X( { }\n", + []string{"Legacy", "Legacy.Real"}, []string{"Real", "Real.Go"}}, + {"P14-cpp-comment-qualified-name", "a.cpp", "int broken( {\nvoid helper(int x) { // see Widget::draw(x)\n (void)x;\n}\n", + []string{"Widget.draw", "draw"}, []string{"helper"}}, + {"P2-cpp-raw-string", "a.cpp", "const char* kSrc = R\"(\nint phantom(int x) {\n return x;\n}\n)\";\nint real(int y) { return y; }\n", + []string{"phantom"}, []string{"real"}}, + {"P3-c-multiline-define", "a.c", "#define DECL(n) \\\n static int helper(void) { \\\n return n; \\\n }\nint real(void) { return 0; }\n", + []string{"helper"}, []string{"real"}}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + syms := extractFor(t, tc.path, tc.src) + for _, name := range tc.absent { + if hasSymbolNamed(syms, name) { + t.Errorf("phantom symbol %q from comment/string/inactive text%s", name, describe(syms)) + } + } + for _, name := range tc.present { + if !hasSymbolNamed(syms, name) { + t.Errorf("real symbol %q missing%s", name, describe(syms)) + } + } + }) + } +} + +// P15: recovered symbols (after the syntax error, so the AST lost them) end +// where their body ends, whatever braces/dedents hide in comments/strings. +func TestCommentStringRecovery_Spans(t *testing.T) { + cases := []struct { + id, path, src, name string + start, end int + }{ + {"P15-java-block-comment-brace", "A.java", + "class A {\n void broken( {\n void real() {\n int a = 0; /* } */\n a++;\n }\n void after() {}\n}\n", "real", 3, 6}, + {"P15-ts-template-brace", "a.ts", + "export function broken( {\nexport function real() {\n const s = `\n}\n`;\n return s;\n}\nexport function after() {}\n", "real", 2, 7}, + {"P15-python-docstring-col0", "a.py", + "x = foo(\n\n\ndef real():\n \"\"\"Doc.\nend\n\"\"\"\n return 1\n\n\ndef after():\n return 2\n", "real", 4, 8}, + // P16: a call statement above a function is not an overload + // signature; the span starts at the function. + {"P16-ts-overload-widening", "a.ts", "foo(1);\nfunction foo(x: number) {\n return x;\n}\n", "foo", 2, 4}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + syms := extractFor(t, tc.path, tc.src) + for _, s := range syms { + if s.Name == tc.name { + // End may include trailing blank lines (existing + // convention); it must reach the last body line. + if s.Span.Start != tc.start || s.Span.End < tc.end { + t.Errorf("%s span = %d-%d, want %d-%d (end >= %d)%s", tc.name, s.Span.Start, s.Span.End, tc.start, tc.end, tc.end, describe(syms)) + } + return + } + } + t.Fatalf("symbol %s not found%s", tc.name, describe(syms)) + }) + } +} + +// P1: precedingCommentBlock must not take code (or a string holding "/**") +// for a doc comment. +func TestCommentStringRecovery_PrecedingCommentDocstring(t *testing.T) { + cases := []struct { + id, path, src, name string + mustNot []string + }{ + {"P1-java-trailing-block-comment", "A.java", + "/** The A class. */\nclass A {\n int x; /* trailing */\n void b() {}\n}\n", "b", []string{"int x", "class A", "trailing"}}, + {"P1-js-glob-string", "a.js", + "const glob = \"src/**\";\n/* helper */\nfunction f() {}\n", "f", []string{"const glob", "src/"}}, + {"P1-go-trailing-block-comment", "a.go", + "/** Package p. */\npackage p\n\nvar x = 1 /* trailing */\nfunc F() {}\n", "F", []string{"var x", "package", "trailing"}}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + syms := extractFor(t, tc.path, tc.src) + for _, s := range syms { + if s.Name != tc.name { + continue + } + for _, bad := range tc.mustNot { + if strings.Contains(s.Docstring, bad) { + t.Errorf("%s docstring %q contains code %q", tc.name, s.Docstring, bad) + } + } + return + } + t.Fatalf("symbol %s not found%s", tc.name, describe(syms)) + }) + } +} + +// P17/A9: a block-comment line in a .h file that starts with a C++ keyword +// must not flip the header's language to C++. +func TestCommentStringRecovery_HeaderLanguageSniff(t *testing.T) { + for _, tc := range []struct{ id, src string }{ + {"P17-namespace-comment-line", "/*\nnamespace handling is not supported here\n */\nint real(void);\n"}, + {"A9-class-comment-line", "/*\n class handling for the C API\n */\nint real(void);\n"}, + } { + t.Run(tc.id, func(t *testing.T) { + if got := DetectLanguageContent("a.h", []byte(tc.src)); got != "c" { + t.Errorf("language = %q, want c", got) + } + }) + } +} + +func TestImportPathsDropComments(t *testing.T) { + clean := "pub use crate::{\n error::{Error, ErrorKind},\n};\nuse std::{\n io,\n};\n" + decoy := "pub use crate::{\n // decoy: use Fake; don't 'quote\n error::{Error, ErrorKind}, // trailing\n};\nuse std::{\n /* block */ io,\n};\n" + want, ok := extractImportsFromAST("rust", []byte(clean)) + if !ok { + t.Fatal("clean imports not extracted") + } + got, ok := extractImportsFromAST("rust", []byte(decoy)) + if !ok { + t.Fatal("decoy imports not extracted") + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("comments leaked into import paths:\n got %q\nwant %q", got, want) + } +} + +func TestCSharpRecoveryIgnoresCallStatements(t *testing.T) { + src := "class T {\n void A() {\n var x = Make(\n new JArray(1, 2),\n new JValue(3));\n return Build(x);\n }\n void Broken( {\n}\n" + for _, s := range extractSymbols("csharp", "t.cs", "sha", src, nil) { + switch s.Name { + case "JArray", "JValue", "Build", "Make": + t.Errorf("call statement recovered as a declaration: %s %s", s.Kind, s.Name) + } + } +} diff --git a/internal/parser/commentcut.go b/internal/parser/commentcut.go new file mode 100644 index 0000000..fc522f9 --- /dev/null +++ b/internal/parser/commentcut.go @@ -0,0 +1,230 @@ +package parser + +import ( + "bytes" + "context" + "strings" + + "github.com/provasign/astkit" + "github.com/provasign/astkit/textmask" + sitter "github.com/smacker/go-tree-sitter" +) + +// brokenParse is the tree parseBrokenCFamily chose, with the copies of the +// source it addresses. +type brokenParse struct { + tree *sitter.Tree + src []byte // what was parsed + text []byte // what symbol text is cut from (src's lines, unmasked) + cuts *commentCuts + uncutText []byte // text with its comments, on the source's lines +} + +// parseBrokenCFamily re-parses a C-family file whose first parse has +// errors from its code alone: comments cut out, comment-only lines +// dropped (cFamilyCommentCuts). Tree-sitter's recovery is sensitive to +// every byte and line it skips, so only an input that is the same for any +// commenting of the program gives the same tree. With branches, the copy +// with every #else/#elif branch and directive blanked is parsed too +// (blankPreprocessorBranches: two else branches of one `if` throw the +// grammar) and kept when it has fewer ERROR nodes. src, text and live are +// extractASTSymbols' offset-aligned copies. It returns nil when nothing +// parses. +func parseBrokenCFamily(ctx context.Context, eng *astkit.Engine, key astkit.LanguageKey, language string, src, text, live []byte, branches bool) *brokenParse { + cuts := cFamilyCommentCuts(language, live) + cutSrc, cutLive := cuts.apply(src), cuts.apply(live) + tree, err := eng.Parse(ctx, key, cutSrc) + if err != nil || tree == nil { + return nil + } + best := &brokenParse{tree: tree, src: cutSrc, text: cuts.apply(text), cuts: cuts, uncutText: text} + if !branches || !tree.RootNode().HasError() { + return best + } + alt := blankPreprocessorBranches(cutSrc) + if alt == nil { + return best + } + altTree, err := eng.Parse(ctx, key, alt) + if err != nil || altTree == nil { + return best + } + if countErrorNodes(altTree.RootNode()) >= countErrorNodes(tree.RootNode()) { + altTree.Close() + return best + } + tree.Close() + // Same line lengths as alt: macro blanking never touches a directive + // line, and live has src's directive lines. + return &brokenParse{tree: altTree, src: alt, text: blankPreprocessorBranches(cutLive), cuts: cuts, + uncutText: blankPreprocessorBranches(live)} +} + +// commentCut removes columns [from, to) of one source line and leaves a +// single space in their place, so the code on either side stays two +// tokens. +type commentCut struct{ from, to int } + +// commentCuts takes the comments out of a C-family source: every comment +// becomes one space and a line holding nothing but comments is dropped. +// It maps the cut copy's lines and columns back to the source. +type commentCuts struct { + lines [][]commentCut // per source line + drop []bool // per source line: dropped from the cut copy + rows []int // per cut line: its source line (0-based) +} + +// cFamilyCommentCuts returns the cuts for src. Comments are the bytes +// textmask.MaskComments changes, joined across the blanks between them. A +// comment-only line that ends a backslash-continued line is kept (blank), +// so no line joins a #define. +func cFamilyCommentCuts(language string, src []byte) *commentCuts { + masked := textmask.MaskComments(language, string(src)) + lines := bytes.Split(src, []byte("\n")) + c := &commentCuts{lines: make([][]commentCut, len(lines)), drop: make([]bool, len(lines))} + off := 0 + for i, line := range lines { + mline := masked[off : off+len(line)] + off += len(line) + 1 + for k := 0; k < len(line); k++ { + if mline[k] == line[k] { + continue + } + from, to := k, k+1 + for j := to; j < len(line); j++ { + if mline[j] != line[j] { + to = j + 1 + } else if line[j] != ' ' && line[j] != '\t' { + break + } + } + c.lines[i] = append(c.lines[i], commentCut{from: from, to: to}) + k = to - 1 + } + if c.lines[i] != nil && strings.TrimSpace(mline) == "" { + if continued := i > 0 && bytes.HasSuffix(bytes.TrimRight(lines[i-1], " \t\r"), []byte("\\")); continued { + c.lines[i] = []commentCut{{from: 0, to: len(line)}} + } else { + c.drop[i] = true + } + } + } + for i := range lines { + if !c.drop[i] { + c.rows = append(c.rows, i) + } + } + return c +} + +// apply returns src with the cuts applied. src must have the lines the +// cuts were computed from (any offset-preserving copy does). +func (c *commentCuts) apply(src []byte) []byte { + lines := bytes.Split(src, []byte("\n")) + out := make([]byte, 0, len(src)) + first := true + for i, line := range lines { + if i < len(c.drop) && c.drop[i] { + continue + } + if !first { + out = append(out, '\n') + } + first = false + prev := 0 + if i < len(c.lines) { + for _, cut := range c.lines[i] { + if cut.to > len(line) { + break + } + out = append(out, line[prev:cut.from]...) + out = append(out, ' ') + prev = cut.to + } + } + out = append(out, line[prev:]...) + } + return out +} + +// row maps a 1-based line of the cut copy to the source. +func (c *commentCuts) row(line int) int { + if line >= 1 && line <= len(c.rows) { + return c.rows[line-1] + 1 + } + return line +} + +// column maps column col of cut line row (0-based) back to the source. +func (c *commentCuts) column(row, col int) int { + if row < 0 || row >= len(c.rows) { + return col + } + orig, k := 0, 0 + for _, cut := range c.lines[c.rows[row]] { + keep := cut.from - orig + if col < k+keep { + return orig + col - k + } + k += keep + if col == k { + return cut.from + } + k++ + orig = cut.to + } + return orig + col - k +} + +// restore maps every line of syms (extracted from text, the cut copy) +// back to the source and replaces each body with the same stretch of +// uncut (the source as text was before the cut), comments included. A +// body that is not a slice of text keeps its text. +func (c *commentCuts) restore(syms []astkit.Symbol, text, uncut []byte) { + starts := func(b []byte) []int { + out := []int{0} + for i, ch := range b { + if ch == '\n' { + out = append(out, i+1) + } + } + return out + } + cutStarts, uncutStarts := starts(text), starts(uncut) + if len(cutStarts) != len(c.rows) || len(uncutStarts) != len(c.drop) { + return + } + // pos is the source offset of column col of cut line row (0-based). + pos := func(row, col int) int { + orig := c.rows[row] + end := len(uncut) + if orig+1 < len(uncutStarts) { + end = uncutStarts[orig+1] - 1 + } + return min(uncutStarts[orig]+c.column(row, col), end) + } + for i := range syms { + s := &syms[i] + if row := s.Span.Start - 1; s.Body != "" && row >= 0 && row < len(cutStarts) { + if at := bytes.Index(text[cutStarts[row]:], []byte(s.Body)); at >= 0 { + from := cutStarts[row] + at + endRow := row + strings.Count(s.Body, "\n") + to := from + len(s.Body) - 1 // last byte + if endRow < len(cutStarts) { + a := pos(row, from-cutStarts[row]) + b := pos(endRow, to-cutStarts[endRow]) + 1 + if a < b && b <= len(uncut) { + s.Body = string(uncut[a:b]) + } + } + } + } + s.Span.Start, s.Span.End = c.row(s.Span.Start), c.row(s.Span.End) + for k := range s.CallSites { + s.CallSites[k].Line = c.row(s.CallSites[k].Line) + } + for k := range s.AttrSites { + s.AttrSites[k].Line = c.row(s.AttrSites[k].Line) + } + } +} diff --git a/internal/parser/engine.go b/internal/parser/engine.go index 7681263..a6655c1 100644 --- a/internal/parser/engine.go +++ b/internal/parser/engine.go @@ -10,6 +10,7 @@ import ( "regexp" "strings" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) @@ -188,6 +189,16 @@ func extractImports(language string, content string) []string { return imports } + // Fallback (unsupported grammar, parse failure or timeout): the + // line patterns read only code. Paths are string literals, so for the + // languages that quote them only comments are blanked; the others + // blank literals too (a docstring line "import os" is prose). + switch language { + case "go", "typescript", "tsx", "javascript", "c", "cpp", "php": + content = textmask.MaskComments(language, content) + default: + content = textmask.Mask(language, content) + } if language == "go" { return extractGoImports(content) } @@ -288,7 +299,10 @@ func extractGoImports(content string) []string { } // extractBody returns the end line number (1-indexed, inclusive) and full body text -// starting from startIdx (0-indexed into lines). +// starting from startIdx (0-indexed into lines). Callers pass the scanMask +// lines (comments and literals blanked) so a brace or dedent inside a +// comment, string, text block or docstring does not end the body, and cut +// the source body by the returned line. func extractBody(lines []string, startIdx int, language string) (endLine int, body string) { switch language { case "go", "typescript", "tsx", "javascript", "java", "rust", "c", "cpp", "csharp": @@ -302,55 +316,38 @@ func extractBody(lines []string, startIdx int, language string) (endLine int, bo // extractBraceBody scans forward from startIdx, tracking brace depth, // and returns when the opening brace is balanced (depth returns to 0). +// lines are masked (scanMask), so every brace counted is code. The scan +// gives up after maxLines lines of code; blank lines (comment lines are +// blank once masked) do not count, so a comment cannot move the cut. func extractBraceBody(lines []string, startIdx int) (endLine int, body string) { - var bodyLines []string depth := 0 opened := false const maxLines = 500 - limit := startIdx + maxLines - if limit > len(lines) { - limit = len(lines) + limit := len(lines) + for i, code := startIdx, 0; i < len(lines); i++ { + if strings.TrimSpace(lines[i]) != "" { + if code == maxLines { + limit = i + break + } + code++ + } } for i := startIdx; i < limit; i++ { line := lines[i] - bodyLines = append(bodyLines, line) - - // Count braces, naively ignoring strings/comments (good enough for typical Go/TS/Java) - inString := false - var stringChar byte for j := 0; j < len(line); j++ { - ch := line[j] - if inString { - if ch == '\\' { - j++ // skip escaped char - continue - } - if ch == stringChar { - inString = false - } - continue - } - if ch == '"' || ch == '\'' || ch == '`' { - inString = true - stringChar = ch - continue - } - // Inline comment - if ch == '/' && j+1 < len(line) && line[j+1] == '/' { - break - } - if ch == '{' { + switch line[j] { + case '{': depth++ opened = true - } else if ch == '}' { + case '}': depth-- } } - if opened && depth <= 0 { - return i + 1, strings.Join(bodyLines, "\n") + return i + 1, strings.Join(lines[startIdx:i+1], "\n") } } @@ -358,7 +355,7 @@ func extractBraceBody(lines []string, startIdx int) (endLine int, body string) { if !opened { return startIdx + 1, lines[startIdx] } - return startIdx + len(bodyLines), strings.Join(bodyLines, "\n") + return limit, strings.Join(lines[startIdx:limit], "\n") } // extractIndentBody collects Python body lines based on indentation. @@ -372,8 +369,12 @@ func extractIndentBody(lines []string, startIdx int) (endLine int, body string) var bodyLines []string bodyLines = append(bodyLines, startLine) + // The window counts code lines only. Callers pass masked lines, so + // comment and docstring lines are blank here; counting them let a + // comment move where a recovered body was cut off. const maxLines = 200 - for i := startIdx + 1; i < len(lines) && i < startIdx+maxLines; i++ { + code := 1 + for i := startIdx + 1; i < len(lines); i++ { line := lines[i] trimmed := strings.TrimSpace(line) if trimmed == "" { @@ -381,9 +382,10 @@ func extractIndentBody(lines []string, startIdx int) (endLine int, body string) continue } indent := len(line) - len(strings.TrimLeft(line, " \t")) - if indent <= baseIndent { + if indent <= baseIndent || code == maxLines { return i, strings.Join(bodyLines, "\n") } + code++ bodyLines = append(bodyLines, line) } return startIdx + len(bodyLines), strings.Join(bodyLines, "\n") @@ -450,7 +452,8 @@ func symbolPatterns(language string) []symbolPattern { // Free function: return-type name( — anchored to avoid matching variable decls {regexp.MustCompile(`^(?:[\w*&:<>\s]+\s+)+\*?([A-Za-z_][A-Za-z0-9_:]*)\s*\([^;]*$`), core.KindFunction, "", false}, {regexp.MustCompile(`^\s*(?:typedef\s+)?struct\s+([A-Za-z_][A-Za-z0-9_]*)\s*[{;]`), core.KindStruct, "", false}, - {regexp.MustCompile(`^\s*class\s+([A-Za-z_][A-Za-z0-9_]*)\b`), core.KindClass, "", false}, + // `class GTEST_API_ RE {`: the export macro is not the name. + {regexp.MustCompile(`^\s*class\s+(?:[A-Z][A-Z0-9]*_[A-Z0-9_]*\s+)?([A-Za-z_][A-Za-z0-9_]*)\b`), core.KindClass, "", false}, {regexp.MustCompile(`^\s*enum\s+(?:class\s+)?([A-Za-z_][A-Za-z0-9_]*)\b`), core.KindEnum, "", false}, {regexp.MustCompile(`^\s*namespace\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{`), core.KindNamespace, "", false}, } @@ -486,22 +489,25 @@ func symbolPatterns(language string) []symbolPattern { // missed is added. This prevents a partially-typed function from disappearing // from the index entirely while the developer is writing it. func extractSymbols(language, filePath, blobSHA, content string, fileImports []string) []core.SymbolRecord { - astSyms, ok, hasErrors := extractSymbolsFromAST(language, filePath, blobSHA, []byte(content), fileImports) + ast := extractASTForMerge(language, filePath, blobSHA, []byte(content), fileImports) + astSyms, ok, hasErrors := ast.syms, ast.ok, ast.hasErrors if !ok { - syms := extractSymbolsRegex(language, filePath, blobSHA, content, fileImports) + scan := scanMask(language, content) + syms := extractSymbolsRegexScan(language, filePath, blobSHA, content, scan, fileImports) if language == "cpp" { - syms = enrichCppNamespaces(syms, content) + syms = enrichCppNamespaces(syms, scan) } attachDocstrings(language, content, syms) return syms } if !hasErrors { if language == "c" || language == "cpp" { - regexSyms := extractSymbolsRegex(language, filePath, blobSHA, content, fileImports) + scan := scanMask(language, content) + regexSyms := extractSymbolsRegexScan(language, filePath, blobSHA, content, scan, fileImports) if language == "cpp" { regexSyms = dropCppNamespaceTwins(astSyms, regexSyms) n := len(astSyms) - combined := enrichCppNamespaces(append(append([]core.SymbolRecord(nil), astSyms...), regexSyms...), content) + combined := enrichCppNamespaces(append(append([]core.SymbolRecord(nil), astSyms...), regexSyms...), scan) astSyms, regexSyms = combined[:n], combined[n:] } astSyms = mergeSymbolsByShape(astSyms, regexSyms) @@ -511,14 +517,17 @@ func extractSymbols(language, filePath, blobSHA, content string, fileImports []s } // Syntax errors present — supplement AST results with regex to recover symbols // that fell inside ERROR subtrees (e.g. a function being actively typed). - regexSyms := extractSymbolsRegex(language, filePath, blobSHA, content, fileImports) + // JSX text is prose the textmask lexer reads as code; the partial tree + // still knows it, so it is blanked by node range. + scan := blankByteRanges(scanMask(language, content), ast.opaque) + regexSyms := extractSymbolsRegexScan(language, filePath, blobSHA, content, scan, fileImports) for idx := range regexSyms { regexSyms[idx].Annotations = append(regexSyms[idx].Annotations, "syntax-recovery") } if language == "cpp" { regexSyms = dropCppNamespaceTwins(astSyms, regexSyms) n := len(astSyms) - combined := enrichCppNamespaces(append(append([]core.SymbolRecord(nil), astSyms...), regexSyms...), content) + combined := enrichCppNamespaces(append(append([]core.SymbolRecord(nil), astSyms...), regexSyms...), scan) astSyms, regexSyms = combined[:n], combined[n:] } merged := mergeSymbols(astSyms, regexSyms) @@ -529,6 +538,43 @@ func extractSymbols(language, filePath, blobSHA, content string, fileImports []s return merged } +// statementKeywords precede a call in a statement, never a declared name: +// `new JArray(1, 2),` and `return Build(x);` are calls. The C# and Java +// method patterns accept any word before the name as a return type, so a +// recovered file's call lines became methods named after the callee. +var statementKeywords = map[string]bool{"new": true, "return": true, "await": true, "throw": true, + "else": true, "yield": true, "case": true, "goto": true, "is": true, "as": true, "in": true, + "using": true, "lock": true, "typeof": true, "nameof": true, "sizeof": true, "default": true, + "when": true, "assert": true, "do": true} + +// statementCallMatch reports whether a method-pattern match is a call +// statement: the word right before the name is a statement keyword. +func statementCallMatch(language, match, name string) bool { + if language != "csharp" && language != "java" { + return false + } + head := match[:strings.LastIndex(match, name)] + fields := strings.Fields(head) + return len(fields) > 0 && statementKeywords[fields[len(fields)-1]] +} + +// blankByteRanges returns s with the bytes of every [start, end) range +// replaced by spaces, newlines kept. +func blankByteRanges(s string, ranges [][2]int) string { + if len(ranges) == 0 { + return s + } + out := []byte(s) + for _, r := range ranges { + for k := r[0]; k < r[1] && k < len(out); k++ { + if out[k] != '\n' && out[k] != '\r' { + out[k] = ' ' + } + } + } + return string(out) +} + // dropCppNamespaceTwins removes the line scanner's copy of every namespace // the AST also reports (same name, same first line). The copy's span is // wrong: with clang-format's `} // namespace foo` its text runs past the @@ -562,8 +608,12 @@ func enrichRecoveredClassParents(symbols []core.SymbolRecord) { if symbol.ParentSymbol != "" || (symbol.Kind != core.KindMethod && symbol.Kind != core.KindConstructor) { continue } + // The innermost enclosing class is the one that opens later, then + // the one that closes earlier: the same answer as the narrowest + // span for proper nesting, but recovered spans can overlap without + // nesting, and there span widths let a comment line elsewhere in + // one of them pick the other (as enrichCppNamespaces). best := -1 - bestWidth := int(^uint(0) >> 1) for parentIdx := range symbols { parent := &symbols[parentIdx] switch parent.Kind { @@ -574,8 +624,13 @@ func enrichRecoveredClassParents(symbols []core.SymbolRecord) { if parent.FilePath != symbol.FilePath || parent.Span.Start > symbol.Span.Start || parent.Span.End < symbol.Span.End { continue } - if width := parent.Span.End - parent.Span.Start; width < bestWidth { - best, bestWidth = parentIdx, width + if best < 0 { + best = parentIdx + continue + } + cur := &symbols[best] + if parent.Span.Start > cur.Span.Start || parent.Span.Start == cur.Span.Start && parent.Span.End < cur.Span.End { + best = parentIdx } } if best < 0 { @@ -612,7 +667,9 @@ func cppStripTemplateArgs(name string) string { return strings.TrimSpace(out.String()) } -func enrichCppNamespaces(symbols []core.SymbolRecord, content string) []core.SymbolRecord { +// enrichCppNamespaces qualifies symbols by their enclosing namespaces and +// attaches the file's namespace bindings. scan is the file's scanMask. +func enrichCppNamespaces(symbols []core.SymbolRecord, scan string) []core.SymbolRecord { type scope struct { index int parent int @@ -624,10 +681,23 @@ func enrichCppNamespaces(symbols []core.SymbolRecord, content string) []core.Sym scopes = append(scopes, scope{index: i, parent: -1}) } } + // inner reports whether namespace a is a closer scope than b, both + // enclosing the same lines: the later opening, then the earlier + // close, then the shorter text. Recovered spans can overlap without + // nesting (`namespace testing {` 10417-11159, `namespace internal {` + // 10418-11161); comparing line counts there let a comment line + // elsewhere in the span pick the other one. + inner := func(a, b *core.SymbolRecord) bool { + if a.Span.Start != b.Span.Start { + return a.Span.Start > b.Span.Start + } + if a.Span.End != b.Span.End { + return a.Span.End < b.Span.End + } + return len(a.RawText) < len(b.RawText) + } for i := range scopes { child := symbols[scopes[i].index] - bestWidth := int(^uint(0) >> 1) - bestRawWidth := int(^uint(0) >> 1) for j := range scopes { if i == j { continue @@ -636,16 +706,12 @@ func enrichCppNamespaces(symbols []core.SymbolRecord, content string) []core.Sym if parent.Span.Start > child.Span.Start || parent.Span.End < child.Span.End { continue } - width := parent.Span.End - parent.Span.Start - rawWidth := len(parent.RawText) - strictlyContains := width > child.Span.End-child.Span.Start || - (rawWidth > len(child.RawText) && strings.Contains(parent.RawText, child.RawText)) + strictlyContains := parent.Span.Start < child.Span.Start || parent.Span.End > child.Span.End || + (len(parent.RawText) > len(child.RawText) && strings.Contains(parent.RawText, child.RawText)) if !strictlyContains { continue } - if width < bestWidth || (width == bestWidth && rawWidth < bestRawWidth) { - bestWidth = width - bestRawWidth = rawWidth + if best := scopes[i].parent; best < 0 || inner(&parent, &symbols[scopes[best].index]) { scopes[i].parent = j } } @@ -676,7 +742,7 @@ func enrichCppNamespaces(symbols []core.SymbolRecord, content string) []core.Sym path := scopePath(i) symbols[scopes[i].index].QualifiedName = path } - bindings := cppFileNamespaceBindings(content) + bindings := cppFileNamespaceBindings(scan) for i := range symbols { symbol := &symbols[i] symbol.Annotations = append(symbol.Annotations, bindings...) @@ -684,21 +750,21 @@ func enrichCppNamespaces(symbols []core.SymbolRecord, content string) []core.Sym continue } namespace := "" - bestWidth := int(^uint(0) >> 1) - bestRawWidth := int(^uint(0) >> 1) + best := -1 for j := range scopes { - ns := symbols[scopes[j].index] + ns := &symbols[scopes[j].index] if ns.Span.Start <= symbol.Span.Start && ns.Span.End >= symbol.Span.End { - width, rawWidth := ns.Span.End-ns.Span.Start, len(ns.RawText) - if width < bestWidth || (width == bestWidth && rawWidth < bestRawWidth) { - bestWidth = width - bestRawWidth = rawWidth + if best < 0 || inner(ns, &symbols[scopes[best].index]) { + best = j namespace = scopePath(j) } } } owner, name := symbol.ParentSymbol, symbol.Name - if match := cppQualifiedCallableRe.FindStringSubmatch(symbol.Signature); len(match) == 3 { + // A line-scanned symbol's Signature is its raw source line, trailing + // comment included: `void helper(int x) { // see Widget::draw(x)` + // must not become Widget::draw. + if match := cppQualifiedCallableRe.FindStringSubmatch(textmask.Mask("cpp", symbol.Signature)); len(match) == 3 { owner, name = cppStripTemplateArgs(match[1]), strings.ReplaceAll(match[2], " ", "") } else if split := strings.LastIndex(symbol.Name, "::"); split >= 0 { owner, name = symbol.Name[:split], symbol.Name[split+2:] @@ -763,11 +829,14 @@ var ( cppUsingTypeLineRe = regexp.MustCompile(`^using\s+([A-Za-z_][A-Za-z0-9_:]*)::([A-Za-z_][A-Za-z0-9_]*)\s*;`) ) -func cppFileNamespaceBindings(content string) []string { - clean := stripCppCommentsAndStrings(content) +// cppFileNamespaceBindings reads file-scope using-directives, namespace +// aliases and using-declarations from scan (scanMask: comments, literals and +// #if 0 branches blanked; a digit separator or `#error don't` no longer +// opens a quote that swallows the rest of the file). +func cppFileNamespaceBindings(scan string) []string { depth := 0 var out []string - for _, line := range strings.Split(clean, "\n") { + for _, line := range strings.Split(scan, "\n") { trimmed := strings.TrimSpace(line) if depth == 0 { switch { @@ -796,57 +865,6 @@ func cppFileNamespaceBindings(content string) []string { return out } -func stripCppCommentsAndStrings(content string) string { - out := []byte(content) - for i := 0; i < len(out); { - if i+1 < len(out) && out[i] == '/' && out[i+1] == '/' { - for i < len(out) && out[i] != '\n' { - out[i] = ' ' - i++ - } - continue - } - if i+1 < len(out) && out[i] == '/' && out[i+1] == '*' { - out[i], out[i+1] = ' ', ' ' - i += 2 - for i+1 < len(out) && !(out[i] == '*' && out[i+1] == '/') { - if out[i] != '\n' { - out[i] = ' ' - } - i++ - } - if i+1 < len(out) { - out[i], out[i+1] = ' ', ' ' - i += 2 - } - continue - } - if out[i] == '\'' || out[i] == '"' { - quote := out[i] - out[i] = ' ' - i++ - for i < len(out) { - if out[i] == '\\' && i+1 < len(out) { - out[i], out[i+1] = ' ', ' ' - i += 2 - continue - } - end := out[i] == quote - if out[i] != '\n' { - out[i] = ' ' - } - i++ - if end { - break - } - } - continue - } - i++ - } - return string(out) -} - // mergeSymbols returns the union of astSyms and regexSyms, preferring AST // results for any symbol name that appears in both. Regex-only symbols are // appended at the end — they cover declarations inside ERROR subtrees. @@ -908,7 +926,7 @@ func cFamilyMarkPrototype(s *core.SymbolRecord) { return } } - body := stripCppCommentsAndStrings(s.RawText) + body := textmask.Mask(s.Language, s.RawText) semi := strings.IndexByte(body, ';') brace := strings.IndexByte(body, '{') if semi < 0 || (brace >= 0 && brace < semi) { @@ -1048,7 +1066,7 @@ func symbolShapeKey(s core.SymbolRecord) string { // (`//`, `///`, `/** ... */`). // - Python: triple-quoted string as the first statement of the symbol body. func attachDocstrings(language, content string, symbols []core.SymbolRecord) { - lines := strings.Split(content, "\n") + var lines, code []string for i := range symbols { sym := &symbols[i] if sym.Docstring != "" || sym.Span.Start <= 0 { @@ -1058,37 +1076,64 @@ func attachDocstrings(language, content string, symbols []core.SymbolRecord) { sym.Docstring = pythonDocstring(sym.RawText) continue } - sym.Docstring = precedingCommentBlock(lines, sym.Span.Start) + if lines == nil { + lines = strings.Split(content, "\n") + code = commentMaskedLines(language, content) + } + sym.Docstring = precedingCommentBlock(lines, code, sym.Span.Start) } } -func precedingCommentBlock(lines []string, startLine int) string { +// precedingCommentBlock returns the doc comment directly above startLine: +// a `/** ... */` block or a run of `//` / `///` lines. code is lines with +// comments blanked (commentMaskedLines; nil when the language has no +// lexer), so a comment line is one that is blank in code but not in lines. +// The block is found by walking up through comment lines only: a code line +// (`int x; /* trailing */`, `const glob = "src/**";`) ends the walk and is +// never part of a docstring, and a plain `/* */` block is not a doc +// comment. +func precedingCommentBlock(lines, code []string, startLine int) string { idx := startLine - 2 // line just above the symbol (0-indexed) if idx < 0 || idx >= len(lines) { return "" } + isComment := func(i int) bool { + t := strings.TrimSpace(lines[i]) + if t == "" { + return false + } + if code == nil || i >= len(code) { + return strings.HasPrefix(t, "//") || strings.HasPrefix(t, "/*") || strings.HasPrefix(t, "*") + } + return strings.TrimSpace(code[i]) == "" + } + if !isComment(idx) { + return "" + } - // /** ... */ block: walk upward to find opening "/**". + // /** ... */ block: walk upward through its lines to the opening "/*". if strings.HasSuffix(strings.TrimSpace(lines[idx]), "*/") { end := idx - for idx >= 0 && !strings.Contains(lines[idx], "/**") { - idx-- - } - if idx < 0 { - return "" + for ; idx >= 0 && isComment(idx); idx-- { + line := strings.TrimSpace(lines[idx]) + if strings.HasPrefix(line, "/*") { + if !strings.HasPrefix(line, "/**") { + return "" + } + return cleanBlockComment(strings.Join(lines[idx:end+1], "\n")) + } } - return cleanBlockComment(strings.Join(lines[idx:end+1], "\n")) + return "" } // Contiguous //-style or ///-style comments (and Rust ///). var collected []string - for idx >= 0 { + for ; idx >= 0 && isComment(idx); idx-- { line := strings.TrimSpace(lines[idx]) - if !(strings.HasPrefix(line, "//") || strings.HasPrefix(line, "///")) { + if !strings.HasPrefix(line, "//") { break } collected = append([]string{stripLineComment(line)}, collected...) - idx-- } if len(collected) == 0 { return "" @@ -1160,30 +1205,43 @@ func pythonDocstring(body string) string { // extractSymbolsRegex is the regex-based fallback extractor. func extractSymbolsRegex(language, filePath, blobSHA, content string, fileImports []string) []core.SymbolRecord { + return extractSymbolsRegexScan(language, filePath, blobSHA, content, scanMask(language, content), fileImports) +} + +// extractSymbolsRegexScan is extractSymbolsRegex over a precomputed +// scanMask(language, content). Patterns match, braces and indentation are +// counted, and export keywords are read on the masked copy (line numbers +// kept); signatures and bodies still come from the source. Syntax recovery +// merges every regex name the AST missed, and the Java patterns are +// unanchored: Javadoc prose such as " * Deserializer class that can ..." +// became a class named "that" spanning hundreds of lines, and a text block, +// raw string, template literal or heredoc holding code became symbols. +func extractSymbolsRegexScan(language, filePath, blobSHA, content, scan string, fileImports []string) []core.SymbolRecord { patterns := symbolPatterns(language) if len(patterns) == 0 { return nil } if language == "go" { - return extractGoSymbols(filePath, blobSHA, content, fileImports) + return extractGoSymbols(filePath, blobSHA, content, scan, fileImports) } if language == "c" || language == "cpp" { - return extractCFamilySymbols(language, filePath, blobSHA, content, fileImports) + return extractCFamilySymbols(language, filePath, blobSHA, content, scan, fileImports) } lines := strings.Split(content, "\n") + clean := strings.Split(scan, "\n") var symbols []core.SymbolRecord for i := 0; i < len(lines); i++ { line := lines[i] - trimmed := strings.TrimSpace(line) + trimmed := strings.TrimSpace(clean[i]) if trimmed == "" || strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "#") { continue } for _, pattern := range patterns { - matches := pattern.regex.FindStringSubmatch(line) + matches := pattern.regex.FindStringSubmatch(clean[i]) if len(matches) < 2 { continue } @@ -1192,6 +1250,9 @@ func extractSymbolsRegex(language, filePath, blobSHA, content string, fileImport if name == "" { continue } + if pattern.kind == core.KindMethod && statementCallMatch(language, matches[0], name) { + continue + } qualifiedName := name if pattern.qualifier != "" { @@ -1201,7 +1262,8 @@ func extractSymbolsRegex(language, filePath, blobSHA, content string, fileImport qualifiedName = parentSymbol + "." + name } - endLine, body := extractBody(lines, i, language) + endLine, _ := extractBody(clean, i, language) + body := strings.Join(lines[i:endLine], "\n") symbol := core.SymbolRecord{ ID: fmt.Sprintf("%s::%s@%s", filePath, qualifiedName, blobSHA), FilePath: filePath, @@ -1212,7 +1274,7 @@ func extractSymbolsRegex(language, filePath, blobSHA, content string, fileImport QualifiedName: qualifiedName, Signature: strings.TrimSpace(line), Span: core.LineRange{Start: i + 1, End: endLine}, - Exports: isExported(language, name, line), + Exports: isExported(language, name, clean[i]), RawText: body, ParentSymbol: parentSymbol, Imports: fileImports, @@ -1227,15 +1289,17 @@ func extractSymbolsRegex(language, filePath, blobSHA, content string, fileImport // extractCFamilySymbols handles top-level C/C++ declarations plus simple // in-class C++ method and constructor declarations. -func extractCFamilySymbols(language, filePath, blobSHA, content string, fileImports []string) []core.SymbolRecord { +func extractCFamilySymbols(language, filePath, blobSHA, content, scan string, fileImports []string) []core.SymbolRecord { patterns := symbolPatterns(language) lines := strings.Split(content, "\n") - // Patterns match, and braces are counted, on a copy with comments and - // string literals blanked (line numbers kept): the scanner read - // ` * Copyright (c) 2009` in license headers as a function named - // Copyright, and prose in block comments as functions spanning hundreds - // of lines. Signatures and bodies still come from the source. - clean := strings.Split(blankCFamilyComments(content), "\n") + // Patterns match, and braces are counted, on scan (scanMask: comments, + // literals, inactive #if 0 branches and directive lines blanked, line + // numbers kept): the scanner read ` * Copyright (c) 2009` in license + // headers as a function named Copyright, prose in block comments as + // functions spanning hundreds of lines, and a raw string or a multi-line + // #define body as definitions. Signatures and bodies still come from + // the source. + clean := strings.Split(scan, "\n") var symbols []core.SymbolRecord for i := 0; i < len(lines); i++ { @@ -1363,60 +1427,83 @@ var cFamilyReservedName = map[string]bool{ "throw": true, "noexcept": true, "static_assert": true, "alignas": true, } -// blankCFamilyComments returns content with every comment and string or -// character literal replaced by spaces; newlines stay, so line numbers and -// line lengths are unchanged. Unlike stripCppCommentsAndStrings a literal -// never runs past the end of its line, so an apostrophe in prose -// (`#error don't`) or a digit separator cannot swallow the rest of the file. -func blankCFamilyComments(content string) string { - out := []byte(content) - for i := 0; i < len(out); { - switch { - case out[i] == '/' && i+1 < len(out) && out[i+1] == '/': - for i < len(out) && out[i] != '\n' { - out[i] = ' ' - i++ - } - case out[i] == '/' && i+1 < len(out) && out[i+1] == '*': - out[i], out[i+1] = ' ', ' ' - i += 2 - for i < len(out) && !(out[i] == '*' && i+1 < len(out) && out[i+1] == '/') { - if out[i] != '\n' { - out[i] = ' ' - } - i++ - } - if i+1 < len(out) { - out[i], out[i+1] = ' ', ' ' - i += 2 - } else { - i = len(out) +// scanMask returns the copy of a whole file that the line scanners match +// on: comments, string literals (raw strings, text blocks, heredocs, +// template literals), PHP inline HTML and inactive `#if 0` / `#if false` +// branches are blanked by astkit textmask, keeping every byte offset and +// newline, so a line index or column into the result indexes content. In +// C, C++ and Objective-C the lines of every preprocessor directive, +// backslash continuations included, are blanked too: a multi-line +// `#define` body is macro text, not declarations, and its braces must not +// move a brace count. Without this the scanners read Javadoc prose, raw +// strings and macro bodies as classes and functions. +func scanMask(language, content string) string { + if !textmask.Supported(language) { + return content + } + masked := textmask.MaskFile(language, textmask.MaskInactivePreprocessor(language, content)) + switch language { + case "c", "cpp", "objc": + masked = blankDirectiveLines(masked) + } + return masked +} + +// blankDirectiveLines blanks every preprocessor directive line of masked +// (comments already blanked), with its backslash-continued lines. +func blankDirectiveLines(masked string) string { + var out []byte + continued := false + for ls := 0; ls < len(masked); { + le := strings.IndexByte(masked[ls:], '\n') + if le < 0 { + le = len(masked) + } else { + le += ls + } + trimmed := strings.TrimSpace(masked[ls:le]) + directive := continued || strings.HasPrefix(trimmed, "#") + continued = directive && strings.HasSuffix(trimmed, "\\") + if directive && trimmed != "" { + if out == nil { + out = []byte(masked) } - case out[i] == '"' || (out[i] == '\'' && !(i > 0 && isHexDigitByte(out[i-1]) && i+1 < len(out) && isHexDigitByte(out[i+1]))): - quote := out[i] - i++ - for i < len(out) && out[i] != '\n' { - if out[i] == '\\' && i+1 < len(out) && out[i+1] != '\n' { - out[i], out[i+1] = ' ', ' ' - i += 2 - continue - } - if out[i] == quote { - i++ - break + for k := ls; k < le; k++ { + if out[k] != '\r' { + out[k] = ' ' } - out[i] = ' ' - i++ } - default: - i++ } + ls = le + 1 + } + if out == nil { + return masked } return string(out) } -func isHexDigitByte(b byte) bool { - return (b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F') +// commentMaskedLines returns content's lines with only comments blanked +// (string literals kept), for telling comment lines from code lines. +func commentMaskedLines(language, content string) []string { + if !textmask.Supported(language) { + return nil + } + if language == "php" { + // MaskComments lexes from byte 0 as code; a PHP file starts in + // inline HTML, where an apostrophe would open a string. textmask + // has no comments-only file mode, so the HTML before the first + // `` HTML is handled. + if open := strings.Index(content, " 0 { + head := []byte(content[:open]) + for k, c := range head { + if c != '\n' && c != '\r' { + head[k] = ' ' + } + } + content = string(head) + content[open:] + } + } + return strings.Split(textmask.MaskComments(language, content), "\n") } // extractNameAndParent returns (name, parentSymbol) from regex matches. @@ -1434,9 +1521,14 @@ func extractNameAndParent(matches []string, pattern symbolPattern) (string, stri // function/method body we can skip i past the end of that body, preventing // local variables inside the body (e.g. "var req struct{...}") from being // mistakenly extracted as package-level symbols. -func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) []core.SymbolRecord { +func extractGoSymbols(filePath, blobSHA, content, scan string, fileImports []string) []core.SymbolRecord { patterns := symbolPatterns("go") lines := strings.Split(content, "\n") + // Declarations are matched, and braces counted, on scan (comments and + // literals blanked, line numbers kept): a func or type inside a block + // comment or a raw string is not a declaration, and `const (` inside + // one would put every following line in block mode. + clean := strings.Split(scan, "\n") var symbols []core.SymbolRecord // Pre-compile block-member regex once. @@ -1446,10 +1538,10 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ inVarBlock := false for i := 0; i < len(lines); i++ { - line := lines[i] + line := clean[i] trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "//") { + if trimmed == "" { continue } @@ -1479,11 +1571,11 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ Kind: core.KindConst, Name: name, QualifiedName: name, - Signature: strings.TrimSpace(line), + Signature: strings.TrimSpace(lines[i]), Span: core.LineRange{Start: i + 1, End: i + 1}, Exports: isExported("go", name, line), - RawText: strings.TrimSpace(line), - TokenEstimate: estimateTokens(line), + RawText: strings.TrimSpace(lines[i]), + TokenEstimate: estimateTokens(lines[i]), }) } continue @@ -1499,11 +1591,11 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ Kind: core.KindVariable, Name: name, QualifiedName: name, - Signature: strings.TrimSpace(line), + Signature: strings.TrimSpace(lines[i]), Span: core.LineRange{Start: i + 1, End: i + 1}, Exports: isExported("go", name, line), - RawText: strings.TrimSpace(line), - TokenEstimate: estimateTokens(line), + RawText: strings.TrimSpace(lines[i]), + TokenEstimate: estimateTokens(lines[i]), }) } continue @@ -1530,7 +1622,8 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ qualifiedName = parentSymbol + "." + name } - endLine, body := extractBody(lines, i, "go") + endLine, _ := extractBody(clean, i, "go") + body := strings.Join(lines[i:endLine], "\n") symbols = append(symbols, core.SymbolRecord{ ID: fmt.Sprintf("%s::%s@%s", filePath, qualifiedName, blobSHA), @@ -1540,7 +1633,7 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ Kind: pattern.kind, Name: name, QualifiedName: qualifiedName, - Signature: strings.TrimSpace(line), + Signature: strings.TrimSpace(lines[i]), Span: core.LineRange{Start: i + 1, End: endLine}, Exports: isExported("go", name, line), RawText: body, @@ -1551,7 +1644,7 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ // re-extracted. During syntax recovery an unclosed function may swallow // every later top-level declaration; keep scanning so column-zero // declarations after the edit point remain visible. - if bracesBalanced(body) { + if bracesBalanced(clean[i:endLine]) { i = endLine - 1 } break @@ -1560,27 +1653,14 @@ func extractGoSymbols(filePath, blobSHA, content string, fileImports []string) [ return symbols } -func bracesBalanced(body string) bool { +// bracesBalanced reports whether the masked lines (comments and literals +// blanked) open no brace or close every brace they open. +func bracesBalanced(masked []string) bool { depth := 0 opened := false - for _, line := range strings.Split(body, "\n") { - inString := false - var quote rune - for idx, char := range line { - if inString { - if char == quote && (idx == 0 || line[idx-1] != '\\') { - inString = false - } - continue - } - if char == '/' && idx+1 < len(line) && line[idx+1] == '/' { - break - } - if char == '\'' || char == '"' || char == '`' { - inString, quote = true, char - continue - } - switch char { + for _, line := range masked { + for k := 0; k < len(line); k++ { + switch line[k] { case '{': depth++ opened = true diff --git a/internal/parser/engine_test.go b/internal/parser/engine_test.go index 7dc6958..9467588 100644 --- a/internal/parser/engine_test.go +++ b/internal/parser/engine_test.go @@ -1581,3 +1581,70 @@ func TestFileBlobSHA_RejectsNonRegularAndOversize(t *testing.T) { t.Fatalf("FileBlobSHA(regular) errored: %v", err) } } + +// Syntax recovery merges regex names the AST missed; prose in comments must +// not become symbols. A broken Java file with "class that" in its Javadoc +// produced a class named "that" spanning the comment through the next +// unbalanced brace. +func TestJavaSyntaxRecoveryIgnoresCommentProse(t *testing.T) { + content := `package p; + +/** + * Deserializer class that can deserialize things. + * The methods in this class want to throw, e.g. + * public static double sqrt(double value) { + */ +public class Foo { + public Object run(Object o) { + if (o == null) { + o = "x"; + } + int broken = 0; + else if (o instanceof String) { + o = "a class named nothing"; + } + return o; + } + + public int other() { return 2; } +} +` + syms := extractSymbols("java", "Foo.java", "sha", content, nil) + names := nameIndex(syms) + for _, phantom := range []string{"that", "want", "sqrt", "named"} { + if s, ok := names[phantom]; ok { + t.Fatalf("syntax recovery emitted %s %q from a comment or string (lines %d-%d)", s.Kind, phantom, s.Span.Start, s.Span.End) + } + } + for _, real := range []string{"Foo", "run", "other"} { + if _, ok := names[real]; !ok { + t.Fatalf("real symbol %q missing after recovery", real) + } + } +} + +func TestJavaSyntaxRecoveryStillRecoversDeclarationInBrokenRegion(t *testing.T) { + content := "/** Holder class for things. */\npublic class Holder {\n void bad() {\n work(\n public void good() { keep(); }\n}\n" + syms := extractSymbols("java", "Holder.java", "sha", content, nil) + names := nameIndex(syms) + if _, ok := names["good"]; !ok { + t.Fatal("declaration after the broken method was not recovered") + } + if _, ok := names["for"]; ok { + t.Fatal("comment prose recovered as class \"for\"") + } +} + +func TestPythonRegexIgnoresDocstringProse(t *testing.T) { + content := "def f(:\n \"\"\"Explain things.\n\n class attribute is shared;\n def example(x):\n \"\"\"\n return 1 # class comment\n\ndef g():\n return 2\n" + syms := extractSymbols("python", "m.py", "sha", content, nil) + names := nameIndex(syms) + for _, phantom := range []string{"attribute", "example", "comment"} { + if _, ok := names[phantom]; ok { + t.Fatalf("python recovery emitted %q from a docstring or comment", phantom) + } + } + if _, ok := names["g"]; !ok { + t.Fatal("real function g missing") + } +} diff --git a/internal/parser/js_alias_e2e_test.go b/internal/parser/js_alias_e2e_test.go index ebcba7a..d3771c9 100644 --- a/internal/parser/js_alias_e2e_test.go +++ b/internal/parser/js_alias_e2e_test.go @@ -111,3 +111,35 @@ func languageForPath(path string) string { } return "typescript" } + +// Comment/string-as-code audit (2026-10-10): import alias patterns read the +// statement with comments masked; `/* b as c */` binds nothing. +func TestImportAliasesIgnoreComments(t *testing.T) { + js := extractImports("typescript", "import { a as x, /* b as c */ } from './m';\n") + py := extractImports("python", "from m import (\n a as x, # b as c\n b,\n)\n") + for _, imp := range js { + if strings.Contains(imp, "c=") { + t.Errorf("typescript: alias from a comment: %q in %v", imp, js) + } + } + if !contains(js, core.JSImportAlias("x", "./m#a")) { + t.Errorf("typescript: real alias missing: %v", js) + } + for _, imp := range py { + if strings.Contains(imp, ":c=") { + t.Errorf("python: alias from a comment: %q in %v", imp, py) + } + } + if !contains(py, core.PythonImportBinding(1, "b", "m#b")) { + t.Errorf("python: real binding b missing: %v", py) + } +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} diff --git a/internal/parser/jsreexports.go b/internal/parser/jsreexports.go index 2e7bc30..154ed05 100644 --- a/internal/parser/jsreexports.go +++ b/internal/parser/jsreexports.go @@ -8,10 +8,30 @@ import ( "sort" "strings" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) -var jsExportClauseRE = regexp.MustCompile(`(?s)\bexport\s+(?:type\s+)?\{([^}]*)\}(?:\s*from\s*["']([^"']+)["'])?`) +// jsExportClauseRE matches an export clause in masked source (comments and +// string literals blanked); jsExportFromRE matches the `from` that may +// follow it, whose module string is read from the original source. +var ( + jsExportClauseRE = regexp.MustCompile(`(?s)\bexport\s+(?:type\s+)?\{([^}]*)\}`) + jsExportFromRE = regexp.MustCompile(`^\s*from\b`) +) + +// jsMaskLanguage is the textmask language key for a TS/JS path. +func jsMaskLanguage(path string) string { + switch strings.ToLower(filepath.Ext(path)) { + case ".tsx": + return "tsx" + case ".ts", ".mts", ".cts": + return "typescript" + case ".jsx": + return "jsx" + } + return "javascript" +} func jsSourceExt(path string) bool { switch strings.ToLower(filepath.Ext(path)) { @@ -86,14 +106,25 @@ func scanJSExports(root string, prefilter []byte, name string) (map[string][]cor // jsExportSpecifiersIn returns the specifiers of one file; name == "" means all. func jsExportSpecifiersIn(rel string, orig []byte, name string) []core.JSExportSpecifier { var out []core.JSExportSpecifier - // Comments are blanked (newlines kept) so offsets and lines stay valid: - // zod's `/** @deprecated Use z.gte() */ _gte as _min` is a specifier. - src := blankJSComments(orig) + // Comments and literals are blanked (offsets kept) so lines stay valid + // and an export clause inside a string, template literal or comment + // never matches: zod's `/** @deprecated Use z.gte() */ _gte as _min` + // is a specifier. + src := []byte(textmask.Mask(jsMaskLanguage(rel), string(orig))) for _, m := range jsExportClauseRE.FindAllSubmatchIndex(src, -1) { bodyStart, bodyEnd := m[2], m[3] source := "" - if m[4] >= 0 { - source = string(orig[m[4]:m[5]]) + if f := jsExportFromRE.FindIndex(src[m[1]:]); f != nil { + // The module string is masked in src; read it from orig. + q := m[1] + f[1] + for q < len(orig) && (orig[q] == ' ' || orig[q] == '\t' || orig[q] == '\n' || orig[q] == '\r') { + q++ + } + if q < len(orig) && (orig[q] == '"' || orig[q] == '\'') { + if end := bytes.IndexByte(orig[q+1:], orig[q]); end > 0 { + source = string(orig[q+1 : q+1+end]) + } + } } offset := bodyStart for _, item := range bytes.Split(src[bodyStart:bodyEnd], []byte(",")) { @@ -126,54 +157,6 @@ func jsExportSpecifiersIn(rel string, orig []byte, name string) []core.JSExportS return out } -// blankJSComments replaces comment bytes and string-literal contents with -// spaces (quotes and newlines kept), so an export clause inside a string or -// comment never matches and every offset still indexes the original source. -func blankJSComments(src []byte) []byte { - out := append([]byte(nil), src...) - var quote byte - for i := 0; i < len(out); i++ { - c := out[i] - if quote != 0 { - switch { - case c == '\\': - out[i] = ' ' - if i+1 < len(out) && out[i+1] != '\n' { - i++ - out[i] = ' ' - } - case c == quote || c == '\n' && quote != '`': - quote = 0 - case c != '\n': - out[i] = ' ' - } - continue - } - switch { - case c == '"' || c == '\'' || c == '`': - quote = c - case c == '/' && i+1 < len(out) && out[i+1] == '/': - for ; i < len(out) && out[i] != '\n'; i++ { - out[i] = ' ' - } - case c == '/' && i+1 < len(out) && out[i+1] == '*': - out[i], out[i+1] = ' ', ' ' - i += 2 - for ; i < len(out); i++ { - if out[i] == '*' && i+1 < len(out) && out[i+1] == '/' { - out[i], out[i+1] = ' ', ' ' - i++ - break - } - if out[i] != '\n' { - out[i] = ' ' - } - } - } - } - return out -} - func lineAt(src []byte, at int) string { start := bytes.LastIndexByte(src[:at], '\n') + 1 end := bytes.IndexByte(src[at:], '\n') diff --git a/internal/parser/jsreexports_test.go b/internal/parser/jsreexports_test.go index 39d7dc6..0df5f9a 100644 --- a/internal/parser/jsreexports_test.go +++ b/internal/parser/jsreexports_test.go @@ -63,3 +63,16 @@ func TestJSExportIndexMatchesPerNameScan(t *testing.T) { t.Fatalf("index = %+v, scan = %+v", idx, one) } } + +// Comment/string-as-code audit (2026-10-10): a template literal holding an +// export clause is text, and a regex literal holding `/*` or a backtick +// does not hide a real clause after it. +func TestJSExportSpecifiersIgnoreLiterals(t *testing.T) { + src := "const doc = `\nexport { a as fromTemplate }\n`;\n" + + "const re = /[/*]/;\nconst tick = /`/;\n" + + "export { a as real } from './m';\n" + got := jsExportSpecifiersIn("x.ts", []byte(src), "a") + if len(got) != 1 || got[0].Exported != "real" || got[0].Source != "./m" || got[0].Line != 6 { + t.Fatalf("specifiers = %+v, want only a as real from ./m on line 6", got) + } +} diff --git a/internal/parser/languages.go b/internal/parser/languages.go index 52a310e..01cc757 100644 --- a/internal/parser/languages.go +++ b/internal/parser/languages.go @@ -30,7 +30,7 @@ var securityExtensions = map[string]bool{ // detector fixes, synthesized records): the indexer re-extracts every file // when the stored stamp differs, because the blobSHA cache is otherwise // blind to binary upgrades and serves stale symbols indefinitely. -const ExtractorVersion = "2026-10-05.1" +const ExtractorVersion = "2026-10-10.1" // DetectLanguage returns the Grove language tag for a file path. // Returns "" for unsupported or security-excluded files. diff --git a/internal/parser/memberrefs.go b/internal/parser/memberrefs.go index 2e483c5..f5d7212 100644 --- a/internal/parser/memberrefs.go +++ b/internal/parser/memberrefs.go @@ -103,6 +103,14 @@ func (e *Engine) MemberOccurrences(root, name string, languages []string) ([]cor continue } lang := astkit.DetectLanguage(t.abs, string(src)) + if strings.EqualFold(filepath.Ext(t.abs), ".h") { + // Parse a header as the language it is indexed in: + // astkit's own .h sniff reads markers in comments + // and `#ifdef __cplusplus` guards as C++. + if key, ok := languageToKey(DetectLanguageContent(t.rel, src)); ok { + lang = key + } + } if lang == astkit.LangObjC && !want[lang] && strings.EqualFold(filepath.Ext(t.abs), ".h") { lang = astkit.LangC } diff --git a/internal/parser/recovered_parent_test.go b/internal/parser/recovered_parent_test.go new file mode 100644 index 0000000..0fd93f0 --- /dev/null +++ b/internal/parser/recovered_parent_test.go @@ -0,0 +1,24 @@ +package parser + +import ( + "testing" + + "github.com/provasign/grove/internal/core" +) + +// Recovered class spans can overlap without nesting. The method's class is +// the one that opens later, whatever the two spans' line counts: a comment +// line inside Outer (making it wider or narrower) must not change it. +func TestEnrichRecoveredClassParents_OverlapPicksLaterOpening(t *testing.T) { + for _, outerEnd := range []int{60, 52, 100} { + syms := []core.SymbolRecord{ + {FilePath: "A.cs", Kind: core.KindClass, Name: "Outer", Span: core.LineRange{Start: 10, End: outerEnd}}, + {FilePath: "A.cs", Kind: core.KindClass, Name: "Inner", Span: core.LineRange{Start: 11, End: 55}}, + {FilePath: "A.cs", Kind: core.KindMethod, Name: "Run", QualifiedName: "Run", Span: core.LineRange{Start: 20, End: 30}}, + } + enrichRecoveredClassParents(syms) + if got := syms[2].ParentSymbol; got != "Inner" { + t.Fatalf("Outer ends at %d: parent = %q, want Inner", outerEnd, got) + } + } +} diff --git a/internal/parser/sniff.go b/internal/parser/sniff.go index 69f877d..d0b1689 100644 --- a/internal/parser/sniff.go +++ b/internal/parser/sniff.go @@ -5,6 +5,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/provasign/astkit/textmask" ) // Content-based detection for mainframe artifacts. Estate exports are routinely @@ -30,6 +32,9 @@ var ( sniffLevelLine = regexp.MustCompile(`(?m)^.{7}\s*\d{2}\s+[A-Z0-9][A-Z0-9-]*`) sniffPIC = regexp.MustCompile(`(?i)\bPIC(?:TURE)?\s+`) sniffCPPHeader = regexp.MustCompile(`(?m)^\s*(?:template\s*<|namespace\b|class\s+[A-Za-z_]|extern\s+"C")`) + // sniffCPlusPlusGuard matches a conditional on __cplusplus; group 1 is + // set when it tests for its absence (`#ifndef`, `#if !defined`). + sniffCPlusPlusGuard = regexp.MustCompile(`^#\s*(?:ifdef\s+__cplusplus\b|if\s+(?:defined\s*\(?\s*__cplusplus\b|__cplusplus\b)|(ifndef\s+__cplusplus\b|if\s+!\s*defined\s*\(?\s*__cplusplus\b))`) // Objective-C headers use a small set of `@`-prefixed keywords with no // C or C++ equivalent — unambiguous even in a header carrying ordinary // C declarations alongside them (the common case: Foundation-style @@ -39,16 +44,75 @@ var ( // sniffHeader classifies a .h file's content as "objc", "cpp", or "" (plain // C, the default DetectLanguage/DetectLanguageFile already assume). +// +// The markers are matched on sniffHeaderCode, so a comment line starting +// with "namespace" or "class", an `#if 0` branch, or the C-library idiom +// `#ifdef __cplusplus / extern "C" { / #endif` does not make a C header C++. func sniffHeader(content []byte) string { - if sniffObjCHeader.Match(content) { + code := sniffHeaderCode(string(content)) + if sniffObjCHeader.MatchString(code) { return "objc" } - if sniffCPPHeader.Match(content) { + if sniffCPPHeader.MatchString(code) { return "cpp" } return "" } +// sniffHeaderCode returns content with comments and `#if 0` branches +// blanked (astkit textmask) and the C++-only branches of __cplusplus +// conditionals blanked: what those branches hold is only ever seen by a C++ +// compiler including a header written to be C as well. +func sniffHeaderCode(content string) string { + code := textmask.MaskComments("cpp", textmask.MaskInactivePreprocessor("cpp", content)) + if !strings.Contains(code, "__cplusplus") { + return code + } + lines := strings.Split(code, "\n") + // One frame per open conditional: cppOnly marks the branch being read + // as visible to C++ only; guard is 1 for `#ifdef __cplusplus`, -1 for + // `#ifndef __cplusplus`, 0 for any other conditional. + type frame struct { + guard int + cppOnly bool + } + var stack []frame + for i, line := range lines { + t := strings.TrimSpace(line) + directive := strings.HasPrefix(t, "#") + if directive { + d := strings.TrimSpace(strings.TrimPrefix(t, "#")) + switch { + case strings.HasPrefix(d, "if"): + f := frame{} + if m := sniffCPlusPlusGuard.FindStringSubmatch(t); m != nil { + f.guard = 1 + if m[1] != "" { + f.guard = -1 + } + f.cppOnly = f.guard == 1 + } + stack = append(stack, f) + case strings.HasPrefix(d, "el"): + if n := len(stack); n > 0 && stack[n-1].guard != 0 { + stack[n-1].cppOnly = stack[n-1].guard == -1 + } + case strings.HasPrefix(d, "endif"): + if n := len(stack); n > 0 { + stack = stack[:n-1] + } + } + } + for _, f := range stack { + if f.cppOnly && !directive { + lines[i] = "" + break + } + } + } + return strings.Join(lines, "\n") +} + // sniffMainframe classifies head bytes as "cobol", "jcl", or "". func sniffMainframe(head []byte) string { s := string(head) diff --git a/internal/parser/sniff_test.go b/internal/parser/sniff_test.go index b96d9f4..ce60041 100644 --- a/internal/parser/sniff_test.go +++ b/internal/parser/sniff_test.go @@ -79,6 +79,13 @@ func TestDetectLanguageFile_HeaderSniff(t *testing.T) { }{ {"plain-c.h", "int add(int a, int b);\n", "c"}, {"cpp.h", "namespace geo {\nclass Shape {};\n}\n", "cpp"}, + // The C-library guard is C: only a C++ compiler sees its branch. + {"c-guard.h", "#ifdef __cplusplus\nextern \"C\" {\n#endif\nint add(int a, int b);\n#ifdef __cplusplus\n}\n#endif\n", "c"}, + {"c-guard-defined.h", "#if defined(__cplusplus)\nextern \"C\" {\n#endif\nint add(int a, int b);\n", "c"}, + {"c-guard-ifndef.h", "#ifndef __cplusplus\ntypedef int bool;\n#else\nclass Wrapper;\n#endif\n", "c"}, + {"cpp-extern-c.h", "extern \"C\" {\nint add(int a, int b);\n}\n", "cpp"}, + {"cpp-after-guard.h", "#ifdef __cplusplus\nextern \"C\" {\n#endif\nint add(int a, int b);\n#ifdef __cplusplus\n}\n#endif\nnamespace geo { int f(); }\n", "cpp"}, + {"if0-namespace.h", "#if 0\nnamespace old {\n#endif\nint add(int a, int b);\n", "c"}, {"objc-interface.h", "@interface Person : NSObject\n@end\n", "objc"}, {"objc-protocol.h", "@protocol Greeter\n- (void)greet;\n@end\n", "objc"}, {"objc-forward-decl.h", "@class Person;\nint helper(void);\n", "objc"}, diff --git a/internal/parser/treesitter.go b/internal/parser/treesitter.go index d6b6b69..823136c 100644 --- a/internal/parser/treesitter.go +++ b/internal/parser/treesitter.go @@ -21,6 +21,7 @@ import ( "github.com/provasign/astkit" "github.com/provasign/astkit/strategies" + "github.com/provasign/astkit/textmask" "github.com/provasign/grove/internal/core" ) @@ -114,9 +115,30 @@ func (e *Engine) ParseTree(language string, src []byte) error { // syntax errors, the extracted symbols are returned with hasErrors=true so the // caller may merge them with regex-extracted ones. func extractSymbolsFromAST(language, filePath, blobSHA string, src []byte, fileImports []string) (syms []core.SymbolRecord, ok bool, hasErrors bool) { + r := extractASTForMerge(language, filePath, blobSHA, src, fileImports) + return r.syms, r.ok, r.hasErrors +} + +// astResult is extractSymbolsFromAST's result plus what the regex recovery +// pass needs from the partial tree. +type astResult struct { + syms []core.SymbolRecord + ok, hasErrors bool + // opaque holds the byte ranges of JSX text when the tree has errors: + // prose the textmask lexer reads as code (`class names are applied` + // became a class named names). + opaque [][2]int +} + +func extractASTForMerge(language, filePath, blobSHA string, src []byte, fileImports []string) (res astResult) { + syms, ok, hasErrors, opaque := extractASTSymbols(language, filePath, blobSHA, src, fileImports) + return astResult{syms: syms, ok: ok, hasErrors: hasErrors, opaque: opaque} +} + +func extractASTSymbols(language, filePath, blobSHA string, src []byte, fileImports []string) (syms []core.SymbolRecord, ok bool, hasErrors bool, opaque [][2]int) { key, supported := languageToKey(language) if !supported { - return nil, false, false + return nil, false, false, nil } eng, reg := bridge() ctx, cancel := context.WithTimeout(context.Background(), parseTimeout) @@ -125,26 +147,58 @@ func extractSymbolsFromAST(language, filePath, blobSHA string, src []byte, fileI // macro lines keeps every byte offset, so the tree parsed from the // blanked copy still addresses the original text. text := src - if key == astkit.LangC || key == astkit.LangCPP { - if blanked := blankCFamilyMacroLines(src); blanked != nil { + // The grammar parses `#if 0` / `#if false` branches as live code; dead + // code there (often unfinished, `int ghost(int a) {` with no close) + // became symbols and its errors swallowed the real declarations after + // the #endif. The parse sees those branches blanked (offsets kept). + // live is the source with only that masking, for the alt parse below. + live := text + // Set when a broken C-family file is parsed with its comments cut: + // uncutText is what text was before the cut. + var commentCuts *commentCuts + var uncutText []byte + if preprocessedLanguage(language) || key == astkit.LangObjC { + if masked := textmask.MaskInactivePreprocessor(language, string(src)); masked != string(src) { + src = []byte(masked) + live = src + } + } + if cFamilyKey(key) { + if blanked := blankCFamilyMacroLines(language, src); blanked != nil { src = blanked } } tree, err := eng.Parse(ctx, key, src) if err != nil { - return nil, false, false + return nil, false, false, nil } if tree == nil { // No grammar for this language. Text-capable strategies (astkit // TextStrategy) extract from src alone; everything else has no // AST path. Unreachable for grammar-backed languages. if !reg.TextCapable(key) { - return nil, false, false + return nil, false, false, nil } } else { - defer tree.Close() hasErrors = tree.RootNode().HasError() - if hasErrors && preprocessedLanguage(language) { + if hasErrors && cFamilyKey(key) { + // Error recovery prices the bytes and lines it skips, + // comments included: a comment line inside a broken region + // changed which declarations the recovered tree kept (a whole + // AFNetworking .m file flipped between two different ERROR + // soups). A broken file is parsed from code alone (see + // parseBrokenCFamily); restore maps lines back and puts the + // comments back into symbol bodies. A file that parses + // cleanly keeps its comment nodes, which signatures cut. + if b := parseBrokenCFamily(ctx, eng, key, language, src, text, live, true); b != nil { + tree.Close() + tree, src, text = b.tree, b.src, b.text + hasErrors = tree.RootNode().HasError() + commentCuts, uncutText = b.cuts, b.uncutText + } + } + defer func() { tree.Close() }() + if hasErrors && preprocessedLanguage(language) && commentCuts == nil { // `#if A ... else ... #else ... else ... #endif` hands the // grammar two else branches; tree-sitter recovers with ERROR // nodes that swallow call sites. Re-parse with the directives @@ -157,8 +211,9 @@ func extractSymbolsFromAST(language, filePath, blobSHA string, src []byte, fileI tree.Close() tree, src = altTree, alt // Same line lengths as alt: macro blanking - // never touches a directive line. - text = blankPreprocessorBranches(text) + // never touches a directive line, and live has + // the same directive lines as src. + text = blankPreprocessorBranches(live) hasErrors = altErrs > 0 } else { altTree.Close() @@ -169,19 +224,29 @@ func extractSymbolsFromAST(language, filePath, blobSHA string, src []byte, fileI } akSyms, err := reg.Extract(key, tree, text) if err != nil { - return nil, false, false + return nil, false, false, nil + } + if commentCuts != nil { + commentCuts.restore(akSyms, text, uncutText) } syms = make([]core.SymbolRecord, 0, len(akSyms)) + var jsCode []string + if (key == astkit.LangJavaScript || key == astkit.LangTypeScript || key == astkit.LangTSX) && + bytes.Contains(src, []byte("export default")) { + jsCode = strings.Split(textmask.Mask(language, string(src)), "\n") + } for _, s := range akSyms { projected := projectSymbol(s, filePath, blobSHA, language, fileImports) - if (key == astkit.LangJavaScript || key == astkit.LangTypeScript || key == astkit.LangTSX) && - jsDefaultExportAt(src, s.Span.Start) { + if jsCode != nil && jsDefaultExportAt(jsCode, s.Span.Start) { projected.Modifiers = append(projected.Modifiers, "default-export") } syms = append(syms, projected) } - if key == astkit.LangTypeScript || key == astkit.LangTSX { - tsExtendOverloadSpans(syms, src) + if tree != nil && (key == astkit.LangTypeScript || key == astkit.LangTSX) { + tsExtendOverloadSpans(syms, tree.RootNode(), string(text)) + } + if tree != nil && hasErrors && (key == astkit.LangJavaScript || key == astkit.LangTSX) { + opaque = jsxTextRanges(tree.RootNode()) } if tree != nil && (key == astkit.LangJavaScript || key == astkit.LangTypeScript || key == astkit.LangTSX) { if topLevel := jsTopLevelSymbol(tree.RootNode(), src, filePath, blobSHA, language, fileImports); topLevel != nil { @@ -205,7 +270,26 @@ func extractSymbolsFromAST(language, filePath, blobSHA string, src []byte, fileI Imports: append([]string(nil), fileImports...), }) } - return syms, true, hasErrors + return syms, true, hasErrors, opaque +} + +// jsxTextRanges returns the byte ranges of every jsx_text node under n. +func jsxTextRanges(n *sitter.Node) [][2]int { + var out [][2]int + var walk func(*sitter.Node) + walk = func(n *sitter.Node) { + if n.Type() == "jsx_text" { + out = append(out, [2]int{int(n.StartByte()), int(n.EndByte())}) + return + } + for i := 0; i < int(n.ChildCount()); i++ { + if c := n.Child(i); c != nil { + walk(c) + } + } + } + walk(n) + return out } var jsCallableContainers = map[string]bool{ @@ -331,6 +415,50 @@ func jsTopLevelSymbol(root *sitter.Node, src []byte, filePath, blobSHA, language // extractImportsFromAST parses imports through astkit when a strategy exists. // Regex extraction remains the fallback for unsupported languages or parse // failures. +// dropComments removes comment text from an import path. Astkit takes a +// grouped import's path from the source (`use std::{ // note\n io, }`), and a +// comment there became part of the import string and of the crate heads +// read from it. Each comment goes with the blanks after it (or, at a line +// end, before it), and a line left empty is dropped, so the result matches +// the same import written without comments. Text without comments comes +// back unchanged. +func dropComments(language, s string) string { + masked := textmask.MaskComments(language, s) + if masked == s { + return s + } + var b strings.Builder + for i := 0; i < len(s); { + if masked[i] == s[i] { + b.WriteByte(s[i]) + i++ + continue + } + // A comment run: masked bytes, then the blanks that follow it. + for i < len(s) && masked[i] != s[i] { + i++ + } + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + } + lines := strings.Split(b.String(), "\n") + orig := strings.Split(s, "\n") + kept := lines[:0] + for i, line := range lines { + if line == orig[i] { + kept = append(kept, line) + continue + } + trimmed := strings.TrimRight(line, " \t") + if trimmed != strings.TrimRight(orig[i], " \t") && strings.TrimSpace(trimmed) == "" { + continue // the line held only a comment + } + kept = append(kept, trimmed) + } + return strings.Join(kept, "\n") +} + func extractImportsFromAST(language string, src []byte) ([]string, bool) { key, supported := languageToKey(language) if !supported { @@ -343,6 +471,16 @@ func extractImportsFromAST(language string, src []byte) ([]string, bool) { if err != nil || (tree == nil && !reg.TextCapable(key)) { return nil, false } + if tree != nil && cFamilyKey(key) && tree.RootNode().HasError() { + // Read a broken file's includes from code alone, as its symbols + // are (see extractASTSymbols): `#include` lines inside an ERROR + // region came and went with nearby comments. The candidates that + // blank directives would blank the includes. + if b := parseBrokenCFamily(ctx, eng, key, language, src, src, src, false); b != nil { + tree.Close() + tree, src = b.tree, b.text + } + } if tree != nil { defer tree.Close() } @@ -356,14 +494,14 @@ func extractImportsFromAST(language string, src []byte) ([]string, bool) { if imp.Path == "" { continue } - importPath := imp.Path + importPath := dropComments(language, imp.Path) if key == astkit.LangRust && strings.HasPrefix(strings.TrimSpace(imp.Raw), "pub ") { // Astkit intentionally normalizes Path to the imported target, but // Grove also needs to know that the dependency is re-exported. Rust // scope traversal follows dependencies transitively only across public // facade edges; losing `pub` here made `grep::printer::X` stop at the // grep facade instead of reaching grep_printer. - importPath = "pub use " + imp.Path + importPath = "pub use " + importPath } if key == astkit.LangGo && imp.Alias != "" && imp.Alias != "_" && imp.Alias != "." { alias := core.GoImportAlias(imp.Alias, imp.Path) @@ -386,7 +524,9 @@ func extractImportsFromAST(language string, src []byte) ([]string, bool) { // are dropped there. The "#" form never reaches consumers. for _, imp := range akImports { aliases := map[string]string{} - for _, match := range pythonAliasedImportRE.FindAllStringSubmatch(imp.Raw, -1) { + // Raw is the statement's source text; a comment inside a + // parenthesized import (`# b as c`) is not an alias. + for _, match := range pythonAliasedImportRE.FindAllStringSubmatch(textmask.Mask(language, imp.Raw), -1) { aliases[match[1]] = match[2] } if len(imp.Names) == 0 { @@ -421,28 +561,31 @@ func extractImportsFromAST(language string, src []byte) ([]string, bool) { } if key == astkit.LangJavaScript || key == astkit.LangTypeScript || key == astkit.LangTSX { for _, imp := range akImports { - if strings.HasPrefix(strings.TrimSpace(imp.Raw), "export ") { + // The alias patterns read imp.Raw with comments and strings + // masked: `import { a, /* b as c */ } from "x"` binds no c. + raw := textmask.Mask(language, imp.Raw) + if strings.HasPrefix(strings.TrimSpace(raw), "export ") { encoded := core.JSImportReExport(imp.Path) if !seen[encoded] { seen[encoded] = true imports = append(imports, encoded) } } - if match := jsDefaultImportRE.FindStringSubmatch(imp.Raw); len(match) == 2 { + if match := jsDefaultImportRE.FindStringSubmatch(raw); len(match) == 2 { encoded := core.JSImportAlias(match[1], imp.Path+"#default") if !seen[encoded] { seen[encoded] = true imports = append(imports, encoded) } } - if match := jsNamespaceImportRE.FindStringSubmatch(imp.Raw); len(match) == 2 { + if match := jsNamespaceImportRE.FindStringSubmatch(raw); len(match) == 2 { encoded := core.JSImportAlias(match[1], imp.Path) if !seen[encoded] { seen[encoded] = true imports = append(imports, encoded) } } - match := jsNamedImportRE.FindStringSubmatch(imp.Raw) + match := jsNamedImportRE.FindStringSubmatch(raw) if len(match) != 2 { continue } @@ -476,24 +619,23 @@ func lastDottedSegment(value string) string { return value } -func jsDefaultExportAt(src []byte, line int) bool { - if line < 1 { +// jsDefaultExportAt reports whether the symbol starting on line is the +// default export: `export default` on its line or alone on the line above. +// code is the file's lines with comments and literals masked, so +// `return "export default"` does not count. +func jsDefaultExportAt(code []string, line int) bool { + if line < 1 || line > len(code) { return false } - lines := bytes.Split(src, []byte{'\n'}) - if line > len(lines) { - return false - } - if bytes.Contains(lines[line-1], []byte("export default")) { + if strings.Contains(code[line-1], "export default") { return true } for previous := line - 2; previous >= 0; previous-- { - trimmed := bytes.TrimSpace(lines[previous]) + trimmed := strings.TrimSpace(code[previous]) if len(trimmed) == 0 { continue } - return bytes.Equal(trimmed, []byte("export default")) || - bytes.Equal(trimmed, []byte("export default;")) + return trimmed == "export default" || trimmed == "export default;" } return false } @@ -549,6 +691,11 @@ func symID(filePath, qualifiedName, blobSHA string) string { return fmt.Sprintf("%s::%s@%s", filePath, qualifiedName, blobSHA) } +// cFamilyKey reports whether key is C, C++ or Objective-C. +func cFamilyKey(key astkit.LanguageKey) bool { + return key == astkit.LangC || key == astkit.LangCPP || key == astkit.LangObjC +} + func preprocessedLanguage(language string) bool { return language == "csharp" || language == "c" || language == "cpp" } @@ -617,25 +764,63 @@ var cMacroLineRe = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*[A-Z][A-Z0-9_]*:?$`) // cMacroCallLineRe matches a line opening an upper-case macro invocation. var cMacroCallLineRe = regexp.MustCompile(`^[A-Z][A-Z0-9_]*_[A-Z0-9_]*\s*\(`) +// cMacroEnumRe matches a `typedef NS_ENUM(...) { ... };` (or NS_OPTIONS, +// CF_ENUM, ...) in masked text. +var cMacroEnumRe = regexp.MustCompile(`\btypedef\s+(?:NS|CF)_(?:ENUM|OPTIONS|CLOSED_ENUM|ERROR_ENUM)\s*\([^(){};]*\)\s*\{[^{}]*\}\s*;?`) + +// cClassKeyMacroRe matches an upper-case macro between a class key and +// the class name. +var cClassKeyMacroRe = regexp.MustCompile(`(?:^|[\s;{}>])(?:class|struct|union)\s+([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s+[A-Za-z_]\w*\s*(?:[:{]|final\b|$)`) + +// cSpecifierKeywordRe matches a leading declaration specifier keyword and +// the blanks after it. +var cSpecifierKeywordRe = regexp.MustCompile(`^(?:static|inline|virtual|explicit|friend|constexpr|extern)\s+`) + +// objcTrailingWordMacroRe matches an upper-case macro word ending an +// Objective-C method declaration. +var objcTrailingWordMacroRe = regexp.MustCompile(`[\w)]\s+([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s*;\s*$`) + +// cTrailingMacroRe matches an upper-case macro call after a name or `)`. +var cTrailingMacroRe = regexp.MustCompile(`[A-Za-z0-9_)]\s+([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s*\(`) + +// cClassHeaderRe matches a masked line opening a class, struct or union +// body, brace on it or on the next line; cNamespaceHeaderRe one opening a +// namespace or an +// `extern "C"` block (the string is blanked), brace on it or on the next +// line. +var cClassHeaderRe = regexp.MustCompile(`^(?:template\s*<.*>\s*)?(?:class|struct|union)\s+[^;()=]*(?:\{|$)`) + +var cNamespaceHeaderRe = regexp.MustCompile(`^(?:(?:inline\s+)?namespace(?:\s+[A-Za-z_][\w:]*)?|extern\s+"\s*")\s*(?:\{|$)`) + // cMacroSpecifierRe matches an upper-case specifier macro in front of a // declaration: `FMT_CONSTEXPR auto f()`, `FMT_API void g()`, -// `FMT_CONSTEXPR typed_node(const Arg& arg)`. -var cMacroSpecifierRe = regexp.MustCompile(`^([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s+(?:auto|void|bool|char|int|long|short|unsigned|signed|float|double|const|constexpr|static|inline|virtual|explicit|friend|typename|struct|class|enum|template|extern|~?[A-Za-z_][A-Za-z0-9_]*\s*\()`) +// `FMT_CONSTEXPR typed_node(const Arg& arg)`, `DOCTEST_CONSTEXPR +// size_type len = 24;` (a type and a declarator after it: a macro there +// cannot be a type itself). +var cMacroSpecifierRe = regexp.MustCompile(`^([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s+(?:auto|void|bool|char|int|long|short|unsigned|signed|float|double|const|constexpr|static|inline|virtual|explicit|friend|typename|struct|class|enum|template|extern|~?[A-Za-z_][A-Za-z0-9_]*\s*\(|[A-Za-z_][\w:]*(?:<[^;{}()]*>)?[\s*&]+(?:operator\b|[A-Za-z_]\w*\s*[=;({\[,]))`) // blankCFamilyMacroLines returns src with the macro uses tree-sitter cannot // expand replaced by spaces, or nil when there is none. Line numbers and -// byte offsets are unchanged. Three shapes are blanked: +// byte offsets are unchanged. Seven shapes are blanked: // // - a line that is a lone upper-case identifier -// (`NLOHMANN_JSON_NAMESPACE_BEGIN`, `G_BEGIN_DECLS`, `Q_OBJECT`); +// (`NLOHMANN_JSON_NAMESPACE_BEGIN`, `G_BEGIN_DECLS`, `Q_OBJECT`, +// `NS_ASSUME_NONNULL_BEGIN`); // - an upper-case macro invocation that is a whole line with no `;`, at -// file scope or right after a `template<...>` header +// file or class scope or right after a `template<...>` header // (`FMT_PRAGMA_GCC(push_options)`, a multi-line -// `GTEST_DISABLE_MSC_WARNINGS_PUSH_(...)`, `JSON_HEDLEY_NON_NULL(1)`), +// `GTEST_DISABLE_MSC_WARNINGS_PUSH_(...)`, `JSON_HEDLEY_NON_NULL(1)`, +// `NS_EXTENSION_UNAVAILABLE_IOS("...")` before `@interface`), // followed by a declaration, not by `{` or an operator // (`GTEST_CHECK_(x)` / `<< "msg";` is an expression whose call must // stay); -// - an upper-case specifier macro before a declaration (`FMT_CONSTEXPR`). +// - an upper-case macro statement (`FOO_(a, b);`) at file or namespace +// scope, where no call can stand; +// - an upper-case macro call ending a declaration there +// (`... *metrics AF_API_AVAILABLE(ios(10));`); +// - an upper-case specifier macro before a declaration (`FMT_CONSTEXPR`) +// or between a class key and its name (`class GTEST_API_ RE {`); +// - a whole `typedef NS_ENUM(T, Name) { ... };`. // // nlohmann/json opens every header with a namespace macro, and the grammar // read the macro, `namespace` and the whole namespace body as one function @@ -645,15 +830,31 @@ var cMacroSpecifierRe = regexp.MustCompile(`^([A-Z][A-Z0-9]*_[A-Z0-9_]*)\s+(?:au // line looks the same, so inside braces a line qualifies only when the // previous code line does not end with `,` and the next one does not start // with `}`. Preprocessor lines (with continuations) never qualify. -func blankCFamilyMacroLines(src []byte) []byte { - clean := blankCFamilyComments(string(src)) +func blankCFamilyMacroLines(language string, src []byte) []byte { + clean := textmask.Mask(language, string(src)) + var out []byte + // `typedef NS_ENUM(NSUInteger, Mode) { ... };` is unknown to the + // grammar, and recovering from it lost the @interface after it. + // astkit reads these enums from the text (objcMacroEnums); the parse + // sees the whole typedef blanked. + if enums := cMacroEnumRe.FindAllStringIndex(clean, -1); enums != nil { + out = append([]byte(nil), src...) + cleanBytes := []byte(clean) + for _, m := range enums { + for k := m[0]; k < m[1]; k++ { + if out[k] != '\n' && out[k] != '\r' { + out[k], cleanBytes[k] = ' ', ' ' + } + } + } + clean = string(cleanBytes) + } lines := strings.Split(clean, "\n") starts := make([]int, len(lines)) for i, off := 0, 0; i < len(lines); i++ { starts[i] = off off += len(lines[i]) + 1 } - var out []byte blank := func(from, to int) { if out == nil { out = append([]byte(nil), src...) @@ -664,8 +865,26 @@ func blankCFamilyMacroLines(src []byte) []byte { } } } - braces, parens := 0, 0 - continued := false + // scopes holds one entry per open brace: a namespace or `extern "C"` + // block, whose body is declarations like file scope, a class body, or + // anything else. + const ( + otherScope = iota + namespaceScope + classScope + ) + var scopes []int + declScope := func() bool { + for _, kind := range scopes { + if kind != namespaceScope { + return false + } + } + return true + } + parens := 0 + continued, objcMemberHeader := false, false + pending := otherScope // the kind of a header line whose `{` is on the next line prevCode := "" for i := 0; i < len(lines); i++ { line := lines[i] @@ -675,20 +894,33 @@ func blankCFamilyMacroLines(src []byte) []byte { if directive { continue } + braces := len(scopes) standalone := braces == 0 || (!strings.HasSuffix(prevCode, ",") && !strings.HasPrefix(nextCodeLine(lines, i+1), "}")) if parens == 0 && standalone { if cMacroLineRe.MatchString(trimmed) { blank(starts[i], starts[i]+len(line)) continue } - if (braces == 0 || strings.HasSuffix(prevCode, ">")) && cMacroCallLineRe.MatchString(trimmed) { + if cMacroCallLineRe.MatchString(trimmed) { if end, ok := cMacroCallEnd(clean, starts[i]+strings.Index(line, "(")); ok { endLine := i for endLine+1 < len(lines) && starts[endLine+1] <= end { endLine++ } rest := strings.TrimSpace(clean[end+1 : starts[endLine]+len(lines[endLine])]) - if next := nextCodeLine(lines, endLine+1); rest == "" && (next == "" || cIdentStartByte(next[0]) || next[0] == '}') { + next := nextCodeLine(lines, endLine+1) + // A macro statement where only declarations may stand + // (`GMOCK_DEFINE_DEFAULT_ACTION_FOR_RETURN_TYPE_(float, + // 0);` in a namespace) is no call; the grammar read + // `float` as a parameter, and recovering from a run of + // them folded all of gmock.h into one ERROR. + statement := rest == ";" && declScope() + // One in a class body (`DOCTEST_DECLARE_INTERFACE(X)` + // before `virtual void f() = 0;`) declares members. + inClass := braces > 0 && scopes[braces-1] == classScope + declaration := rest == "" && (braces == 0 || inClass || strings.HasSuffix(prevCode, ">")) && + (next == "" || cIdentStartByte(next[0]) || next[0] == '}' || next[0] == '@') + if statement || declaration { blank(starts[i], starts[endLine]+len(lines[endLine])) if endLine > i { prevCode = strings.TrimSpace(lines[endLine]) @@ -699,9 +931,54 @@ func blankCFamilyMacroLines(src []byte) []byte { } } } + objcMethod := language == "objc" && declScope() && (strings.HasPrefix(trimmed, "-") || strings.HasPrefix(trimmed, "+")) + objcMember := objcMethod || (language == "objc" && declScope() && strings.HasPrefix(trimmed, "@property")) + if parens == 0 && declScope() { + // A trailing attribute macro on a declaration or definition + // header (`@property NSURLSessionTaskMetrics *metrics + // AF_API_AVAILABLE(ios(10));`, `- (void)f:(id)x + // AF_API_AVAILABLE(ios(10)) {`, `void f(void) + // FOO_DEPRECATED("x");`) is unknown to the grammar; + // recovering from a run of them folded a whole .m file into + // one ERROR. After a `)` the call ends a declarator; after a + // name only in an Objective-C member (`int FOO_F(int);` + // declares FOO_F, `typedef NS_ENUM(...)` is a type). + if m := cTrailingMacroRe.FindAllStringSubmatchIndex(line, -1); m != nil { + last := m[len(m)-1] + open := starts[i] + strings.IndexByte(line[last[2]:], '(') + last[2] + if end, ok := cMacroCallEnd(clean, open); ok && end < starts[i]+len(line) && (line[last[0]] == ')' || objcMember || objcMemberHeader) { + if rest := strings.TrimSpace(clean[end+1 : starts[i]+len(line)]); rest == ";" || rest == "{" || (rest == "" && strings.HasPrefix(nextCodeLine(lines, i+1), "{")) { + blank(starts[i]+last[2], end+1) + } + } + } + } + // `- (instancetype)initWithURL:(NSURL *)url + // NS_DESIGNATED_INITIALIZER;`: an argument-less attribute macro + // read as one more selector part. + if objcMethod || objcMemberHeader { + if m := objcTrailingWordMacroRe.FindStringSubmatchIndex(line); m != nil { + blank(starts[i]+m[2], starts[i]+m[3]) + } + } + // A method header continues over lines until its `;` or `{`. + if objcMember || objcMemberHeader { + objcMemberHeader = !strings.HasSuffix(trimmed, ";") && !strings.HasSuffix(trimmed, "{") + } if parens == 0 { + // `class GTEST_API_ RE {`, `struct DOCTEST_INTERFACE + // AssertData`: an export macro between the key and the name. + if m := cClassKeyMacroRe.FindStringSubmatchIndex(line); m != nil { + blank(starts[i]+m[2], starts[i]+m[3]) + } lead := len(line) - len(strings.TrimLeft(line, " \t")) for { + // Specifier keywords may come first + // (`static DOCTEST_CONSTEXPR size_type len = 24;`). + if k := cSpecifierKeywordRe.FindStringIndex(line[lead:]); k != nil { + lead += k[1] + continue + } m := cMacroSpecifierRe.FindStringSubmatchIndex(line[lead:]) if m == nil { break @@ -711,13 +988,20 @@ func blankCFamilyMacroLines(src []byte) []byte { lead += len(line[lead:]) - len(strings.TrimLeft(line[lead:], " \t")) } } + header := otherScope + if cNamespaceHeaderRe.MatchString(trimmed) { + header = namespaceScope + } else if cClassHeaderRe.MatchString(trimmed) { + header = classScope + } for k := 0; k < len(line); k++ { switch line[k] { case '{': - braces++ + scopes = append(scopes, max(header, pending)) + header, pending = otherScope, otherScope case '}': - if braces > 0 { - braces-- + if len(scopes) > 0 { + scopes = scopes[:len(scopes)-1] } case '(': parens++ @@ -729,6 +1013,7 @@ func blankCFamilyMacroLines(src []byte) []byte { } if trimmed != "" { prevCode = trimmed + pending = header } } return out @@ -785,32 +1070,65 @@ func countErrorNodes(n *sitter.Node) int { return c } -// tsOverloadLineRe matches one TS overload signature line: an optional -// modifier run, the member name, a parameter list, and a terminating `;` -// (no body). -func tsOverloadLineRe(name string) *regexp.Regexp { - return regexp.MustCompile(`^\s*(?:(?:export|declare|public|private|protected|static|async|abstract|override|readonly)\s+)*(?:function\s+)?` + - regexp.QuoteMeta(name) + `\s*[<(].*;\s*$`) -} - // tsExtendOverloadSpans widens a TS function or method to cover the // overload signatures stacked directly above its implementation. They are // part of its declaration -- renaming the member renames them too (h3 // EventStream.push has four) -- but carry no body, so Astkit does not -// extract them. -func tsExtendOverloadSpans(syms []core.SymbolRecord, src []byte) { - lines := strings.Split(string(src), "\n") +// extract them. An overload is a bodiless signature node +// (function_signature, method_signature, abstract_method_signature) that +// declares the same name and ends on the line directly above; a call +// statement `foo(1);` above `function foo` is not one. +func tsExtendOverloadSpans(syms []core.SymbolRecord, root *sitter.Node, src string) { + type signature struct { + name string + start int + } + byEnd := map[int][]signature{} // 1-based end line -> signatures ending there + var walk func(*sitter.Node) + walk = func(n *sitter.Node) { + switch n.Type() { + case "function_signature", "method_signature", "abstract_method_signature": + if name := n.ChildByFieldName("name"); name != nil { + outer := n + for p := n.Parent(); p != nil && (p.Type() == "export_statement" || p.Type() == "ambient_declaration"); p = p.Parent() { + outer = p + } + end := int(outer.EndPoint().Row) + 1 + byEnd[end] = append(byEnd[end], signature{name.Content([]byte(src)), int(outer.StartPoint().Row) + 1}) + } + return + } + for i := 0; i < int(n.ChildCount()); i++ { + if c := n.Child(i); c != nil { + walk(c) + } + } + } + if root == nil { + return + } + walk(root) + if len(byEnd) == 0 { + return + } + lines := strings.Split(src, "\n") for i := range syms { s := &syms[i] if (s.Kind != core.KindMethod && s.Kind != core.KindFunction) || s.Name == "" || s.Span.Start < 2 { continue } - re := tsOverloadLineRe(s.Name) start := s.Span.Start - for start > 1 && start-2 < len(lines) && re.MatchString(lines[start-2]) { - start-- + stack: + for start > 1 { + for _, sig := range byEnd[start-1] { + if sig.name == s.Name && sig.start < start { + start = sig.start + continue stack + } + } + break } - if start < s.Span.Start { + if start < s.Span.Start && s.Span.Start-1 <= len(lines) { // Keep RawText aligned with Span: line offsets into the body are // computed as line - Span.Start everywhere. s.RawText = strings.Join(lines[start-1:s.Span.Start-1], "\n") + "\n" + s.RawText diff --git a/internal/parser/ts_test_callbacks_e2e_test.go b/internal/parser/ts_test_callbacks_e2e_test.go index f39283f..b6ed596 100644 --- a/internal/parser/ts_test_callbacks_e2e_test.go +++ b/internal/parser/ts_test_callbacks_e2e_test.go @@ -86,3 +86,31 @@ func TestTSOverloadSignaturesJoinTheDeclaration(t *testing.T) { } t.Fatal("EventStream.push not extracted") } + +// Function overloads are bodiless function_signature nodes naming the same +// function (exported, multi-line); a call statement or another function's +// signature above is not one. +func TestTSFunctionOverloadsJoinTheDeclaration(t *testing.T) { + src := "export function g(a: string): void;\n" + + "export function f(a: string): void;\n" + + "export function f(\n a: number,\n): void;\n" + + "export function f(a: any) {\n return a;\n}\n" + + "f(1);\n" + + "function h(x: number) {\n return x;\n}\n" + syms, err := parser.NewEngine().ExtractContent("a.ts", []byte(src)) + if err != nil { + t.Fatal(err) + } + want := map[string][2]int{"f": {2, 8}, "h": {10, 12}} + for _, s := range syms { + if w, ok := want[s.Name]; ok { + if s.Span.Start != w[0] || s.Span.End != w[1] { + t.Errorf("%s spans %d-%d, want %d-%d", s.Name, s.Span.Start, s.Span.End, w[0], w[1]) + } + delete(want, s.Name) + } + } + if len(want) > 0 { + t.Errorf("missing %v", want) + } +} diff --git a/pkg/grove/comment_string_tools_test.go b/pkg/grove/comment_string_tools_test.go new file mode 100644 index 0000000..6a27b6e --- /dev/null +++ b/pkg/grove/comment_string_tools_test.go @@ -0,0 +1,233 @@ +package grove + +import ( + "context" + "fmt" + "strings" + "testing" +) + +// Comment/string-as-code audit (2026-10-10), tool-output findings: the +// answers of missing-implementations, rename-plan, and change-impact must +// not change when text inside comments, docstrings, or string literals +// looks like code. The graph-shaped findings are in +// internal/invariance/twin_test.go. Each test names its audit ID. + +func editLines(es []RenameEdit) []string { + var out []string + for _, e := range es { + out = append(out, fmt.Sprintf("%s:%d %s", e.FilePath, e.Line, strings.TrimSpace(e.Before))) + } + return out +} + +func hasEditLine(es []RenameEdit, file string, line int) bool { + for _, e := range es { + if e.FilePath == file && e.Line == line { + return true + } + } + return false +} + +func symNames(ss []Symbol) []string { + var out []string + for _, s := range ss { + out = append(out, s.FilePath+"::"+s.QualifiedName) + } + return out +} + +// E5: a docstring that mentions NotImplementedError must not make an +// implementation look abstract. +func TestCommentString_E5_MissingImplPythonDocstring(t *testing.T) { + for _, form := range []struct{ name, doc string }{ + {"clean", `"""Saves the record and returns 1."""`}, + {"decoy", `"""Saves the record; never raises NotImplementedError."""`}, + } { + t.Run(form.name, func(t *testing.T) { + eng := openMemberFixture(t, map[string]string{"models.py": `class Contract: + def save(self): + raise NotImplementedError + + +class BaseImpl(Contract): + def save(self): + ` + form.doc + ` + return 1 + + +class Child(BaseImpl): + pass +`}) + r, err := eng.MissingImplementations(context.Background(), "Contract.save") + if err != nil { + t.Fatal(err) + } + if len(r.Missing)+len(r.AbstractMissing) > 0 { + t.Errorf("missing=%v abstractMissing=%v (implemented=%d), want none: BaseImpl.save returns 1 and Child inherits it", + symNames(r.Missing), symNames(r.AbstractMissing), r.ImplementedCount) + } + }) + } +} + +// E7/E8: rename-plan must find edit sites on lines that also carry +// attributes, private fields, or string interpolation. +func TestCommentString_E7E8_RenamePlanSites(t *testing.T) { + cases := []struct { + id, file, src, query, newName string + wantLines []int + }{ + { + // E7: the attribute's string on the declaration line hid the + // declaration from the plan. + id: "E7-php-attribute-decl-line", file: "Svc.php", query: "Svc.act", newName: "perform", + src: `act($n - 1); + } +} +`, + wantLines: []int{3, 5}, + }, + { + // E8: "#" of a private field read as a comment start. + id: "E8-js-private-field-line", file: "store.js", query: "Store.render", newName: "draw", + src: `export class Store { + #cache; + render(n) { return n; } + load() { + this.#cache = this.render(1); + return this.#cache; + } +} +`, + wantLines: []int{3, 5}, + }, + { + id: "E8-kotlin-string-template", file: "V.kt", query: "V.label", newName: "tag", + src: `class V { + fun label(n: Int): String = "x" + n + fun show(): String = "value=${label(3)}" +} +`, + wantLines: []int{2, 3}, + }, + { + id: "E8-swift-string-interpolation", file: "V.swift", query: "V.label", newName: "tag", + src: `class V { + func label(_ n: Int) -> String { return "x" } + func show() -> String { return "value=\(label(3))" } +} +`, + wantLines: []int{2, 3}, + }, + { + id: "E8-php-string-interpolation", file: "Svc.php", query: "Svc.act", newName: "perform", + src: `act(2)}"; } +} +`, + wantLines: []int{3, 4}, + }, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + eng := openMemberFixture(t, map[string]string{tc.file: tc.src}) + r, err := eng.RenamePlan(context.Background(), tc.query, tc.newName) + if err != nil { + t.Fatal(err) + } + for _, line := range tc.wantLines { + if !hasEditLine(r.Edits, tc.file, line) { + t.Errorf("rename %s: no confirmed edit on %s:%d\nedits=%v\nambiguous=%v\nunresolved=%v", + tc.query, tc.file, line, editLines(r.Edits), editLines(r.Ambiguous), r.Unresolved) + } + } + }) + } +} + +// E9: a Go raw string holding a backslash (`\`) must not hide the receiver +// type of a later member access. +func TestCommentString_E9_ChangeImpactGoRawString(t *testing.T) { + for _, form := range []struct{ name, lit string }{ + {"clean", `"\\"`}, + {"decoy", "`\\`"}, + } { + t.Run(form.name, func(t *testing.T) { + eng := openMemberFixture(t, map[string]string{ + "go.mod": "module example.com/p\n\ngo 1.21\n", + "p.go": `package p + +import "strings" + +type Cfg struct{ Name string } + +type Other struct{ Name string } + +func mk() (*Cfg, error) { return &Cfg{}, nil } + +func Path(p string) string { + p = strings.ReplaceAll(p, ` + form.lit + `, "/") + c, _ := mk() + return c.Name + p +} +`}) + r, err := eng.ChangeImpact(context.Background(), "Cfg.Name") + if err != nil { + t.Fatal(err) + } + const site = "p.go:14" + if got := accessLines(r.Accesses); !contains(got, site) { + t.Errorf("%s (return c.Name + p) not receiver-typed: confirmed=%v ambiguous=%v", + site, got, accessLines(r.AmbiguousAccesses)) + for _, a := range r.AmbiguousAccesses { + t.Logf(" ambiguous %s:%d %s", a.FilePath, a.Line, a.Evidence) + } + } + }) + } +} + +// E4 (change-impact half): a multi-line base list must still put the +// subclass override in Base.method's family. +func TestCommentString_E4_ChangeImpactMultilineBases(t *testing.T) { + for _, form := range []struct{ name, header string }{ + {"clean", "class Multi(Base,):"}, + {"decoy", "class Multi(\n Base, # the (old) Mixin\n):"}, + } { + t.Run(form.name, func(t *testing.T) { + eng := openMemberFixture(t, map[string]string{"a.py": `class Base: + def method(self): + return 0 + + +` + form.header + ` + def method(self): + return 1 +`}) + r, err := eng.ChangeImpact(context.Background(), "Base.method") + if err != nil { + t.Fatal(err) + } + if !contains(symNames(r.Family), "a.py::Multi.method") { + t.Errorf("family=%v, want a.py::Multi.method", symNames(r.Family)) + } + }) + } +} + +func contains(xs []string, x string) bool { + for _, v := range xs { + if v == x { + return true + } + } + return false +}